Clean up the reporting from continious-discrete comparison.

Author: BenChung

Project.toml

@@ -4,12 +4,19 @@ authors = ["Ben Chung <benjamin.chung@juliahub.com> and contributors"]
 version = "1.0.0-DEV"
 
 [deps]
+CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 DiffEqDevTools = "f3b72e0c-5b89-59e1-b016-84e28bfd966d"
 DifferentialEquations = "0c46a032-eb83-5123-abaf-570d42b7fbaa"
 ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78"
 SymbolicIndexingInterface = "2efcf032-c050-4f8e-a9bb-153293bab1f5"
 
+[weakdeps]
+JuliaSimModelOptimizer = "e76a656c-db09-4a9a-8f5e-6a4d9ca4a950"
+
+[extensions]
+ModelTestingCalibration = "JuliaSimModelOptimizer"
+
 [compat]
 julia = "1.10"
 
@@ -18,9 +25,3 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
 test = ["Test", "JuliaSimModelOptimizer"]
-
-[weakdeps]
-JuliaSimModelOptimizer = "e76a656c-db09-4a9a-8f5e-6a4d9ca4a950"
-
-[extensions]
-ModelTestingCalibration = "JuliaSimModelOptimizer"

src/test/discrete/single_shooting.jl

@@ -25,9 +25,12 @@ function compare_discrete_to_continous(
 
     data_matrix = collect.(eachrow(data[!, string.(measured)]))  # lame and slow
     solution_data = sol[measured]
-    compute_error_metrics(results, solution_data, data_matrix)
-    results[:final] = recursive_mean(abs.(solution_data[end] - collect(data[end, string.(measured)])))
+    metrics = Dict{Symbol, Any}()
+    compute_error_metrics(metrics, solution_data, data_matrix)
+    metrics[:final] = recursive_mean(abs.(solution_data[end] - collect(data[end, string.(measured)])))
+    results[:metrics] = metrics
     cols = []
+    push!(cols, :timestamp => data[:, :timestamp])
     append!(cols, make_cols(namespace_symbol.((:simulated,), measured), solution_data))
     append!(cols, make_cols(namespace_symbol.((:data,), measured), data_matrix))
     results[:data] = DataFrame(cols)

test/timeseries.jl

@@ -1,5 +1,5 @@
 Base.isapprox((v1, t1)::Tuple{Vector{Float64}, Float64}, (v2, t2)::Tuple{Vector{Float64}, Float64}) = v1 ≈ v2 && t1 ≈ t2
-tol = 1e-8
+tol = 1e-6
 
 @testset "Continous" begin 
     @testset "Model-Model Mixed (self-discretized) Comparison" begin 
@@ -75,11 +75,14 @@ end
         ref_sol = solve(ref_prob, Tsit5(), reltol = 1e-8, abstol = 1e-8)
         data = DataFrame(["timestamp" => Float64[], "x(t)" => Float64[], "y(t)" => Float64[]])
         for t in LinRange(0.0, 1.0, 1000)
-            push!(data, [t, ref_sol(t, idxs=fol.x) + randn()*0.1, ref_sol(t, idxs=fol.y) + randn()*0.1])
+            push!(data, [t, ref_sol(t, idxs=fol.x) + randn() * 0.1, ref_sol(t, idxs=fol.y) * randn() * 0.1])
         end
 
-        results = ModelTesting.validate(fol, data; search_space=[fol.τ => (0.5,0.5)], params = [fol.τ => 0.5], u0 = [fol.x => 0.0])
-
-        println(results)
+        results_bad = ModelTesting.validate(fol, data; search_space=[fol.τ => (0.5,0.5)], params = [fol.τ => 0.5], u0 = [fol.x => 0.0])
+        results_good = ModelTesting.validate(fol, data; search_space=[fol.τ => (1.0,1.0)], params = [fol.τ => 1.0], u0 = [fol.x => 0.0])
+        @test results_good[:metrics][:l∞] < results_bad[:metrics][:l∞]
+        @test results_good[:metrics][:l2] < results_bad[:metrics][:l2]
+        CSV.write("results.csv", results_bad[:data])
+        display(results_good[:data])
     end
 end