Regression testing integration method

Author: BenChung

Project.toml

@@ -12,6 +12,7 @@ ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78"
 SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 SymbolicIndexingInterface = "2efcf032-c050-4f8e-a9bb-153293bab1f5"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [compat]
 CSV = "^0.10"

src/DynamicModelTestUtils.jl

@@ -1,6 +1,7 @@
 module DynamicModelTestUtils
 using DataFrames, StatsBase, LinearAlgebra
 using ModelingToolkit, SciMLBase, SymbolicIndexingInterface
+using CSV, Test
 abstract type DiscreteEvaluation end
 abstract type Metric end
 include("test/measured.jl")
@@ -10,4 +11,5 @@ include("test/compare.jl")
 include("problem_config/problem.jl")
 include("deploy/deploy.jl")
 include("deploy/restore.jl")
+include("integration/regression.jl")
 end

src/integration/regression.jl

@@ -0,0 +1,84 @@
+function default_comparison(name, sol, df; tol=1e-3, show_results=true, kwargs...)
+    cmp_df = compare(sol, df; kwargs...)
+    if show_results
+        @show cmp_df
+    end
+
+    # this is lame but is better than the alternatives
+    for (idx, var) in enumerate(cmp_df[:, :var])
+        for (cidx, col) in enumerate(names(cmp_df))
+            if col ∈ ["observed", "var"]
+                continue
+            end
+            @test cmp_df[idx, cidx] < tol # "Solution $name differs from regression data using metric $col on variable $var by $(cmp_df[idx, cidx])."
+        end
+    end
+    return cmp_df
+end
+
+
+function regression_test(
+    runs::Dict{Symbol, <:SciMLBase.AbstractTimeseriesSolution}, 
+    rootdir=currentdir(),
+    data_dir="data",
+    should_deploy="--deploy_data" ∈ ARGS;
+    make_output_dir=false,
+    clean_output_dir=false,
+    deploy_only=false,
+    restore=(go, root) -> load_data(go; root=root),
+    cmp=(name, sol, df) -> default_comparison(name, sol, df),
+    do_load=(filename) -> CSV.read(filename, DataFrame),
+    discretize=(sol) -> discretize_solution(sol),
+    save=(filename, df) -> CSV.write(filename, df),
+    do_deploy=deploy,
+    mk_filename=(symbol) -> string(symbol) * ".csv")
+    should_save = false
+    data_fulldir = joinpath(rootdir, data_dir)
+    if !isfile(data_fulldir) && make_output_dir
+        mkpath(data_fulldir)
+        should_save = true
+    elseif isfile(data_fulldir) && make_output_dir
+        if clean_output_dir
+            if !isempty(readdir(data_fulldir))
+                for rmfile in readdir(data_fulldir)
+                    @warn "Removing existing data file $rmfile"
+                    rm(rmfile)
+                end
+            end
+        end
+        should_save = true
+    end
+
+
+    if !deploy_only
+        restore(rootdir) do restored
+            cd(restored) do
+                for (name, new_sol) in runs 
+                    @assert isfile(mk_filename(name))
+                    data = do_load(mk_filename(name))
+                    cmp(name, new_sol, data)
+                end
+            end
+        end
+    end
+    if should_save
+        for (name, new_sol) in runs 
+            save(joinpath(data_fulldir, mk_filename(name)), discretize(new_sol))
+        end
+    end
+    if should_save && should_deploy
+        do_deploy(rootdir, data_dir,
+            deploy_type = DynamicModelTestUtils.FilesystemDeployConfig(rootdir, "."),
+            repo = rootdir)
+    elseif !should_save && should_deploy
+        mktempdir() do temp 
+            cd(temp) do 
+                mkdir(data_dir)
+                do_deploy(temp, data_dir,
+                    deploy_type = DynamicModelTestUtils.FilesystemDeployConfig(rootdir, "."),
+                    repo = rootdir)
+
+            end
+        end
+    end
+end

test/deployment.jl

@@ -58,7 +58,6 @@ using Git, CSV
                 full_repo_path = joinpath(pwd(), "repo")
                 mkdir("data")
                 CSV.write("data/output.csv", d1)
-                println(readdir())
                 DynamicModelTestUtils.deploy(
                     pwd(), "data",
                     deploy_type = DynamicModelTestUtils.FilesystemDeployConfig(full_repo_path, "."),
@@ -72,4 +71,54 @@ using Git, CSV
             end
         end
     end
+    @testset "RC regression" begin 
+        R = 1.0
+        C = 1.0
+        V = 1.0
+        @variables t
+        @named resistor = Resistor(R = R)
+        @named capacitor = Capacitor(C = C)
+        @named source = Voltage()
+        @named constant = Constant(k = V)
+        @named ground = Ground()
+    
+        @named sensor = PotentialSensor()
+        @named key_parameter = MeasureComponent()
+    
+        rc_eqs = [connect(constant.output, source.V)
+                connect(source.p, resistor.p)
+                connect(resistor.n, capacitor.p)
+                connect(capacitor.n, source.n, ground.g)
+                connect(sensor.p, capacitor.p)
+                sensor.phi ~ key_parameter.value]
+    
+        @named rc_model = ODESystem(rc_eqs, t,
+            systems = [resistor, capacitor, constant, source, ground, sensor, key_parameter])
+        sys = structural_simplify(rc_model)
+        prob1 = ODEProblem(sys, Pair[], (0, 10.0))
+        sol = solve(prob1, Tsit5())
+        prob2 = ODEProblem(sys, Pair[capacitor.C => 0.5], (0, 10.0))
+        sol2 = solve(prob2, Tsit5())
+        
+        println("\n\n\nstart deploy test")
+        mktempdir() do dir
+            cd(dir) do 
+                mkdir("repo")
+                run(`$(git()) -C repo init -q --bare`)
+                full_repo_path = joinpath(pwd(), "repo")
+                DynamicModelTestUtils.regression_test(Dict(:rc => sol), full_repo_path, "data", true; 
+                    deploy_only = true,
+                    make_output_dir = true)
+                println("deployed! to $full_repo_path")
+                run(`$(git()) clone -q -b data $(full_repo_path) worktree`)
+                println(readdir("worktree/"))
+                DynamicModelTestUtils.regression_test(Dict(
+                    :rc => sol
+                ), full_repo_path)
+                #DynamicModelTestUtils.regression_test(Dict(
+                #    :rc => sol2
+                #), full_repo_path)
+            end
+        end
+    end
 end