Skip to content

Commit 6f936b0

Browse files
BenChungBenChung
committed
Add support for managed data deploy & restore
1 parent ec1bd35 commit 6f936b0

File tree

6 files changed

+311
-0
lines changed

6 files changed

+311
-0
lines changed

‎Project.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ version = "0.1.1"
66
[deps]
77
CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
88
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
9+
Git = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2"
910
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
1011
ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78"
1112
SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"

‎src/DynamicModelTestUtils.jl‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,6 @@ include("test/discretize.jl")
88
include("test/instant.jl")
99
include("test/compare.jl")
1010
include("problem_config/problem.jl")
11+
include("deploy/deploy.jl")
12+
include("deploy/restore.jl")
1113
end

‎src/deploy/deploy.jl‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
using Git
2+
# this is substantially derivative of Documenter.jl's usage of git to run gh-pages
3+
abstract type DeployTarget end
4+
Base.@kwdef struct DeployConfig
5+
branch::String = ""
6+
repo::String = ""
7+
subfolder::String = ""
8+
all_ok::Bool = false
9+
end
10+
11+
@enum AuthenticationMethod SSH HTTPS
12+
authentication_method(::DeployTarget) = SSH
13+
14+
function authenticated_repo_url end
15+
post_status(cfg::Union{DeployTarget,Nothing}; kwargs...) = nothing
16+
17+
function data_key(::DeployTarget)
18+
return ENV["DATA_KEY"]
19+
end
20+
21+
function deploy_folder end
22+
23+
function currentdir()
24+
d = Base.source_dir()
25+
d === nothing ? pwd() : d
26+
end
27+
28+
const NO_KEY_ENV = Dict(
29+
"DOCUMENTER_KEY" => nothing,
30+
"DOCUMENTER_KEY_PREVIEWS" => nothing,
31+
)
32+
33+
34+
function deploy(
35+
root = currentdir(),
36+
target = "data",
37+
dirname = "";
38+
repo = error("no 'repo' keyword provided."),
39+
branch = "data",
40+
deploy_type::DeployTarget = nothing,
41+
archive = nothing
42+
)
43+
deploy_config = deploy_folder(deploy_type;
44+
branch=branch,
45+
repo=repo,
46+
dataurl=dirname)
47+
if !deploy_config.all_ok
48+
return
49+
end
50+
51+
deploy_branch = deploy_config.branch
52+
deploy_repo = deploy_config.repo
53+
deploy_subfolder = deploy_config.subfolder
54+
55+
cd(root) do
56+
sha = try
57+
readchomp(`$(git()) rev-parse --short HEAD`)
58+
catch
59+
# git rev-parse will throw an error and return code 128 if it is not being
60+
# run in a git repository, which will make run/readchomp throw an exception.
61+
# We'll assume that if readchomp fails it is due to this and set the sha
62+
# variable accordingly.
63+
"(not-git-repo)"
64+
end
65+
@debug "setting up target directory."
66+
isdir(target) || mkpath(target)
67+
startswith(realpath(target), realpath(root)) || error("""
68+
target must be a subdirectory of root, got:
69+
target: $(realpath(target))
70+
root: $(realpath(root))
71+
""")
72+
@debug "pushing new data to remote: '$deploy_repo:$deploy_branch'."
73+
mktempdir() do temp
74+
git_push(
75+
temp, deploy_repo;
76+
branch=deploy_branch, dirname=dirname, target=target,
77+
sha=sha, deploy_type=deploy_type, subfolder=deploy_subfolder, forcepush=false,
78+
archive=archive
79+
)
80+
end
81+
end
82+
end
83+
84+
function git_push(
85+
temp, repo;
86+
branch="gh-pages", dirname="", target="site", sha="",
87+
forcepush=false, deploy_type, subfolder,
88+
archive = false
89+
)
90+
dirname = isempty(dirname) ? temp : joinpath(temp, dirname)
91+
isdir(dirname) || mkpath(dirname)
92+
93+
target_dir = abspath(target)
94+
95+
# Generate a closure with common commands for ssh and https
96+
function git_commands(sshconfig=nothing)
97+
# Setup git.
98+
run(`$(git()) init`)
99+
run(`$(git()) config user.name "todo"`)
100+
run(`$(git()) config user.email "todo@example.com"`)
101+
if sshconfig !== nothing
102+
run(`$(git()) config core.sshCommand "ssh -F $(sshconfig)"`)
103+
end
104+
105+
# Fetch from remote and checkout the branch.
106+
run(`$(git()) remote add upstream $upstream`)
107+
try
108+
run(`$(git()) fetch upstream`)
109+
catch e
110+
@error """
111+
Git failed to fetch $upstream
112+
This can be caused by a DATA_KEY variable that is not correctly set up.
113+
Make sure that the environment variable is properly set up as a Base64-encoded string
114+
of the SSH private key. You may need to re-generate the keys.
115+
"""
116+
rethrow(e)
117+
end
118+
119+
try
120+
run(`$(git()) checkout -b $branch upstream/$branch`)
121+
catch e
122+
@info """
123+
Checking out $branch failed, creating a new orphaned branch.
124+
This usually happens when deploying to a repository for the first time and
125+
the $branch branch does not exist yet. The fatal error above is expected output
126+
from Git in this situation.
127+
"""
128+
@debug "checking out $branch failed with error: $e"
129+
run(`$(git()) checkout --orphan $branch`)
130+
run(`$(git()) commit --allow-empty -m "Initial empty commit for data"`)
131+
end
132+
133+
# Copy docs to `subfolder` directory.
134+
deploy_dir = subfolder === nothing ? dirname : joinpath(dirname, subfolder)
135+
gitrm_copy(target_dir, deploy_dir)
136+
137+
# Add, commit, and push the docs to the remote.
138+
run(`$(git()) add -A -- ':!.data-identity-file.tmp' ':!**/.data-identity-file.tmp'`)
139+
if !success(`$(git()) diff --cached --exit-code`)
140+
if !isnothing(archive)
141+
run(`$(git()) commit -m "build based on $sha"`)
142+
@info "Skipping push and writing repository to an archive" archive
143+
run(`$(git()) archive -o $(archive) HEAD`)
144+
elseif forcepush
145+
run(`$(git()) commit --amend --date=now -m "build based on $sha"`)
146+
run(`$(git()) push -fq upstream HEAD:$branch`)
147+
else
148+
run(`$(git()) commit -m "build based on $sha"`)
149+
run(`$(git()) push -q upstream HEAD:$branch`)
150+
end
151+
else
152+
@info "new data identical to the old -- not committing nor pushing."
153+
end
154+
end
155+
# The upstream URL to which we push new content authenticated with token
156+
upstream = authenticated_repo_url(deploy_type)
157+
try
158+
cd(() -> withenv(git_commands, NO_KEY_ENV...), temp)
159+
post_status(deploy_type; repo=repo, type="success", subfolder=subfolder)
160+
catch e
161+
@error "Failed to push:" exception=(e, catch_backtrace())
162+
post_status(deploy_type; repo=repo, type="error")
163+
rethrow(e)
164+
end
165+
end
166+
167+
"""
168+
gitrm_copy(src, dst)
169+
170+
Uses `git rm -r` to remove `dst` and then copies `src` to `dst`. Assumes that the working
171+
directory is within the git repository of `dst` is when the function is called.
172+
173+
This is to get around [#507](https://github.com/JuliaDocs/Documenter.jl/issues/507) on
174+
filesystems that are case-insensitive (e.g. on OS X, Windows). Without doing a `git rm`
175+
first, `git add -A` will not detect case changes in filenames.
176+
"""
177+
function gitrm_copy(src, dst)
178+
# Remove individual entries since with versions=nothing the root
179+
# would be removed and we want to preserve previews
180+
if isdir(dst)
181+
for x in filter!(!in((".git", "previews")), readdir(dst))
182+
# --ignore-unmatch so that we wouldn't get errors if dst does not exist
183+
run(`$(git()) rm -rf --ignore-unmatch $(joinpath(dst, x))`)
184+
end
185+
end
186+
# git rm also remove parent directories
187+
# if they are empty so need to mkpath after
188+
mkpath(dst)
189+
# Copy individual entries rather then the full folder since with
190+
# versions=nothing it would replace the root including e.g. the .git folder
191+
for x in readdir(src)
192+
cp(joinpath(src, x), joinpath(dst, x); force=true)
193+
end
194+
end
195+
196+
197+
198+
struct FilesystemDeployConfig <: DynamicModelTestUtils.DeployTarget
199+
repo_path :: String
200+
subfolder :: String
201+
end
202+
function DynamicModelTestUtils.deploy_folder(c::FilesystemDeployConfig; branch, repo, kwargs...)
203+
DynamicModelTestUtils.DeployConfig(; all_ok = true, subfolder = c.subfolder, branch, repo)
204+
end
205+
DynamicModelTestUtils.authentication_method(::FilesystemDeployConfig) = DynamicModelTestUtils.HTTPS
206+
DynamicModelTestUtils.authenticated_repo_url(c::FilesystemDeployConfig) = c.repo_path

‎src/deploy/restore.jl‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
2+
function load_data(action; root=currentdir(), branch="data")
3+
mktempdir() do temp
4+
cd(temp) do
5+
run(`$(git()) init`)
6+
run(`$(git()) config user.name "todo"`)
7+
run(`$(git()) config user.email "todo@example.com"`)
8+
run(`$(git()) remote add upstream $root`)
9+
10+
run(`$(git()) fetch upstream`)
11+
12+
try
13+
run(`$(git()) checkout -b $branch upstream/$branch`)
14+
catch e
15+
@info """
16+
Checking out $branch failed, creating a new orphaned branch.
17+
This usually happens when deploying to a repository for the first time and
18+
the $branch branch does not exist yet. The fatal error above is expected output
19+
from Git in this situation. Generate and commit some data to $branch, then try again.
20+
"""
21+
throw(Exception("checking out $branch failed with error: $e"))
22+
end
23+
action(temp)
24+
end
25+
end
26+
end

‎test/deployment.jl‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
using Git
2+
@testset "deploy" begin
3+
@testset "bare" begin
4+
mktempdir() do dir
5+
cd(dir) do
6+
mkdir("repo")
7+
run(`$(git()) -C repo init -q --bare`)
8+
full_repo_path = joinpath(pwd(), "repo")
9+
mkdir("runs")
10+
write("runs/run.csv", "1,2,3")
11+
DynamicModelTestUtils.deploy(
12+
pwd(), "runs",
13+
deploy_type = DynamicModelTestUtils.FilesystemDeployConfig(full_repo_path, "."),
14+
repo = full_repo_path
15+
)
16+
run(`$(git()) clone -q -b data $(full_repo_path) worktree`)
17+
println(readdir("worktree/"))
18+
@test isfile(joinpath("worktree", "run.csv"))
19+
DynamicModelTestUtils.load_data(root=full_repo_path) do localdir
20+
@test isfile("run.csv")
21+
end
22+
end
23+
end
24+
end
25+
@testset "RC deploy" begin
26+
R = 1.0
27+
C = 1.0
28+
V = 1.0
29+
@variables t
30+
@named resistor = Resistor(R = R)
31+
@named capacitor = Capacitor(C = C)
32+
@named source = Voltage()
33+
@named constant = Constant(k = V)
34+
@named ground = Ground()
35+
36+
@named sensor = PotentialSensor()
37+
@named key_parameter = MeasureComponent()
38+
39+
rc_eqs = [connect(constant.output, source.V)
40+
connect(source.p, resistor.p)
41+
connect(resistor.n, capacitor.p)
42+
connect(capacitor.n, source.n, ground.g)
43+
connect(sensor.p, capacitor.p)
44+
sensor.phi ~ key_parameter.value]
45+
46+
@named rc_model = ODESystem(rc_eqs, t,
47+
systems = [resistor, capacitor, constant, source, ground, sensor, key_parameter])
48+
sys = structural_simplify(rc_model)
49+
prob1 = ODEProblem(sys, Pair[], (0, 10.0))
50+
sol = solve(prob1, Tsit5())
51+
52+
d1 = discretize_solution(sol)
53+
54+
mktempdir() do dir
55+
cd(dir) do
56+
mkdir("repo")
57+
run(`$(git()) -C repo init -q --bare`)
58+
full_repo_path = joinpath(pwd(), "repo")
59+
mkdir("data")
60+
CSV.write("data/output.csv", d1)
61+
println(readdir())
62+
DynamicModelTestUtils.deploy(
63+
pwd(), "data",
64+
deploy_type = DynamicModelTestUtils.FilesystemDeployConfig(full_repo_path, "."),
65+
repo = full_repo_path,
66+
)
67+
DynamicModelTestUtils.load_data(root=full_repo_path) do localdir
68+
@test isfile("output.csv")
69+
restored = CSV.read("output.csv", DataFrame)
70+
@test sum(compare(sol, restored)[:, :L∞]) < 0.01
71+
end
72+
end
73+
end
74+
end
75+
end

‎test/runtests.jl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,5 @@ using Test
88
#include("timeseries.jl")
99
include("block_modeling.jl")
1010
include("instantaneous.jl")
11+
include("deployment.jl")
1112
end

0 commit comments

Comments
 (0)