Add support for overlays in releases, closes #9651

José Valim · José Valim · commit a54e90147996 · 2019-12-30T11:41:41.000+01:00
diff --git a/lib/mix/lib/mix/release.ex b/lib/mix/lib/mix/release.ex
@@ -24,6 +24,10 @@ defmodule Mix.Release do
       first element is a module that implements the `Config.Provider` behaviour
       and `term` is the value given to it on `c:Config.Provider.init/1`
     * `:options` - a keyword list with all other user supplied release options
+    * `:overlays` - a list of extra files added to the release. If you have a custom
+      step adding extra files to a release, you can add these files to the `:overlays`
+      field so they are also considered on further commands, such as tar/zip. Each entry
+      in overlays is the relative path to the release root of each file
     * `:steps` - a list of functions that receive the release and returns a release.
       Must also contain the atom `:assemble` which is the internal assembling step.
       May also contain the atom `:tar` to create a tarball of the release.
@@ -40,6 +44,7 @@ defmodule Mix.Release do
     :erts_version,
     :config_providers,
     :options,
+    :overlays,
     :steps
   ]
 
@@ -147,6 +152,7 @@ defmodule Mix.Release do
       boot_scripts: %{start: start_boot, start_clean: start_clean_boot},
       config_providers: config_providers,
       options: opts,
+      overlays: [],
       steps: steps
     }
   end
diff --git a/lib/mix/lib/mix/tasks/release.ex b/lib/mix/lib/mix/tasks/release.ex
@@ -431,12 +431,55 @@ defmodule Mix.Tasks.Release do
             ]
           ]
 
+    * `:overlays` - a directory with extra files to be copied as is to the
+      release. See the "Overlays" section for more information. Defaults to
+      "rel/overlays" if said directory exists.
+
     * `:steps` - a list of steps to execute when assembling the release. See
       the "Steps" section for more information.
 
   Besides the options above, it is possible to customize the generated
-  release with custom template files or by tweaking the release steps.
-  We will detail both approaches next.
+  release with custom files, by tweaking the release steps or by running
+  custom options and commands on boot. We will detail both approaches next.
+
+  ### Overlays
+
+  Often it is necessary to copy extra files to the release root after
+  the release is assembled. This can be easily done by placing such
+  files in the `rel/overlays` directory. Any file in there is copied
+  as is to the release root. For example, if you have place a
+  "rel/overlays/Dockerfile" file, the "Dockerfile" will be copied as
+  is to the release root. If you need to copy files dynamically, see
+  the "Steps" section.
+
+  ### Steps
+
+  It is possible to add one or more steps before and after the release is
+  assembled. This can be done with the `:steps` option:
+
+      releases: [
+        demo: [
+          steps: [&set_configs/1, :assemble, &copy_extra_files/1]
+        ]
+      ]
+
+  The `:steps` option must be a list and it must always include the
+  atom `:assemble`, which does most of the release assembling. You
+  can pass anonymous functions before and after the `:assemble` to
+  customize your release assembling pipeline. Those anonymous functions
+  will receive a `Mix.Release` struct and must return the same or
+  an updated `Mix.Release` struct. It is also possible to build a tarball
+  of the release by passing the `:tar` step anywhere after `:assemble`.
+  The tarball is created in `_build/MIX_ENV/RELEASE_NAME-RELEASE_VSN.tar.gz`
+
+  See `Mix.Release` for more documentation on the struct and which
+  fields can be modified. Note that `:steps` field itself can be
+  modified and it is updated every time a step is called. Therefore,
+  if you need to execute a command before and after assembling the
+  release, you only need to declare the first steps in your pipeline
+  and then inject the last step into the release struct. The steps
+  field can also be used to verify if the step was set before or
+  after assembling the release.
 
   ### vm.args and env.sh (env.bat)
 
@@ -489,35 +532,6 @@ defmodule Mix.Tasks.Release do
         set ELIXIR_ERL_OPTIONS="-kernel inet_dist_listen_min %BEAM_PORT% inet_dist_listen_max %BEAM_PORT%"
       )
 
-  ### Steps
-
-  It is possible to add one or more steps before and after the release is
-  assembled. This can be done with the `:steps` option:
-
-      releases: [
-        demo: [
-          steps: [&set_configs/1, :assemble, &copy_extra_files/1]
-        ]
-      ]
-
-  The `:steps` option must be a list and it must always include the
-  atom `:assemble`, which does most of the release assembling. You
-  can pass anonymous functions before and after the `:assemble` to
-  customize your release assembling pipeline. Those anonymous functions
-  will receive a `Mix.Release` struct and must return the same or
-  an updated `Mix.Release` struct. It is also possible to build a tarball
-  of the release by passing the `:tar` step anywhere after `:assemble`.
-  The tarball is created in `_build/MIX_ENV/RELEASE_NAME-RELEASE_VSN.tar.gz`
-
-  See `Mix.Release` for more documentation on the struct and which
-  fields can be modified. Note that `:steps` field itself can be
-  modified and it is updated every time a step is called. Therefore,
-  if you need to execute a command before and after assembling the
-  release, you only need to declare the first steps in your pipeline
-  and then inject the last step into the release struct. The steps
-  field can also be used to verify if the step was set before or
-  after assembling the release.
-
   ## Application configuration
 
   Releases provides two mechanisms for configuring OTP applications:
@@ -1029,7 +1043,7 @@ defmodule Mix.Tasks.Release do
     |> Task.async_stream(&copy(&1, release), ordered: false, timeout: :infinity)
     |> Stream.run()
 
-    release
+    copy_overlays(release)
   end
 
   defp make_tar(release) do
@@ -1055,6 +1069,7 @@ defmodule Mix.Tasks.Release do
     files =
       dirs
       |> Enum.filter(&File.exists?(Path.join(release.path, &1)))
+      |> Kernel.++(release.overlays)
       |> Enum.map(&{String.to_charlist(&1), String.to_charlist(Path.join(release.path, &1))})
 
     File.rm(out_path)
@@ -1202,6 +1217,38 @@ defmodule Mix.Tasks.Release do
     Mix.shell().info([:yellow, "* skipping ", :reset, message])
   end
 
+  ## Overlays
+
+  defp copy_overlays(release) do
+    target = release.path
+    overlays = release.options[:overlays]
+
+    copied =
+      cond do
+        is_nil(overlays) and File.dir?("rel/overlays") ->
+          File.cp_r!("rel/overlays", target)
+
+        is_nil(overlays) ->
+          []
+
+        is_binary(overlays) and File.dir?(overlays) ->
+          File.cp_r!(overlays, target)
+
+        true ->
+          Mix.raise(
+            ":overlays release configuration must be a string pointing to an existing directory, " <>
+              "got: #{inspect(overlays)}"
+          )
+      end
+
+    relative =
+      copied
+      |> List.delete(target)
+      |> Enum.map(&Path.relative_to(&1, target))
+
+    update_in(release.overlays, &(relative ++ &1))
+  end
+
   ## Copy operations
 
   defp copy(:erts, release) do
diff --git a/lib/mix/test/mix/tasks/release_test.exs b/lib/mix/test/mix/tasks/release_test.exs
@@ -9,7 +9,7 @@ defmodule Mix.Tasks.ReleaseTest do
   defmacrop release_node(name), do: :"#{name}@#{@hostname}"
 
   describe "customize" do
-    test "rel with EEx" do
+    test "env and vm.args with EEx" do
       in_fixture("release_test", fn ->
         Mix.Project.in_project(:release_test, ".", fn _ ->
           File.mkdir_p!("rel")
@@ -36,7 +36,106 @@ defmodule Mix.Tasks.ReleaseTest do
       end)
     end
 
-    test "tar" do
+    test "steps" do
+      in_fixture("release_test", fn ->
+        last_step = fn release ->
+          send(self(), {:last_step, release})
+          release
+        end
+
+        first_step = fn release ->
+          send(self(), {:first_step, release})
+          update_in(release.steps, &(&1 ++ [last_step]))
+        end
+
+        config = [releases: [demo: [steps: [first_step, :assemble]]]]
+
+        Mix.Project.in_project(:release_test, ".", config, fn _ ->
+          Mix.Task.run("release")
+          assert_received {:mix_shell, :info, ["* assembling demo-0.1.0 on MIX_ENV=dev"]}
+
+          # Discard info messages from inbox for upcoming assertions
+          Mix.shell().flush(& &1)
+
+          {:messages,
+           [
+             {:first_step, %Mix.Release{steps: [:assemble]}},
+             {:last_step, %Mix.Release{steps: []}}
+           ]} = Process.info(self(), :messages)
+        end)
+      end)
+    end
+
+    test "include_executables_for" do
+      in_fixture("release_test", fn ->
+        config = [releases: [release_test: [include_executables_for: []]]]
+
+        Mix.Project.in_project(:release_test, ".", config, fn _ ->
+          root = Path.absname("_build/dev/rel/release_test")
+          Mix.Task.run("release")
+          assert_received {:mix_shell, :info, ["* assembling release_test-0.1.0 on MIX_ENV=dev"]}
+
+          refute root |> Path.join("bin/start") |> File.exists?()
+          refute root |> Path.join("bin/start.bat") |> File.exists?()
+          refute root |> Path.join("releases/0.1.0/elixir") |> File.exists?()
+          refute root |> Path.join("releases/0.1.0/elixir.bat") |> File.exists?()
+          refute root |> Path.join("releases/0.1.0/iex") |> File.exists?()
+          refute root |> Path.join("releases/0.1.0/iex.bat") |> File.exists?()
+        end)
+      end)
+    end
+
+    test "default overlays" do
+      in_fixture("release_test", fn ->
+        Mix.Project.in_project(:release_test, ".", fn _ ->
+          File.mkdir_p!("rel/overlays/empty/directory")
+          File.write!("rel/overlays/hello", "world")
+
+          root = Path.absname("_build/dev/rel/release_test")
+          Mix.Task.run("release")
+
+          assert root |> Path.join("empty/directory") |> File.dir?()
+          assert root |> Path.join("hello") |> File.read!() == "world"
+        end)
+      end)
+    end
+
+    test "custom overlays" do
+      in_fixture("release_test", fn ->
+        config = [releases: [release_test: [overlays: "rel/another"]]]
+
+        Mix.Project.in_project(:release_test, ".", config, fn _ ->
+          assert_raise Mix.Error, ~r"a string pointing to an existing directory", fn ->
+            Mix.Task.run("release", ["--overwrite"])
+          end
+
+          File.mkdir_p!("rel/another/empty/directory")
+          File.write!("rel/another/hello", "world")
+
+          root = Path.absname("_build/dev/rel/release_test")
+          Mix.Task.rerun("release", ["--overwrite"])
+
+          assert root |> Path.join("empty/directory") |> File.dir?()
+          assert root |> Path.join("hello") |> File.read!() == "world"
+        end)
+      end)
+    end
+  end
+
+  describe "errors" do
+    test "requires a matching name" do
+      in_fixture("release_test", fn ->
+        Mix.Project.in_project(:release_test, ".", fn _ ->
+          assert_raise Mix.Error, ~r"Unknown release :unknown", fn ->
+            Mix.Task.run("release", ["unknown"])
+          end
+        end)
+      end)
+    end
+  end
+
+  describe "tar" do
+    test "with ERTS" do
       in_fixture("release_test", fn ->
         config = [releases: [demo: [steps: [:assemble, :tar]]]]
 
@@ -51,6 +150,10 @@ defmodule Mix.Tasks.ReleaseTest do
           File.mkdir_p!(ignored_release_path)
           File.touch(Path.join(ignored_release_path, "ignored"))
 
+          # Overlays
+          File.mkdir_p!("rel/overlays/empty/directory")
+          File.write!("rel/overlays/hello", "world")
+
           Mix.Task.run("release")
           tar_path = Path.expand(Path.join([root, "..", "..", "demo-0.1.0.tar.gz"]))
           message = "* building #{tar_path}"
@@ -67,6 +170,8 @@ defmodule Mix.Tasks.ReleaseTest do
           assert "releases/0.1.0/vm.args" in files
           assert "releases/COOKIE" in files
           assert "releases/start_erl.data" in files
+          assert "hello" in files
+          assert "empty/directory" in files
           assert Enum.any?(files, &(&1 =~ "erts"))
           assert Enum.any?(files, &(&1 =~ "stdlib"))
 
@@ -81,7 +186,7 @@ defmodule Mix.Tasks.ReleaseTest do
       end)
     end
 
-    test "tar without ERTS" do
+    test "without ERTS" do
       in_fixture("release_test", fn ->
         config = [releases: [demo: [include_erts: false, steps: [:assemble, :tar]]]]
 
@@ -103,55 +208,6 @@ defmodule Mix.Tasks.ReleaseTest do
         end)
       end)
     end
-
-    test "steps" do
-      in_fixture("release_test", fn ->
-        last_step = fn release ->
-          send(self(), {:last_step, release})
-          release
-        end
-
-        first_step = fn release ->
-          send(self(), {:first_step, release})
-          update_in(release.steps, &(&1 ++ [last_step]))
-        end
-
-        config = [releases: [demo: [steps: [first_step, :assemble]]]]
-
-        Mix.Project.in_project(:release_test, ".", config, fn _ ->
-          Mix.Task.run("release")
-          assert_received {:mix_shell, :info, ["* assembling demo-0.1.0 on MIX_ENV=dev"]}
-
-          # Discard info messages from inbox for upcoming assertions
-          Mix.shell().flush(& &1)
-
-          {:messages,
-           [
-             {:first_step, %Mix.Release{steps: [:assemble]}},
-             {:last_step, %Mix.Release{steps: []}}
-           ]} = Process.info(self(), :messages)
-        end)
-      end)
-    end
-
-    test "include_executables_for" do
-      in_fixture("release_test", fn ->
-        config = [releases: [release_test: [include_executables_for: []]]]
-
-        Mix.Project.in_project(:release_test, ".", config, fn _ ->
-          root = Path.absname("_build/dev/rel/release_test")
-          Mix.Task.run("release")
-          assert_received {:mix_shell, :info, ["* assembling release_test-0.1.0 on MIX_ENV=dev"]}
-
-          refute root |> Path.join("bin/start") |> File.exists?()
-          refute root |> Path.join("bin/start.bat") |> File.exists?()
-          refute root |> Path.join("releases/0.1.0/elixir") |> File.exists?()
-          refute root |> Path.join("releases/0.1.0/elixir.bat") |> File.exists?()
-          refute root |> Path.join("releases/0.1.0/iex") |> File.exists?()
-          refute root |> Path.join("releases/0.1.0/iex.bat") |> File.exists?()
-        end)
-      end)
-    end
   end
 
   test "assembles a bootable release with ERTS" do
@@ -555,16 +611,6 @@ defmodule Mix.Tasks.ReleaseTest do
     end)
   end
 
-  test "requires a matching name" do
-    in_fixture("release_test", fn ->
-      Mix.Project.in_project(:release_test, ".", fn _ ->
-        assert_raise Mix.Error, ~r"Unknown release :unknown", fn ->
-          Mix.Task.run("release", ["unknown"])
-        end
-      end)
-    end)
-  end
-
   defp open_port(command, args, env \\ []) do
     Port.open({:spawn_executable, to_charlist(command)}, [:hide, args: args, env: env])
   end
