Move rebar and other deps to _build

Author: José Valim

lib/mix/lib/mix/deps.ex

@@ -342,28 +342,36 @@ defmodule Mix.Deps do
   end
 
   @doc """
-  Returns all load paths for the dependency.
-  Expects a loaded dependency.
+  Returns all load paths for the given dependency. Automatically
+  derived from source paths.
   """
-  def load_paths(Mix.Dep[manager: :mix, opts: opts]) do
-    [Path.join(opts[:build], "ebin")]
+  def load_paths(Mix.Dep[opts: opts] = dep) do
+    build_path = Path.dirname(opts[:build])
+    Enum.map source_paths(dep), fn path ->
+      Path.join [build_path, Path.basename(path), "ebin"]
+    end
   end
 
-  def load_paths(Mix.Dep[manager: :rebar, opts: opts, extra: extra]) do
+  @doc """
+  Returns all source paths.
+
+  Source paths are the directories that contains ebin files for a given
+  dependency. All managers, except rebar, have only one source path.
+  """
+  def source_paths(Mix.Dep[manager: :rebar, opts: opts, extra: extra]) do
     # Add root dir and all sub dirs with ebin/ directory
     sub_dirs = Enum.map(extra[:sub_dirs] || [], fn path ->
       Path.join(opts[:dest], path)
     end)
 
-    [ opts[:dest] | sub_dirs ]
-      |> Enum.map(&Path.wildcard(&1))
-      |> Enum.concat
-      |> Enum.map(&Path.join(&1, "ebin"))
-      |> Enum.filter(&File.dir?(&1))
+    [opts[:dest] | sub_dirs]
+    |> Enum.map(&Path.wildcard(&1))
+    |> Enum.concat
+    |> Enum.filter(fn p -> p |> Path.join("ebin") |> File.dir? end)
   end
 
-  def load_paths(Mix.Dep[manager: manager, opts: opts]) when manager in [:make, nil] do
-    [ Path.join(opts[:dest], "ebin") ]
+  def source_paths(Mix.Dep[opts: opts]) do
+    [opts[:dest]]
   end
 
   @doc """

lib/mix/lib/mix/deps/retriever.ex

@@ -139,22 +139,19 @@ defmodule Mix.Deps.Retriever do
 
   ## Fetching
 
-  defp mix_dep(Mix.Dep[opts: opts, app: app] = dep, config) do
+  defp mix_dep(Mix.Dep[opts: opts] = dep, config) do
     Mix.Deps.in_dependency(dep, config, fn _ ->
       config    = Mix.project
       umbrella? = Mix.Project.umbrella?
-      app_path  =
-        if umbrella? do
-          false
-        else
-          Path.join(Mix.Project.compile_path(config), "#{app}.app")
-        end
+
+      if umbrella? do
+        opts = Keyword.put_new(opts, :app, false)
+      end
 
       if req = old_elixir_req(config) do
         dep = dep.status({ :elixirreq, req })
       end
 
-      opts = Keyword.put_new(opts, :app, app_path)
       { dep.manager(:mix).opts(opts).extra(umbrella: umbrella?), children }
     end)
   end
@@ -182,7 +179,7 @@ defmodule Mix.Deps.Retriever do
       dep
     else
       path = if is_binary(opts_app), do: opts_app, else: "ebin/#{app}.app"
-      path = Path.expand(path, opts[:dest])
+      path = Path.expand(path, opts[:build])
       dep.status app_status(path, app, req)
     end
   end

lib/mix/lib/mix/deps/umbrella.ex

@@ -9,12 +9,13 @@ defmodule Mix.Deps.Umbrella do
 
     if apps_path = config[:apps_path] do
       paths = Path.wildcard(Path.join(apps_path, "*"))
+      build = Mix.Project.build_path
 
       paths
       |> Enum.filter(&File.dir?(&1))
       |> extract_umbrella
       |> filter_umbrella(config[:apps])
-      |> to_umbrella_dep()
+      |> to_umbrella_dep(build)
     else
       []
     end
@@ -49,10 +50,12 @@ defmodule Mix.Deps.Umbrella do
     lc { app, _ } = pair inlist pairs, app in apps, do: pair
   end
 
-  defp to_umbrella_dep(paths) do
+  defp to_umbrella_dep(paths, build) do
     Enum.map paths, fn({ app, path }) ->
+      opts = [path: path, dest: Path.expand(path),
+              env: Mix.env, build: Path.join([build, "lib", app])]
       Mix.Dep[scm: Mix.SCM.Path, app: app, requirement: nil, manager: :mix,
-              status: { :ok, nil }, opts: [path: path, dest: Path.expand(path), env: Mix.env]]
+              status: { :ok, nil }, opts: opts]
     end
   end
 end

lib/mix/lib/mix/project.ex

@@ -245,18 +245,26 @@ defmodule Mix.Project do
   @doc """
   Builds the project structure for the current application.
   """
-  def build_structure(config // config()) do
-    ebin = compile_path(config)
-    File.mkdir_p!(ebin)
-
-    lib = Path.dirname(ebin)
-    old = Path.expand("priv")
-
-    case :file.make_symlink(old, Path.join(lib, "priv")) do
-      :ok -> :ok
-      { :error, :eexist } -> :ok
-      { :error, _ } -> File.cp_r!(old, lib) |> IO.inspect
+  def build_structure(config // config(), opts // []) do
+    app = app_path(config)
+    File.mkdir_p!(app)
+
+    source = Path.expand("ebin")
+    target = Path.join(app, "ebin")
+
+    cond do
+      opts[:symlink_ebin?] ->
+        Mix.Utils.symlink_or_copy(source, target)
+      match?({ :ok, _ }, :file.read_link(target)) ->
+        File.rm_rf!(target)
+        File.mkdir_p!(target)
+      true ->
+        File.mkdir_p!(target)
     end
+
+    source = Path.expand("priv")
+    target = Path.join(app, "priv")
+    Mix.Utils.symlink_or_copy(source, target)
   end
 
   @doc """

lib/mix/lib/mix/tasks/deps.compile.ex

@@ -48,28 +48,27 @@ defmodule Mix.Tasks.Deps.Compile do
     compiled =
       Enum.map deps, fn(dep) ->
         Mix.Dep[app: app, status: status, opts: opts] = dep
-        deps_path = opts[:dest]
-
         check_unavailable!(app, status)
+
         unless run_opts[:quiet] || opts[:compile] == false do
           shell.info "* Compiling #{app}"
         end
 
         compiled = cond do
           not nil?(opts[:compile]) ->
-            do_compile app, deps_path, opts[:compile]
+            do_compile dep
           mix?(dep) ->
             do_mix dep, config
           rebar?(dep) ->
-            do_rebar app, deps_path, config[:deps_path]
+            do_rebar dep, config
           make?(dep) ->
-            do_command app, deps_path, "make"
+            do_make dep
           true ->
             shell.error "Could not compile #{app}, no mix.exs, rebar.config or Makefile " <>
               "(pass :compile as an option to customize compilation, set it to false to do nothing)"
         end
 
-        Enum.each(Mix.Deps.load_paths(dep), &Code.prepend_path/1)
+        unless mix?(dep), do: build_structure(dep, config)
         compiled
       end
 
@@ -85,16 +84,12 @@ defmodule Mix.Tasks.Deps.Compile do
     :ok
   end
 
-  defp app_path_for(Mix.Dep[opts: opts]) do
-    opts[:build]
-  end
-
-  defp do_mix(dep, config) do
+  defp do_mix(Mix.Dep[opts: opts] = dep, config) do
     # Set the app_path to be the one stored in the dependency.
     # This is important because the name of application in the
     # mix.exs file can be different than the actual name and we
     # choose to respect the one in the mix.exs.
-    config = Keyword.put(config, :app_path, app_path_for(dep))
+    config = Keyword.put(config, :app_path, opts[:build])
 
     Mix.Deps.in_dependency dep, config, fn _ ->
       try do
@@ -111,8 +106,8 @@ defmodule Mix.Tasks.Deps.Compile do
     end
   end
 
-  defp do_rebar(app, deps_path, root_path) do
-    do_command app, deps_path, rebar_cmd(app), "compile skip_deps=true deps_dir=#{inspect root_path}"
+  defp do_rebar(Mix.Dep[app: app] = dep, config) do
+    do_command dep, rebar_cmd(app), "compile skip_deps=true deps_dir=#{inspect config[:deps_path]}"
   end
 
   defp rebar_cmd(app) do
@@ -133,22 +128,42 @@ defmodule Mix.Tasks.Deps.Compile do
     Mix.Rebar.local_rebar_cmd || raise Mix.Error, message: "rebar instalation failed"
   end
 
-  defp do_compile(_, _deps_path, false) do
-    false
+  defp do_make(dep) do
+    do_command(dep, "make")
   end
 
-  defp do_compile(app, deps_path, command) when is_binary(command) do
-    Mix.shell.info("#{app}: #{command}")
-    do_command(app, deps_path, command)
+  defp do_compile(Mix.Dep[app: app, opts: opts] = dep) do
+    if command = opts[:compile] do
+      Mix.shell.info("#{app}: #{command}")
+      do_command(dep, command)
+    else
+      false
+    end
   end
 
-  defp do_command(app, deps_path, command, extra // "") do
-    File.cd! deps_path, fn ->
+  defp do_command(Mix.Dep[app: app, opts: opts], command, extra // "") do
+    File.cd! opts[:dest], fn ->
       if Mix.shell.cmd("#{command} #{extra}") != 0 do
         raise Mix.Error, message: "Could not compile dependency #{app}, #{command} command failed. " <>
           "If you want to recompile this dependency, please run: mix deps.compile #{app}"
       end
     end
     true
   end
+
+  defp build_structure(Mix.Dep[opts: opts] = dep, config) do
+    build_path = Path.dirname(opts[:build])
+    Enum.each Mix.Deps.source_paths(dep), fn source ->
+      app = Path.join(build_path, Path.basename(source))
+      build_structure(source, app, config)
+      Code.prepend_path(Path.join(app, "ebin"))
+    end
+  end
+
+  defp build_structure(dest, build, config) do
+    File.cd! dest, fn ->
+      config = Keyword.put(config, :app_path, build)
+      Mix.Project.build_structure(config, symlink_ebin?: true)
+    end
+  end
 end

lib/mix/lib/mix/utils.ex

@@ -314,6 +314,29 @@ defmodule Mix.Utils do
   defp to_lower_char(char) when char in ?A..?Z, do: char + 32
   defp to_lower_char(char), do: char
 
+  @doc """
+  Symlink directory `source` to `target` or copy it recursively
+  in case symlink fails.
+  """
+  def symlink_or_copy(source, target) do
+    if File.exists?(source) do
+      case :file.make_symlink(source, target) do
+        :ok -> :ok
+        { :error, :eexist } ->
+          case :file.read_link(target) do
+            { :ok, _ } -> :ok
+            { :error, _ } -> do_copy(source, target)
+          end
+        { :error, _ } -> do_copy(source, target)
+      end
+    end
+  end
+
+  defp do_copy(source, target) do
+    File.rm_rf!(target)
+    File.cp_r!(source, Path.join(target, "."))
+  end
+
   @doc """
   Opens and reads content from either a URL or a local filesystem path.
 

…atus/deps/invalidvsn/ebin/invalidvsn.app …build/lib/invalidvsn/ebin/invalidvsn.applib/mix/test/fixtures/deps_status/deps/invalidvsn/ebin/invalidvsn.app renamed to lib/mix/test/fixtures/deps_status/_build/lib/invalidvsn/ebin/invalidvsn.app lib/mix/test/fixtures/deps_status/deps/invalidvsn/ebin/invalidvsn.app renamed to lib/mix/test/fixtures/deps_status/_build/lib/invalidvsn/ebin/invalidvsn.app

@@ -52,7 +52,7 @@ defmodule Mix.Tasks.DepsTest do
       assert_received { :mix_shell, :info, ["* invalidapp (deps/invalidapp)"] }
       assert_received { :mix_shell, :info, ["  the app file at _build/lib/invalidapp/ebin/invalidapp.app is invalid"] }
       assert_received { :mix_shell, :info, ["* noappfile (deps/noappfile)"] }
-      assert_received { :mix_shell, :info, ["  could not find an app file at deps/noappfile/ebin/noappfile.app" <> _] }
+      assert_received { :mix_shell, :info, ["  could not find an app file at _build/lib/noappfile/ebin/noappfile.app" <> _] }
       assert_received { :mix_shell, :info, ["* uncloned (https://github.com/elixir-lang/uncloned.git)"] }
       assert_received { :mix_shell, :info, ["  the dependency is not available, run `mix deps.get`"] }
     end
@@ -146,7 +146,7 @@ defmodule Mix.Tasks.DepsTest do
       assert_received { :mix_shell, :error, ["* invalidapp (deps/invalidapp)"] }
       assert_received { :mix_shell, :error, ["  the app file at _build/lib/invalidapp/ebin/invalidapp.app is invalid"] }
       assert_received { :mix_shell, :error, ["* noappfile (deps/noappfile)"] }
-      assert_received { :mix_shell, :error, ["  could not find an app file at deps/noappfile/ebin/noappfile.app" <> _] }
+      assert_received { :mix_shell, :error, ["  could not find an app file at _build/lib/noappfile/ebin/noappfile.app" <> _] }
       assert_received { :mix_shell, :error, ["* uncloned (https://github.com/elixir-lang/uncloned.git)"] }
       assert_received { :mix_shell, :error, ["  the dependency is not available, run `mix deps.get`"] }
     end

lib/mix/test/fixtures/deps_status/deps/invalidvsn/.gitkeep

@@ -44,8 +44,14 @@ defmodule Mix.UmbrellaTest do
 
     in_fixture "umbrella_dep/deps/umbrella", fn ->
       File.mkdir_p!("deps/some_dep/ebin")
+      File.mkdir_p!("_build/lib/some_dep/ebin")
+      File.mkdir_p!("_build/lib/foo/ebin")
+      File.mkdir_p!("_build/lib/bar/ebin")
+
       Mix.Task.run "loadpaths", ["--no-deps-check", "--no-elixir-version-check"]
-      assert Path.expand('deps/some_dep/ebin') in :code.get_path
+      assert Path.expand('_build/lib/some_dep/ebin') in :code.get_path
+      assert Path.expand('_build/lib/foo/ebin') in :code.get_path
+      assert Path.expand('_build/lib/bar/ebin') in :code.get_path
     end
   after
     Mix.Project.pop