Support build_embedded and consolidate_protocols

Author: José Valim

lib/elixir/lib/code.ex

@@ -51,16 +51,7 @@ defmodule Code do
     :code.del_path(to_char_list(Path.expand path))
   end
 
-  @doc """
-  Re-add paths given to the command line to keep their position
-  on the overall code path.
-
-  Some tools may change the code path by prepending new items but
-  still want the paths given by the user to have higher priority.
-  Calling this function guarantees the paths are re-added on
-  top of the user given ones.
-  """
-  @spec readd_paths() :: :ok
+  @doc false
   def readd_paths() do
     {pa, pz} = :elixir_code_server.call(:paths)
     :code.add_pathsa(pa)

lib/mix/lib/mix/project.ex

@@ -80,7 +80,8 @@ defmodule Mix.Project do
   # The configuration that is pushed down to dependencies.
   @doc false
   def deps_config(config \\ config()) do
-    [build_path: build_path(config),
+    [build_embedded: config[:build_embedded],
+     build_path: build_path(config),
      build_per_environment: config[:build_per_environment],
      deps_path: deps_path(config)]
   end
@@ -320,19 +321,30 @@ defmodule Mix.Project do
 
     _ = cond do
       opts[:symlink_ebin] ->
-        _ = Mix.Utils.symlink_or_copy(source, target)
+        _ = symlink_or_copy(config, source, target)
       match?({:ok, _}, :file.read_link(target)) ->
         _ = File.rm_rf!(target)
         File.mkdir_p!(target)
       true ->
         File.mkdir_p!(target)
     end
 
-    _ = Mix.Utils.symlink_or_copy(Path.expand("include"), Path.join(app, "include"))
-    _ = Mix.Utils.symlink_or_copy(Path.expand("priv"), Path.join(app, "priv"))
+    _ = symlink_or_copy(config, Path.expand("include"), Path.join(app, "include"))
+    _ = symlink_or_copy(config, Path.expand("priv"), Path.join(app, "priv"))
     :ok
   end
 
+  defp symlink_or_copy(config, source, target) do
+    if config[:build_embedded] do
+      if File.exists?(source) do
+        File.rm_rf!(target)
+        File.cp_r!(source, target)
+      end
+    else
+      Mix.Utils.symlink_or_copy(source, target)
+    end
+  end
+
   @doc """
   Returns all load paths for this project.
   """
@@ -378,6 +390,8 @@ defmodule Mix.Project do
   defp default_config do
     [aliases: [],
      build_per_environment: true,
+     build_embedded: false,
+     consolidate_protocols: false,
      default_task: "run",
      deps: [],
      deps_path: "deps",

lib/mix/lib/mix/tasks/app.start.ex

@@ -9,21 +9,35 @@ defmodule Mix.Tasks.App.Start do
 
   ## Command line options
 
-    * `--force`         - force compilation regardless of compilation times
-    * `--no-compile`    - do not compile even if files require compilation
+    * `--force` - force compilation regardless of compilation times
+    * `--no-compile` - do not compile even if files require compilation
+    * `--no-protocols` - do not load consolidated protocols
     * `--no-deps-check` - do not check dependencies
-    * `--no-elixir-version-check`
-                        - do not check elixir version
-    * `--no-start`      - do not start applications after compilation
+    * `--no-elixir-version-check` - do not check elixir version
+    * `--no-start` - do not start applications after compilation
 
   """
   @spec run(OptionParser.argv) :: :ok
   def run(args) do
     Mix.Project.get!
-    Mix.Task.run "loadpaths", ["--no-readd"|args]
+    Mix.Task.run "loadpaths", args
 
     unless "--no-compile" in args do
-      Mix.Task.run "compile", ["--no-readd"|args]
+      Mix.Task.run "compile", args
+    end
+
+    unless "--no-protocols" in args do
+      path = Path.join(Mix.Project.build_path, "consolidated")
+
+      if File.dir?(path) do
+        Code.prepend_path(path)
+
+        Enum.each(File.ls!(path), fn file ->
+          module = file |> Path.rootname() |> String.to_atom()
+          :code.purge(module)
+          :code.delete(module)
+        end)
+      end
     end
 
     unless "--no-start" in args do
@@ -35,8 +49,6 @@ defmodule Mix.Tasks.App.Start do
       if Code.ensure_loaded?(Logger), do: Logger.App.stop()
       start(Mix.Project.config[:app])
     end
-
-    Code.readd_paths()
   end
 
   @doc false

lib/mix/lib/mix/tasks/compile.all.ex

@@ -0,0 +1,22 @@
+defmodule Mix.Tasks.Compile.All do
+  use Mix.Task
+
+  @moduledoc false
+  @recursive true
+
+  # This is an internal task used by mix compile which
+  # is meant to be recursive and be invoked for each child
+  # project.
+
+  def run(args) do
+    Mix.Project.get!
+
+    res =
+      Enum.map(Mix.Tasks.Compile.compilers(), fn(compiler) ->
+        Mix.Task.run("compile.#{compiler}", args)
+      end)
+
+    true = Code.prepend_path(Mix.Project.compile_path)
+    if :ok in res, do: :ok, else: :noop
+  end
+end

lib/mix/lib/mix/tasks/compile.ex

@@ -15,6 +15,12 @@ defmodule Mix.Tasks.Compile do
     * `:compilers` - compilers to run, defaults to:
       `[:leex, :yeec, :erlang, :elixir, :app]`
 
+    * `:consolidate_protocols` - when true, runs protocol
+      consolidation via the `compile.protocols` task
+
+    * `:build_embedded` - when true, activates protocol
+      consolidation and does not generate symlinks in builds
+
   ## Command line options
 
     * `--list`          - list all enabled compilers
@@ -47,21 +53,20 @@ defmodule Mix.Tasks.Compile do
       shell.info format('mix ~-#{max}s # ~ts', [task, doc])
     end
 
-    shell.info "\nEnabled compilers: #{Enum.join compilers(), ", "}"
+    compilers = compilers() ++ if(consolidate_protocols?(), do: [:protocols], else: [])
+    shell.info "\nEnabled compilers: #{Enum.join compilers, ", "}"
   end
 
   def run(args) do
     Mix.Project.get!
-    Mix.Task.run "loadpaths", ["--no-readd"|args]
+    Mix.Task.run "loadpaths", args
+    res = Mix.Task.run "compile.all", args
 
-    res =
-      Enum.map(compilers(), fn(compiler) ->
-        List.wrap Mix.Task.run("compile.#{compiler}", args)
-      end)
+    if consolidate_protocols?() do
+      Mix.Task.run "compile.protocols", args
+    end
 
-    true = Code.prepend_path(Mix.Project.compile_path)
-    unless "--no-readd" in args, do: Code.readd_paths()
-    if Enum.any?(res, &(:ok in &1)), do: :ok, else: :noop
+    res
   end
 
   # Loadpaths without checks because compilers may be defined in deps.
@@ -70,6 +75,11 @@ defmodule Mix.Tasks.Compile do
     Mix.Task.reenable "loadpaths"
   end
 
+  defp consolidate_protocols? do
+    config = Mix.Project.config
+    config[:build_embedded] || config[:consolidate_protocols]
+  end
+
   @doc """
   Returns all compilers.
   """

lib/mix/lib/mix/tasks/compile.protocols.ex

@@ -1,21 +1,26 @@
 defmodule Mix.Tasks.Compile.Protocols do
   use Mix.Task
 
-  @recursive true
-
-  @shortdoc "Consolidates all protocols in all paths"
-
   @moduledoc ~S"""
   Consolidates all protocols in all paths.
 
-  This module consolidates all protocols in the code path
+  This task is automatically invoked whenever the project
+  enables `:consolidate_protocols` or `:build_embedded` in
+  its configuration.
+
+  ## Consolidation
+
+  Protocol consolidation is useful in production when no
+  dynamic code loading will happen, effectively optimizing
+  protocol dispatches by not accounting for code loading.
+
+  This task consolidates all protocols in the code path
   and output the new binary files to the given directory
-  (defaults to "consolidated").
+  (defaults to "_build/MIX_ENV/consolidated").
 
-  A new directory will be created with the consolidated
-  protocol versions in the build directory for the given
-  environment. Simply add it to your loadpath to make use
-  of it according to your MIX_ENV:
+  In case you are manually compiling protocols or building
+  releases, you need to take the generated protocols into
+  account. This can be done with:
 
       $ elixir -pa _build/MIX_ENV/consolidated -S mix run
 

lib/mix/lib/mix/tasks/deps.compile.ex

@@ -102,7 +102,7 @@ defmodule Mix.Tasks.Deps.Compile do
       end
 
       try do
-        res = Mix.Task.run("compile", ["--no-deps", "--no-elixir-version-check", "--no-readd"])
+        res = Mix.Task.run("compile", ["--no-deps", "--no-elixir-version-check"])
         :ok in List.wrap(res)
       catch
         kind, reason ->

lib/mix/lib/mix/tasks/escript.build.ex

@@ -48,11 +48,6 @@ defmodule Mix.Tasks.Escript.Build do
 
       Defaults to `true` for Elixir projects.
 
-    * `:consolidate_protocols` - if `true`, all protocols will be consolidated
-      before being embedded into the escript.
-
-      Defaults to `true` for Elixir projects.
-
     * `:shebang` - shebang interpreter directive used to execute the escript.
       Defaults to `"#! /usr/bin/env escript\n"`.
 
@@ -93,19 +88,13 @@ defmodule Mix.Tasks.Escript.Build do
       Mix.Task.run :compile, args
     end
 
-    project = Mix.Project.config
+    project  = Mix.Project.config
     language = Keyword.get(project, :language, :elixir)
-    should_consolidate =
-      Keyword.get(project, :consolidate_protocols, language == :elixir)
-
-    if should_consolidate do
-      Mix.Task.run "compile.protocols", []
-    end
 
-    escriptize(project, language, opts[:force], should_consolidate)
+    escriptize(project, language, opts[:force])
   end
 
-  defp escriptize(project, language, force, should_consolidate) do
+  defp escriptize(project, language, force) do
     escript_opts = project[:escript] || []
 
     script_name  = to_string(escript_opts[:name] || project[:app])
@@ -131,7 +120,7 @@ defmodule Mix.Tasks.Escript.Build do
           |> Stream.concat
           |> prepare_beam_paths
 
-        if should_consolidate do
+        if project[:build_embedded] || project[:consolidate_protocols] do
           beam_paths =
             Path.wildcard(consolidated_path <> "/*")
             |> prepare_beam_paths(beam_paths)

lib/mix/lib/mix/tasks/loadpaths.ex

@@ -29,10 +29,6 @@ defmodule Mix.Tasks.Loadpaths do
       load_project(config, args)
     end
 
-    unless "--no-readd" in args do
-      Code.readd_paths()
-    end
-
     :ok
   end