Merge pull request #1909 from vanstee/exunit-support-filters

Support ExUnit filters

Author: José Valim

lib/ex_unit/lib/ex_unit.ex

@@ -144,6 +144,10 @@ defmodule ExUnit do
                and prints each test case and test while running;
 
   * `:autorun` - If ExUnit should run by default on exit, defaults to `true`;
+
+  * `:include` - Specify which tests are run by skipping tests that do not match the filter
+
+  * `:exclude` - Specify which tests are run by skipping tests that match the filter
   """
   def configure(options) do
     Enum.each options, fn { k, v } ->
@@ -169,4 +173,22 @@ defmodule ExUnit do
     opts = Keyword.put_new(configuration, :color, IO.ANSI.terminal?)
     ExUnit.Runner.run async, sync, opts, load_us
   end
+
+  @doc false
+  def parse_filters(filters) do
+    Enum.map filters, fn filter ->
+      [key, value] = case String.split(filter, ":", global: false) do
+        [key, value] -> [key, value]
+        [key] -> [key, true]
+      end
+
+      value = case value do
+        "true"  -> true
+        "false" -> false
+        value   -> value
+      end
+
+      { Kernel.binary_to_atom(key), value }
+    end
+  end
 end

lib/ex_unit/lib/ex_unit/case.ex

@@ -110,6 +110,37 @@ defmodule ExUnit.Case do
   * `:test` - the test name
   * `:line` - the line the test was defined
 
+  ## Filters
+
+  Tags can also be used to identify specific tests, which can then be included
+  or excluded using filters. Filters are defined as key-value pairs, similar to
+  tags, and are used to match against the tags given for each test. For example
+  the following command will skip any test that contains the `:os` tag but has
+  a value other than `"unix"`.
+
+      mix test --include os:unix
+
+  If your tags are defined using boolean values, you can use the shorthand
+  version by only specifying the tag name. The value will be automatically set
+  to `true`. To skip all tests with a tag of `slow: true` run the following
+  command:
+
+      mix test --exclude slow
+
+  Filters can also be combined to further limit the tests to be run. When
+  defining filters with the same tag name, tests that match either filter will
+  be run. The following command will skip any test that contains the `:os` tag
+  but has a value other than `"unix"` or `"win32"`:
+
+      mix test --include os:unix --include os:win32
+
+  However, if multiple filters with different tag names are given, only tests
+  that match the filter defined for each unique tag name will be run. This
+  command will only run tests that have both `os: "unix"` and `type: "unit"`
+  tags.
+
+      mix test --include os:unix --include type:unit
+
   """
 
   @doc false

lib/ex_unit/lib/ex_unit/cli_formatter.ex

@@ -5,10 +5,10 @@ defmodule ExUnit.CLIFormatter do
 
   use GenServer.Behaviour
 
-  import ExUnit.Formatter, only: [format_time: 2, format_test_failure: 5, format_test_case_failure: 4]
+  import ExUnit.Formatter, only: [format_time: 2, format_filters: 2, format_test_failure: 5, format_test_case_failure: 4]
 
   defrecord Config, tests_counter: 0, invalids_counter: 0, failures_counter: 0,
-                    trace: false, color: true, previous: nil
+                    skips_counter: 0, trace: false, color: true, previous: nil
 
   ## Behaviour
 
@@ -40,6 +40,7 @@ defmodule ExUnit.CLIFormatter do
   ## Callbacks
 
   def init(opts) do
+    print_filters(Keyword.take(opts, [:include, :exclude]))
     { :ok, Config.new(opts) }
   end
 
@@ -77,6 +78,10 @@ defmodule ExUnit.CLIFormatter do
                       .update_invalids_counter(&(&1 + 1)) }
   end
 
+  def handle_cast({ :test_finished, ExUnit.Test[state: { :skip, _ }] }, config = Config[]) do
+    { :noreply, config.previous(:skip).update_skips_counter(&(&1 + 1)) }
+  end
+
   def handle_cast({ :test_finished, test }, config) do
     if config.trace do
       IO.puts failure(trace_test_result(test), config)
@@ -150,6 +155,11 @@ defmodule ExUnit.CLIFormatter do
     end
   end
 
+  defp print_filters([include: include, exclude: exclude]) do
+    unless Enum.empty?(include), do: IO.puts format_filters(include, :include)
+    unless Enum.empty?(exclude), do: IO.puts format_filters(exclude, :exclude)
+  end
+
   defp print_test_failure(ExUnit.Test[name: name, case: mod, state: { :failed, tuple }], config) do
     formatted = format_test_failure(mod, name, tuple, config.failures_counter + 1, &formatter(&1, &2, config))
     print_any_failure formatted, config

lib/ex_unit/lib/ex_unit/formatter.ex

@@ -67,6 +67,23 @@ defmodule ExUnit.Formatter do
     end
   end
 
+  @doc """
+  Formats filters used to constain cases to be run.
+
+  ## Examples
+
+    iex> format_filters([run: true, slow: false], :include)
+    "Including tags: [run: true, slow: false]"
+
+  """
+  @spec format_filters(Keyword.t, atom) :: String.t
+  def format_filters(filters, type) do
+    case type do
+      :include -> "Including tags: #{inspect filters}"
+      :exclude -> "Excluding tags: #{inspect filters}"
+    end
+  end
+
   @doc %S"""
   Receives a test and formats its failure.
   """

lib/ex_unit/lib/ex_unit/runner.ex

@@ -2,7 +2,8 @@ defmodule ExUnit.Runner do
   @moduledoc false
 
   defrecord Config, formatter: ExUnit.CLIFormatter, formatter_id: nil,
-                    max_cases: 4, taken_cases: 0, async_cases: [], sync_cases: []
+                    max_cases: 4, taken_cases: 0, async_cases: [],
+                    sync_cases: [], include: nil, exclude: nil
 
   def run(async, sync, opts, load_us) do
     opts = normalize_opts(opts)
@@ -125,9 +126,22 @@ defmodule ExUnit.Runner do
   end
 
   defp run_test(config, test, context) do
-    case_name = test.case
     config.formatter.test_started(config.formatter_id, test)
 
+    filters = combine_filters([include: config.include, exclude: config.exclude])
+    result = evaluate_filters(filters, test.tags)
+
+    test = case result do
+      { :error, tag } -> skip_test(test, tag)
+      :ok -> spawn_test(config, test, context)
+    end
+
+    config.formatter.test_finished(config.formatter_id, test)
+  end
+
+  defp spawn_test(_config, test, context) do
+    case_name = test.case
+
     # Run test in a new process so that we can trap exits for a single test
     self_pid = self
     { test_pid, test_ref } = Process.spawn_monitor fn ->
@@ -157,13 +171,16 @@ defmodule ExUnit.Runner do
 
     receive do
       { ^test_pid, :test_finished, test } ->
-        config.formatter.test_finished(config.formatter_id, test)
+        test
       { :DOWN, ^test_ref, :process, ^test_pid, { error, stacktrace } } ->
-        test = test.state { :failed, { :EXIT, error, prune_stacktrace(stacktrace) } }
-        config.formatter.test_finished(config.formatter_id, test)
+        test.state { :failed, { :EXIT, error, prune_stacktrace(stacktrace) } }
     end
   end
 
+  defp skip_test(test, mismatch) do
+    test.state { :skip, "due to #{mismatch} filter" }
+  end
+
   ## Helpers
 
   defp take_async_cases(Config[] = config, count) do
@@ -182,6 +199,42 @@ defmodule ExUnit.Runner do
     end
   end
 
+  def evaluate_filters(filters, tags) do
+    Enum.find_value tags, :ok, fn { tag_key, _ } = tag ->
+      unless tag_accepted?(filters, tag), do: { :error, tag_key }
+    end
+  end
+
+  defp tag_accepted?([include: include, exclude: exclude], tag) do
+    tag_included?(include, tag) and not tag_excluded?(exclude, tag)
+  end
+
+  defp tag_included?(include, { tag_key, tag_value }) do
+    case Dict.fetch(include, tag_key) do
+      { :ok, allowed } -> tag_value in allowed
+      :error -> true
+    end
+  end
+
+  defp tag_excluded?(exclude, { tag_key, tag_value })  do
+    case Dict.fetch(exclude, tag_key) do
+      { :ok, forbidden } -> tag_value in forbidden
+      :error -> false
+    end
+  end
+
+  defp combine_filters([include: include, exclude: exclude]) do
+    include = group_by_key(include)
+    exclude = group_by_key(exclude)
+    [include: include, exclude: exclude]
+  end
+
+  defp group_by_key(dict) do
+    Enum.reduce dict, HashDict.new, fn { key, value }, acc ->
+      Dict.update acc, key, [value], &[value|&1]
+    end
+  end
+
   defp pruned_stacktrace, do: prune_stacktrace(System.stacktrace)
 
   # Assertions can pop-up in the middle of the stack

lib/ex_unit/mix.exs

@@ -11,6 +11,8 @@ defmodule ExUnit.Mixfile do
       env: [
         autorun: true,
         trace: false,
-        formatter: ExUnit.CLIFormatter ] ]
+        formatter: ExUnit.CLIFormatter,
+        include: [],
+        exclude: [] ] ]
   end
 end

lib/ex_unit/test/ex_unit/formatter_test.exs

@@ -18,6 +18,12 @@ defmodule ExUnit.FormatterTest do
     end
   end
 
+  test "formats test case filters" do
+    filters = [run: true, slow: false]
+    assert format_filters(filters, :include) =~ "Including tags: [run: true, slow: false]"
+    assert format_filters(filters, :exclude) =~ "Excluding tags: [run: true, slow: false]"
+  end
+
   test "formats test errors" do
     failure = { :error, catch_error(raise "oops"), [] }
     assert format_test_failure(Hello, :world, failure, 1, nil) =~ """

lib/ex_unit/test/ex_unit/runner_test.exs

@@ -0,0 +1,17 @@
+Code.require_file "../test_helper.exs", __DIR__
+
+defmodule ExUnit.RunnerTest do
+  use ExUnit.Case, async: true
+
+  test "filters with matching tags are ored and non-matching tags are anded" do
+    filters = [include: [os: [:unix, :win32], type: [:unit]], exclude: []]
+
+    assert :ok = ExUnit.Runner.evaluate_filters(filters, [os: :unix])
+    assert :ok = ExUnit.Runner.evaluate_filters(filters, [os: :unix, type: :unit])
+    refute :ok = ExUnit.Runner.evaluate_filters(filters, [os: :unix, type: :integration])
+    assert :ok = ExUnit.Runner.evaluate_filters(filters, [os: :win32])
+    assert :ok = ExUnit.Runner.evaluate_filters(filters, [os: :win32, type: :unit])
+    refute :ok = ExUnit.Runner.evaluate_filters(filters, [os: :win32, type: :integration])
+    assert :ok = ExUnit.Runner.evaluate_filters(filters, [os: :unix, os: :win32])
+  end
+end

lib/ex_unit/test/ex_unit_test.exs

@@ -28,6 +28,54 @@ defmodule ExUnit.NilFormatter do
   end
 end
 
+defmodule ExUnit.TestsCounterFormatter do
+  @timeout 30_000
+  @behaviour ExUnit.Formatter
+
+  use GenServer.Behaviour
+
+  def suite_started(opts) do
+    { :ok, pid } = :gen_server.start_link(__MODULE__, opts, [])
+    pid
+  end
+
+  def suite_finished(id, _run_us, _load_us) do
+    :gen_server.call(id, { :suite_finished }, @timeout)
+  end
+
+  def case_started(_id, _test_case) do
+    :ok
+  end
+
+  def case_finished(_id, _test_case) do
+    :ok
+  end
+
+  def test_started(_id, _test) do
+    :ok
+  end
+
+  def test_finished(id, test) do
+    :gen_server.cast(id, { :test_finished, test })
+  end
+
+  def init(_opts) do
+    { :ok, 0 }
+  end
+
+  def handle_call({ :suite_finished }, _from, tests_counter) do
+    { :stop, :normal, tests_counter, tests_counter }
+  end
+
+  def handle_cast({ :test_finished, ExUnit.Test[state: { :skip, _ }] }, tests_counter) do
+    { :noreply, tests_counter }
+  end
+
+  def handle_cast({ :test_finished, _ }, tests_counter) do
+    { :noreply, tests_counter + 1 }
+  end
+end
+
 defmodule ExUnitTest do
   use ExUnit.Case, async: false
 
@@ -48,4 +96,41 @@ defmodule ExUnitTest do
 
     assert ExUnit.run == 1
   end
+
+  test "filtering cases with tags" do
+    ExUnit.configure(formatter: ExUnit.TestsCounterFormatter)
+
+    defmodule ParityTest do
+      use ExUnit.Case, async: false
+
+      test "zero", do: assert true
+
+      @tag even: false
+      test "one", do: assert true
+
+      @tag even: true
+      test "two", do: assert true
+
+      @tag even: false
+      test "three", do: assert true
+    end
+
+    test_cases = ExUnit.Server.start_run
+
+    assert run_with_filter([include: [even: true]], test_cases) == 2
+    assert run_with_filter([exclude: [even: true]], test_cases) == 3
+    assert run_with_filter([include: [even: false]], test_cases) == 3
+    assert run_with_filter([exclude: [even: false]], test_cases) == 2
+  end
+
+  test "parsing filters" do
+    assert ExUnit.parse_filters(["run"]) == [run: true]
+    assert ExUnit.parse_filters(["run:true"]) == [run: true]
+    assert ExUnit.parse_filters(["run:test"]) == [run: "test"]
+  end
+
+  defp run_with_filter(filters, { async, sync, load_us }) do
+    opts = Keyword.merge(ExUnit.configuration, filters)
+    ExUnit.Runner.run(async, sync, opts, load_us)
+  end
 end