Return values from running doctests and order tests

Author: josevalim

lib/ex_unit/lib/ex_unit/doc_test.ex

@@ -238,9 +238,9 @@ defmodule ExUnit.DocTest do
 
     extract(module)
     |> filter_by_opts(module, opts)
-    |> Stream.with_index(1)
-    |> Enum.map(fn {test, acc} ->
-      compile_test(test, module, import, acc, file, tags)
+    |> Enum.sort_by(& &1.line)
+    |> Enum.with_index(fn test, index ->
+      compile_test(test, module, import, index + 1, file, tags)
     end)
   end
 
@@ -380,7 +380,7 @@ defmodule ExUnit.DocTest do
   def __test__(value, expected, doctest, last_expr, expected_expr, stack) do
     case value do
       ^expected ->
-        :ok
+        {:ok, value}
 
       _ ->
         error = [
@@ -411,7 +411,7 @@ defmodule ExUnit.DocTest do
 
     case result do
       :ok ->
-        :ok
+        {:ok, value}
 
       {extra, stack} ->
         expr = "inspect(#{last_expr}) === #{String.trim(expected_expr)}"

lib/ex_unit/test/ex_unit/doc_test_test.exs

@@ -635,18 +635,17 @@ defmodule ExUnit.DocTestTest do
            """
 
     assert output =~ """
-             9) doctest ExUnit.DocTestTest.Invalid.dedented_past_fence/0 (9) (ExUnit.DocTestTest.ActuallyCompiled)
+            9) doctest ExUnit.DocTestTest.Invalid.b/0 (9) (ExUnit.DocTestTest.ActuallyCompiled)
                 test/ex_unit/doc_test_test.exs:#{doctest_line}
-                Doctest did not compile, got: (SyntaxError) test/ex_unit/doc_test_test.exs:#{starting_line + 57}:5: unexpected token: "`" (column 5, code point U+0060)
-                 #{line_placeholder(starting_line + 57)} |
-                 #{starting_line + 57} |     ```
-                 #{line_placeholder(starting_line + 57)} |     ^
+                Doctest did not compile, got: (SyntaxError) test/ex_unit/doc_test_test.exs:#{starting_line + 33}:6: syntax error before: '*'
+                 #{line_placeholder(starting_line + 33)} |
+                 #{starting_line + 33} |  1 + * 1
+                 #{line_placeholder(starting_line + 33)} |      ^
                 doctest:
-                  iex> 1 + 2
-                  3
-                      ```
+                  iex> 1 + * 1
+                  1
                 stacktrace:
-                  test/ex_unit/doc_test_test.exs:#{starting_line + 56}: ExUnit.DocTestTest.Invalid (module)
+                  test/ex_unit/doc_test_test.exs:#{starting_line + 33}: ExUnit.DocTestTest.Invalid (module)
            """
 
     assert output =~ """
@@ -680,15 +679,30 @@ defmodule ExUnit.DocTestTest do
            """
 
     assert output =~ """
-            12) doctest ExUnit.DocTestTest.Invalid.invalid_utf8/0 (12) (ExUnit.DocTestTest.ActuallyCompiled)
+            12) doctest ExUnit.DocTestTest.Invalid.dedented_past_fence/0 (12) (ExUnit.DocTestTest.ActuallyCompiled)
+                test/ex_unit/doc_test_test.exs:#{doctest_line}
+                Doctest did not compile, got: (SyntaxError) test/ex_unit/doc_test_test.exs:#{starting_line + 57}:5: unexpected token: "`" (column 5, code point U+0060)
+                 #{line_placeholder(starting_line + 57)} |
+                 #{starting_line + 57} |     ```
+                 #{line_placeholder(starting_line + 57)} |     ^
+                doctest:
+                  iex> 1 + 2
+                  3
+                      ```
+                stacktrace:
+                  test/ex_unit/doc_test_test.exs:#{starting_line + 56}: ExUnit.DocTestTest.Invalid (module)
+           """
+
+    assert output =~ """
+            13) doctest ExUnit.DocTestTest.Invalid.invalid_utf8/0 (13) (ExUnit.DocTestTest.ActuallyCompiled)
                 test/ex_unit/doc_test_test.exs:#{doctest_line}
                 Doctest did not compile, got: (UnicodeConversionError) invalid encoding starting at <<255, 34, 41>>
                 stacktrace:
                   test/ex_unit/doc_test_test.exs:#{starting_line + 63}: ExUnit.DocTestTest.Invalid (module)
            """
 
     assert output =~ """
-            13) doctest ExUnit.DocTestTest.Invalid.misplaced_opaque_type/0 (13) (ExUnit.DocTestTest.ActuallyCompiled)
+            14) doctest ExUnit.DocTestTest.Invalid.misplaced_opaque_type/0 (14) (ExUnit.DocTestTest.ActuallyCompiled)
                 test/ex_unit/doc_test_test.exs:#{doctest_line}
                 Doctest did not compile, got: (TokenMissingError) test/ex_unit/doc_test_test.exs:#{starting_line + 69}:7: missing terminator: } (for "{" starting at line #{starting_line + 69})
                  #{line_placeholder(starting_line + 69)} |
@@ -702,28 +716,8 @@ defmodule ExUnit.DocTestTest do
                   test/ex_unit/doc_test_test.exs:#{starting_line + 69}: ExUnit.DocTestTest.Invalid (module)
            """
 
-    assert output =~ "14) doctest ExUnit.DocTestTest.Invalid.raising_inspect/0"
-    assert output =~ "iex> ExUnit.DocTestTest.Haiku.new(:this, :is, {:not, :a, :haiku})"
-
-    assert output =~
-             "test/ex_unit/doc_test_test.exs:#{starting_line + 82}: ExUnit.DocTestTest.Invalid (module)"
-
     assert output =~ """
-            15) doctest ExUnit.DocTestTest.Invalid.b/0 (15) (ExUnit.DocTestTest.ActuallyCompiled)
-                test/ex_unit/doc_test_test.exs:#{doctest_line}
-                Doctest did not compile, got: (SyntaxError) test/ex_unit/doc_test_test.exs:#{starting_line + 33}:6: syntax error before: '*'
-                 #{line_placeholder(starting_line + 33)} |
-                 #{starting_line + 33} |  1 + * 1
-                 #{line_placeholder(starting_line + 33)} |      ^
-                doctest:
-                  iex> 1 + * 1
-                  1
-                stacktrace:
-                  test/ex_unit/doc_test_test.exs:#{starting_line + 33}: ExUnit.DocTestTest.Invalid (module)
-           """
-
-    assert output =~ """
-            16) doctest ExUnit.DocTestTest.Invalid.t/0 (16) (ExUnit.DocTestTest.ActuallyCompiled)
+            15) doctest ExUnit.DocTestTest.Invalid.t/0 (15) (ExUnit.DocTestTest.ActuallyCompiled)
                 test/ex_unit/doc_test_test.exs:#{doctest_line}
                 Doctest did not compile, got: (SyntaxError) test/ex_unit/doc_test_test.exs:#{starting_line + 75}:6: syntax error before: '*'
                  #{line_placeholder(starting_line + 75)} |
@@ -735,6 +729,12 @@ defmodule ExUnit.DocTestTest do
                 stacktrace:
                   test/ex_unit/doc_test_test.exs:#{starting_line + 75}: ExUnit.DocTestTest.Invalid (module)
            """
+
+    assert output =~ "16) doctest ExUnit.DocTestTest.Invalid.raising_inspect/0"
+    assert output =~ "iex> ExUnit.DocTestTest.Haiku.new(:this, :is, {:not, :a, :haiku})"
+
+    assert output =~
+             "test/ex_unit/doc_test_test.exs:#{starting_line + 82}: ExUnit.DocTestTest.Invalid (module)"
   end
 
   test "pattern matching assertions in doctests" do
@@ -900,24 +900,7 @@ defmodule ExUnit.DocTestTest do
     end
   end
 
-  test "fails when testing single function not found" do
-    message = """
-    test/ex_unit/doc_test_test\.exs: undefined or private function given to doctest:
-
-        ExUnit.DocTestTest.SomewhatGoodModuleWithOnly.three/0
-
-    """
-
-    assert_raise ExUnit.DocTest.Error, message, fn ->
-      defmodule NeverCompiled do
-        import ExUnit.DocTest
-
-        doctest ExUnit.DocTestTest.SomewhatGoodModuleWithOnly, only: [three: 0], import: true
-      end
-    end
-  end
-
-  test "fails when testing multiple functions not found" do
+  test "fails when testing functions are not found" do
     message = """
     test/ex_unit/doc_test_test\.exs: undefined or private functions given to doctest:
 
@@ -989,6 +972,16 @@ defmodule ExUnit.DocTestTest do
     end
   end
 
+  test "doctest direct invocation" do
+    defmodule Direct do
+      use ExUnit.Case, register: false
+      doctest ExUnit.DocTestTest.GoodModule, import: true
+    end
+
+    [head | _] = Direct.__ex_unit__().tests
+    assert apply(Direct, head.name, [%{}]) == {:ok, 2}
+  end
+
   defp line_placeholder(line_number) do
     digits =
       line_number