Show code snippet on syntax and token missing errors (#11332)

Closes #11280.

Author: v0idpwn

lib/elixir/lib/code.ex

@@ -1008,7 +1008,13 @@ defmodule Code do
         {forms, comments}
 
       {:error, {location, error, token}} ->
-        :elixir_errors.parse_error(location, Keyword.get(opts, :file, "nofile"), error, token)
+        :elixir_errors.parse_error(
+          location,
+          Keyword.get(opts, :file, "nofile"),
+          error,
+          token,
+          {string, Keyword.get(opts, :line, 1), Keyword.get(opts, :column, 1)}
+        )
     end
   end
 

lib/elixir/lib/exception.ex

@@ -697,6 +697,17 @@ defmodule Exception do
     end
   end
 
+  @doc false
+  def format_snippet(snippet, error_line) do
+    line_digits = error_line |> Integer.to_string() |> byte_size()
+    placeholder = String.duplicate(" ", max(line_digits, 2))
+    padding = if line_digits < 2, do: " "
+
+    " #{placeholder} |\n" <>
+      " #{padding}#{error_line} | #{snippet.content}\n" <>
+      " #{placeholder} | #{String.duplicate(" ", snippet.offset)}^"
+  end
+
   defp format_location(opts) when is_list(opts) do
     format_file_line(Keyword.get(opts, :file), Keyword.get(opts, :line), " ")
   end
@@ -795,20 +806,56 @@ defmodule SystemLimitError do
 end
 
 defmodule SyntaxError do
-  defexception [:file, :line, :column, description: "syntax error"]
+  defexception [:file, :line, :column, :snippet, description: "syntax error"]
+
+  @impl true
+  def message(%{
+        file: file,
+        line: line,
+        column: column,
+        description: description,
+        snippet: snippet
+      })
+      when not is_nil(snippet) and not is_nil(column) do
+    Exception.format_file_line_column(Path.relative_to_cwd(file), line, column) <>
+      " " <> description <> "\n" <> Exception.format_snippet(snippet, line)
+  end
 
   @impl true
-  def message(%{file: file, line: line, column: column, description: description}) do
+  def message(%{
+        file: file,
+        line: line,
+        column: column,
+        description: description
+      }) do
     Exception.format_file_line_column(Path.relative_to_cwd(file), line, column) <>
       " " <> description
   end
 end
 
 defmodule TokenMissingError do
-  defexception [:file, :line, :column, description: "expression is incomplete"]
+  defexception [:file, :line, :snippet, :column, description: "expression is incomplete"]
+
+  @impl true
+  def message(%{
+        file: file,
+        line: line,
+        column: column,
+        description: description,
+        snippet: snippet
+      })
+      when not is_nil(snippet) and not is_nil(column) do
+    Exception.format_file_line_column(Path.relative_to_cwd(file), line, column) <>
+      " " <> description <> "\n" <> Exception.format_snippet(snippet, line)
+  end
 
   @impl true
-  def message(%{file: file, line: line, column: column, description: description}) do
+  def message(%{
+        file: file,
+        line: line,
+        column: column,
+        description: description
+      }) do
     Exception.format_file_line_column(file && Path.relative_to_cwd(file), line, column) <>
       " " <> description
   end

lib/elixir/src/elixir.erl

@@ -397,10 +397,10 @@ parser_location(Meta) ->
         {ok, Forms} ->
           Forms;
         {error, {Meta, Error, Token}} ->
-          elixir_errors:parse_error(Meta, File, Error, Token)
+          elixir_errors:parse_error(Meta, File, Error, Token, {String, StartLine, StartColumn})
       end;
     {error, {Meta, Error, Token}} ->
-      elixir_errors:parse_error(Meta, File, Error, Token)
+      elixir_errors:parse_error(Meta, File, Error, Token, {String, StartLine, StartColumn})
   end.
 
 to_binary(List) when is_list(List) -> elixir_utils:characters_to_binary(List);

lib/elixir/src/elixir_errors.erl

@@ -4,7 +4,7 @@
 %% Note that this is also called by the Erlang backend, so we also support
 %% the line number to be none (as it may happen in some erlang errors).
 -module(elixir_errors).
--export([compile_error/3, compile_error/4,  form_error/4, parse_error/4]).
+-export([compile_error/3, compile_error/4,  form_error/4, parse_error/5]).
 -export([warning_prefix/0, erl_warn/3, print_warning/3, log_and_print_warning/4, form_warn/4]).
 -include("elixir.hrl").
 -type location() :: non_neg_integer() | {non_neg_integer(), non_neg_integer()}.
@@ -82,24 +82,36 @@ compile_error(Meta, File, Format, Args) when is_list(Format)  ->
   compile_error(Meta, File, io_lib:format(Format, Args)).
 
 %% Tokenization parsing/errors.
+snippet(InputString, Location, StartLine, StartColumn) ->
+  {line, Line} = lists:keyfind(line, 1, Location),
+  case lists:keyfind(column, 1, Location) of
+    {column, Column} ->
+      Lines = string:split(InputString, "\n", all),
+      Snippet = elixir_utils:characters_to_binary(lists:nth(Line - StartLine + 1, Lines)),
+      #{content => Snippet, offset => (Column - StartColumn)};
+
+    false ->
+      nil
+  end.
 
 -spec parse_error(elixir:keyword(), binary() | {binary(), binary()},
-                  binary(), binary()) -> no_return().
-parse_error(Location, File, Error, <<>>) ->
+                  binary(), binary(), {list(), integer(), integer()}) -> no_return().
+parse_error(Location, File, Error, <<>>, {InputString, StartLine, StartColumn}) ->
   Message = case Error of
     <<"syntax error before: ">> -> <<"syntax error: expression is incomplete">>;
-    _ -> Error
+    _ -> <<Error/binary>>
   end,
-  raise(Location, File, 'Elixir.TokenMissingError', Message);
+  Snippet = snippet(InputString, Location, StartLine, StartColumn),
+  raise(Location, File, 'Elixir.TokenMissingError', Message, Snippet);
 
 %% Show a nicer message for end of line
-parse_error(Location, File, <<"syntax error before: ">>, <<"eol">>) ->
+parse_error(Location, File, <<"syntax error before: ">>, <<"eol">>, _Input) ->
   raise(Location, File, 'Elixir.SyntaxError',
         <<"unexpectedly reached end of line. The current expression is invalid or incomplete">>);
 
 
 %% Show a nicer message for keywords pt1 (Erlang keywords show up wrapped in single quotes)
-parse_error(Location, File, <<"syntax error before: ">>, Keyword)
+parse_error(Location, File, <<"syntax error before: ">>, Keyword, _Input)
     when Keyword == <<"'not'">>;
          Keyword == <<"'and'">>;
          Keyword == <<"'or'">>;
@@ -110,7 +122,7 @@ parse_error(Location, File, <<"syntax error before: ">>, Keyword)
   raise_reserved(Location, File, binary_part(Keyword, 1, byte_size(Keyword) - 2));
 
 %% Show a nicer message for keywords pt2 (Elixir keywords show up as is)
-parse_error(Location, File, <<"syntax error before: ">>, Keyword)
+parse_error(Location, File, <<"syntax error before: ">>, Keyword, _Input)
     when Keyword == <<"fn">>;
          Keyword == <<"else">>;
          Keyword == <<"rescue">>;
@@ -121,7 +133,7 @@ parse_error(Location, File, <<"syntax error before: ">>, Keyword)
   raise_reserved(Location, File, Keyword);
 
 %% Produce a human-readable message for errors before a sigil
-parse_error(Location, File, <<"syntax error before: ">>, <<"{sigil,", _Rest/binary>> = Full) ->
+parse_error(Location, File, <<"syntax error before: ">>, <<"{sigil,", _Rest/binary>> = Full, _Input) ->
   {sigil, _, Sigil, [Content | _], _, _, _} = parse_erl_term(Full),
   Content2 = case is_binary(Content) of
     true -> Content;
@@ -131,15 +143,15 @@ parse_error(Location, File, <<"syntax error before: ">>, <<"{sigil,", _Rest/bina
   raise(Location, File, 'Elixir.SyntaxError', Message);
 
 %% Binaries (and interpolation) are wrapped in [<<...>>]
-parse_error(Location, File, Error, <<"[", _/binary>> = Full) when is_binary(Error) ->
+parse_error(Location, File, Error, <<"[", _/binary>> = Full, _Input) when is_binary(Error) ->
   Term = case parse_erl_term(Full) of
     [H | _] when is_binary(H) -> <<$", H/binary, $">>;
     _ -> <<$">>
   end,
   raise(Location, File, 'Elixir.SyntaxError', <<Error/binary, Term/binary>>);
 
 %% Given a string prefix and suffix to insert the token inside the error message rather than append it
-parse_error(Location, File, {ErrorPrefix, ErrorSuffix}, Token) when is_binary(ErrorPrefix), is_binary(ErrorSuffix), is_binary(Token) ->
+parse_error(Location, File, {ErrorPrefix, ErrorSuffix}, Token, _Input) when is_binary(ErrorPrefix), is_binary(ErrorSuffix), is_binary(Token) ->
   Message = <<ErrorPrefix/binary, Token/binary, ErrorSuffix/binary >>,
   raise(Location, File, 'Elixir.SyntaxError', Message);
 
@@ -148,14 +160,15 @@ parse_error(Location, File, {ErrorPrefix, ErrorSuffix}, Token) when is_binary(Er
 %% because {char, _, _} is a valid Erlang token for an Erlang char literal. We
 %% want to represent that token as ?a in the error, according to the Elixir
 %% syntax.
-parse_error(Location, File, <<"syntax error before: ">>, <<$$, Char/binary>>) ->
+parse_error(Location, File, <<"syntax error before: ">>, <<$$, Char/binary>>, _Input) ->
   Message = <<"syntax error before: ?", Char/binary>>,
   raise(Location, File, 'Elixir.SyntaxError', Message);
 
 %% Everything else is fine as is
-parse_error(Location, File, Error, Token) when is_binary(Error), is_binary(Token) ->
-  Message = <<Error/binary, Token/binary >>,
-  raise(Location, File, 'Elixir.SyntaxError', Message).
+parse_error(Location, File, Error, Token, {InputString, StartLine, StartColumn}) when is_binary(Error), is_binary(Token) ->
+  Message = <<Error/binary, Token/binary>>,
+  Snippet = snippet(InputString, Location, StartLine, StartColumn),
+  raise(Location, File, 'Elixir.SyntaxError', Message, Snippet).
 
 parse_erl_term(Term) ->
   {ok, Tokens, _} = erl_scan:string(binary_to_list(Term)),
@@ -196,6 +209,9 @@ meta_location(Meta, File) ->
     nil    -> [{file, File}, {line, ?line(Meta)}]
   end.
 
+raise(Location, File, Kind, Message, Snippet) when is_binary(File) ->
+  raise(Kind, Message, [{file, File}, {snippet, Snippet} | Location]).
+
 raise(Location, File, Kind, Message) when is_binary(File) ->
   raise(Kind, Message, [{file, File} | Location]).
 

lib/elixir/test/elixir/code_test.exs

@@ -140,6 +140,26 @@ defmodule CodeTest do
     Code.unrequire_files([fixture_path("code_sample.exs")])
   end
 
+  test "string_to_quoted!/2 errors take lines and columns into account" do
+    message = "nofile:1:5: syntax error before: '*'\n    |\n  1 | 1 + * 3\n    |     ^"
+
+    assert_raise SyntaxError, message, fn ->
+      Code.string_to_quoted!("1 + * 3")
+    end
+
+    message = "nofile:10:5: syntax error before: '*'\n    |\n 10 | 1 + * 3\n    |     ^"
+
+    assert_raise SyntaxError, message, fn ->
+      Code.string_to_quoted!("1 + * 3", line: 10)
+    end
+
+    message = "nofile:10:7: syntax error before: '*'\n    |\n 10 | 1 + * 3\n    |     ^"
+
+    assert_raise SyntaxError, message, fn ->
+      Code.string_to_quoted!("1 + * 3", line: 10, column: 3)
+    end
+  end
+
   test "compile source" do
     assert __MODULE__.__info__(:compile)[:source] == String.to_charlist(__ENV__.file)
   end

lib/elixir/test/elixir/kernel/errors_test.exs

@@ -233,6 +233,11 @@ defmodule Kernel.ErrorsTest do
                       '%{foo: 1, :bar => :bar}'
   end
 
+  test "syntax errors include formatted snippet" do
+    message = "nofile:1:5: syntax error before: '*'\n    |\n  1 | 1 + * 3\n    |     ^"
+    assert_eval_raise SyntaxError, message, "1 + * 3"
+  end
+
   test "struct fields on defstruct" do
     assert_eval_raise ArgumentError, "struct field names must be atoms, got: 1", '''
     defmodule Kernel.ErrorsTest.StructFieldsOnDefstruct do
@@ -478,12 +483,12 @@ defmodule Kernel.ErrorsTest do
 
   test "invalid fn args" do
     assert_eval_raise TokenMissingError,
-                      "nofile:1:5: missing terminator: end (for \"fn\" starting at line 1)",
+                      ~r/nofile:1:5: missing terminator: end \(for "fn" starting at line 1\).*/,
                       'fn 1'
   end
 
   test "invalid escape" do
-    assert_eval_raise TokenMissingError, "nofile:1:3: invalid escape \\ at end of file", '1 \\'
+    assert_eval_raise TokenMissingError, ~r/nofile:1:3: invalid escape \\ at end of file/, '1 \\'
   end
 
   test "function local conflict" do