Always capture delimiter metadata for sigils (#9671)

This makes it so we're always capturing delimiter metadata for sigils
instead of requiring the `token_metadata` option. This means that we can
now show the correct sigils in ExUnit diffs (and elsewhere) for code
examples.

Author: devonestes

lib/elixir/lib/macro.ex

@@ -1007,7 +1007,13 @@ defmodule Macro do
     false
   end
 
-  defp interpolate({:<<>>, _, parts}, fun) do
+  defp interpolate(ast, fun), do: interpolate(ast, "\"", "\"", fun)
+
+  defp interpolate({:<<>>, _, [parts]}, left, right, _) when left in [~s["""\n], ~s['''\n]] do
+    <<left::binary, parts::binary, right::binary>>
+  end
+
+  defp interpolate({:<<>>, _, parts}, left, right, fun) do
     parts =
       Enum.map_join(parts, "", fn
         {:"::", _, [{{:., _, [Kernel, :to_string]}, _, [arg]}, {:binary, _, _}]} ->
@@ -1018,9 +1024,16 @@ defmodule Macro do
           binary_part(binary, 1, byte_size(binary) - 2)
       end)
 
-    <<?", parts::binary, ?">>
+    escaped = escape_sigil(parts, left)
+    <<left::binary, escaped::binary, right::binary>>
   end
 
+  defp escape_sigil(parts, "("), do: String.replace(parts, ")", ~S"\)")
+  defp escape_sigil(parts, "{"), do: String.replace(parts, "}", ~S"\}")
+  defp escape_sigil(parts, "["), do: String.replace(parts, "]", ~S"\]")
+  defp escape_sigil(parts, "<"), do: String.replace(parts, ">", ~S"\>")
+  defp escape_sigil(parts, delimiter), do: String.replace(parts, delimiter, "\\#{delimiter}")
+
   defp module_to_string(atom, _fun) when is_atom(atom) do
     inspect_no_limit(atom)
   end
@@ -1080,25 +1093,22 @@ defmodule Macro do
     :error
   end
 
-  defp sigil_call({sigil, _, [{:<<>>, _, _} = parts, args]} = ast, fun)
+  defp sigil_call({sigil, meta, [{:<<>>, _, _} = parts, args]} = ast, fun)
        when is_atom(sigil) and is_list(args) do
+    delimiter = Keyword.get(meta, :delimiter, "\"")
+    {left, right} = delimiter_pair(delimiter)
+
     case Atom.to_string(sigil) do
       <<"sigil_", name>> when name >= ?A and name <= ?Z ->
+        args = sigil_args(args, fun)
         {:<<>>, _, [binary]} = parts
-
-        formatted =
-          if :binary.last(binary) == ?\n do
-            binary = String.replace(binary, ~s["""], ~s["\\""])
-            <<?~, name, ~s["""\n], binary::binary, ~s["""], sigil_args(args, fun)::binary>>
-          else
-            {left, right} = select_sigil_container(binary)
-            <<?~, name, left, binary::binary, right, sigil_args(args, fun)::binary>>
-          end
-
+        formatted = <<?~, name, left::binary, binary::binary, right::binary, args::binary>>
         {:ok, fun.(ast, formatted)}
 
       <<"sigil_", name>> when name >= ?a and name <= ?z ->
-        {:ok, fun.(ast, "~" <> <<name>> <> interpolate(parts, fun) <> sigil_args(args, fun))}
+        args = sigil_args(args, fun)
+        formatted = "~" <> <<name>> <> interpolate(parts, left, right, fun) <> args
+        {:ok, fun.(ast, formatted)}
 
       _ ->
         :error
@@ -1109,17 +1119,13 @@ defmodule Macro do
     :error
   end
 
-  defp select_sigil_container(binary) do
-    cond do
-      :binary.match(binary, ["\""]) == :nomatch -> {?", ?"}
-      :binary.match(binary, ["\'"]) == :nomatch -> {?', ?'}
-      :binary.match(binary, ["(", ")"]) == :nomatch -> {?(, ?)}
-      :binary.match(binary, ["[", "]"]) == :nomatch -> {?[, ?]}
-      :binary.match(binary, ["{", "}"]) == :nomatch -> {?{, ?}}
-      :binary.match(binary, ["<", ">"]) == :nomatch -> {?<, ?>}
-      true -> {?/, ?/}
-    end
-  end
+  defp delimiter_pair("["), do: {"[", "]"}
+  defp delimiter_pair("{"), do: {"{", "}"}
+  defp delimiter_pair("("), do: {"(", ")"}
+  defp delimiter_pair("<"), do: {"<", ">"}
+  defp delimiter_pair("\"\"\""), do: {"\"\"\"\n", "\"\"\""}
+  defp delimiter_pair("'''"), do: {"'''\n", "'''"}
+  defp delimiter_pair(str), do: {str, str}
 
   defp sigil_args([], _fun), do: ""
   defp sigil_args(args, fun), do: fun.(args, List.to_string(args))

lib/elixir/src/elixir_parser.yrl

@@ -868,10 +868,7 @@ build_access(Expr, {List, Location}) ->
 
 build_sigil({sigil, Location, Sigil, Parts, Modifiers, Delimiter}) ->
   Meta = meta_from_location(Location),
-  MetaWithDelimiter = case ?token_metadata() of
-    true -> [{delimiter, Delimiter} | Meta];
-    false -> Meta
-  end,
+  MetaWithDelimiter = [{delimiter, Delimiter} | Meta],
   {list_to_atom("sigil_" ++ [Sigil]),
    MetaWithDelimiter,
    [{'<<>>', Meta, string_parts(Parts)}, Modifiers]}.

lib/elixir/test/elixir/code_test.exs

@@ -236,6 +236,27 @@ defmodule CodeTest do
         Code.string_to_quoted!("1 +")
       end
     end
+
+    test "delimiter information for sigils is included" do
+      string_to_quoted = &Code.string_to_quoted!(&1, token_metadata: false)
+
+      assert string_to_quoted.("~r/foo/") ==
+               {:sigil_r, [delimiter: "/", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]}
+
+      assert string_to_quoted.("~r[foo]") ==
+               {:sigil_r, [delimiter: "[", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]}
+
+      assert string_to_quoted.("~r\"foo\"") ==
+               {:sigil_r, [delimiter: "\"", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]}
+
+      meta = [delimiter: "\"\"\"", line: 1]
+      args = {:sigil_S, meta, [{:<<>>, [line: 1], ["sigil heredoc\n"]}, []]}
+      assert string_to_quoted.("~S\"\"\"\nsigil heredoc\n\"\"\"") == args
+
+      meta = [delimiter: "'''", line: 1]
+      args = {:sigil_S, meta, [{:<<>>, [line: 1], ["sigil heredoc\n"]}, []]}
+      assert string_to_quoted.("~S'''\nsigil heredoc\n'''") == args
+    end
   end
 
   describe "string_to_quoted/2 with :token_metadata" do
@@ -301,27 +322,6 @@ defmodule CodeTest do
                {:foo, [do: [line: 2], end: [line: 3], newlines: 1, closing: [line: 2], line: 1],
                 [[do: {:__block__, [], []}]]}
     end
-
-    test "adds delimiter information to sigils" do
-      string_to_quoted = &Code.string_to_quoted!(&1, token_metadata: true)
-
-      assert string_to_quoted.("~r/foo/") ==
-               {:sigil_r, [delimiter: "/", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]}
-
-      assert string_to_quoted.("~r[foo]") ==
-               {:sigil_r, [delimiter: "[", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]}
-
-      assert string_to_quoted.("~r\"foo\"") ==
-               {:sigil_r, [delimiter: "\"", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]}
-
-      meta = [delimiter: "\"\"\"", line: 1]
-      args = {:sigil_S, meta, [{:<<>>, [line: 1], ["sigil heredoc\n"]}, []]}
-      assert string_to_quoted.("~S\"\"\"\nsigil heredoc\n\"\"\"") == args
-
-      meta = [delimiter: "'''", line: 1]
-      args = {:sigil_S, meta, [{:<<>>, [line: 1], ["sigil heredoc\n"]}, []]}
-      assert string_to_quoted.("~S'''\nsigil heredoc\n'''") == args
-    end
   end
 
   describe "string_to_quoted/2 with :literal_encoder" do

lib/elixir/test/elixir/macro_test.exs

@@ -327,18 +327,56 @@ defmodule MacroTest do
 
     test "sigil call" do
       assert Macro.to_string(quote(do: ~r"123")) == ~S/~r"123"/
-      assert Macro.to_string(quote(do: ~r"123"u)) == ~S/~r"123"u/
       assert Macro.to_string(quote(do: ~r"\n123")) == ~S/~r"\\n123"/
+      assert Macro.to_string(quote(do: ~r"12\"3")) == ~S/~r"12\\"3"/
+      assert Macro.to_string(quote(do: ~r/12\/3/u)) == ~S"~r/12\/3/u"
+      assert Macro.to_string(quote(do: ~r{\n123})) == ~S/~r{\\n123}/
+      assert Macro.to_string(quote(do: ~r((1\)(2\)3))) == ~S/~r((1\)(2\)3)/
+      assert Macro.to_string(quote(do: ~r{\n1{1\}23})) == ~S/~r{\\n1{1\}23}/
+      assert Macro.to_string(quote(do: ~r|12\|3|)) == ~S"~r|12\|3|"
 
-      assert Macro.to_string(quote(do: ~r"1#{two}3")) == ~S/~r"1#{two}3"/
-      assert Macro.to_string(quote(do: ~r"1#{two}3"u)) == ~S/~r"1#{two}3"u/
+      assert Macro.to_string(quote(do: ~r[1#{two}3])) == ~S/~r[1#{two}3]/
+      assert Macro.to_string(quote(do: ~r[1[#{two}\]3])) == ~S/~r[1[#{two}\]3]/
+      assert Macro.to_string(quote(do: ~r'1#{two}3'u)) == ~S/~r'1#{two}3'u/
 
       assert Macro.to_string(quote(do: ~R"123")) == ~S/~R"123"/
       assert Macro.to_string(quote(do: ~R"123"u)) == ~S/~R"123"u/
       assert Macro.to_string(quote(do: ~R"\n123")) == ~S/~R"\n123"/
 
       assert Macro.to_string(quote(do: ~S["'(123)'"])) == ~S/~S["'(123)'"]/
 
+      assert Macro.to_string(
+               quote do
+                 ~s"""
+                 "\""foo"\""
+                 """
+               end
+             ) == ~s[~s"""\n"\\""foo"\\""\n"""]
+
+      assert Macro.to_string(
+               quote do
+                 ~s'''
+                 '\''foo'\''
+                 '''
+               end
+             ) == ~s[~s'''\n'\\''foo'\\''\n''']
+
+      assert Macro.to_string(
+               quote do
+                 ~s"""
+                 "\"foo\""
+                 """
+               end
+             ) == ~s[~s"""\n"\\"foo\\""\n"""]
+
+      assert Macro.to_string(
+               quote do
+                 ~s'''
+                 '\"foo\"'
+                 '''
+               end
+             ) == ~s[~s'''\n'\\"foo\\"'\n''']
+
       assert Macro.to_string(
                quote do
                  ~S"""

lib/ex_unit/test/ex_unit/diff_test.exs

@@ -557,7 +557,7 @@ defmodule ExUnit.DiffTest do
   test "structs with inspect" do
     refute_diff(
       ~D[2017-10-01] = ~D[2017-10-02],
-      ~s/-~D"2017-10-01"-/,
+      ~s/-~D[2017-10-01]-/,
       "~D[2017-10-0+2+]"
     )
 
@@ -569,7 +569,7 @@ defmodule ExUnit.DiffTest do
 
     refute_diff(
       ~D[2017-10-02] = "2017-10-01",
-      ~s/-~D"2017-10-02"-/,
+      ~s/-~D[2017-10-02]-/,
       ~s/+"2017-10-01"+/
     )
   end