Handle error result from unescape_tokens in tokenizer (#14587)

Author: lukaszsamson

lib/elixir/src/elixir_tokenizer.erl

@@ -948,17 +948,22 @@ handle_dot([$., H | T] = Original, Line, Column, DotInfo, BaseScope, Tokens) whe
           InterScope
       end,
 
-      {ok, [UnescapedPart]} = unescape_tokens([Part], Line, Column, NewScope),
+      case unescape_tokens([Part], Line, Column, NewScope) of
+        {ok, [UnescapedPart]} ->
+          case unsafe_to_atom(UnescapedPart, Line, Column, NewScope) of
+            {ok, Atom} ->
+              Token = check_call_identifier(Line, Column, H, Atom, Rest),
+              TokensSoFar = add_token_with_eol({'.', DotInfo}, Tokens),
+              tokenize(Rest, NewLine, NewColumn, NewScope, [Token | TokensSoFar]);
 
-      case unsafe_to_atom(UnescapedPart, Line, Column, NewScope) of
-        {ok, Atom} ->
-          Token = check_call_identifier(Line, Column, H, Atom, Rest),
-          TokensSoFar = add_token_with_eol({'.', DotInfo}, Tokens),
-          tokenize(Rest, NewLine, NewColumn, NewScope, [Token | TokensSoFar]);
+            {error, Reason} ->
+              error(Reason, Original, NewScope, Tokens)
+          end;
 
         {error, Reason} ->
           error(Reason, Original, NewScope, Tokens)
       end;
+
     {_NewLine, _NewColumn, _Parts, Rest, NewScope} ->
       Message = "interpolation is not allowed when calling function/macro. Found interpolation in a call starting with: ",
       error({?LOC(Line, Column), Message, [H]}, Rest, NewScope, Tokens);

lib/elixir/test/elixir/code_test.exs

@@ -514,6 +514,48 @@ defmodule CodeTest do
              }
   end
 
+  test "string_to_quoted handles unescape errors properly" do
+    # Test invalid hex escape character
+    assert {:error, {meta, message, token}} = Code.string_to_quoted("a.'\\xg'")
+
+    assert meta[:line] == 1
+    assert meta[:column] == 3
+
+    assert message ==
+             "invalid hex escape character, expected \\xHH where H is a hexadecimal digit. Syntax error after: "
+
+    assert token == "\\x"
+
+    # Test invalid Unicode escape character
+    assert {:error, {meta2, message2, token2}} = Code.string_to_quoted("a.'\\ug'")
+
+    assert meta2[:line] == 1
+    assert meta2[:column] == 3
+
+    assert message2 ==
+             "invalid Unicode escape character, expected \\uHHHH or \\u{H*} where H is a hexadecimal digit. Syntax error after: "
+
+    assert token2 == "\\u"
+
+    # Test invalid Unicode code point (surrogate pair)
+    assert {:error, {meta3, message3, token3}} = Code.string_to_quoted("a.'\\u{D800}'")
+
+    assert meta3[:line] == 1
+    assert meta3[:column] == 3
+
+    assert message3 == "invalid or reserved Unicode code point \\u{D800}. Syntax error after: "
+    assert token3 == "\\u"
+
+    # Test Unicode code point beyond valid range
+    assert {:error, {meta4, message4, token4}} = Code.string_to_quoted("a.'\\u{110000}'")
+
+    assert meta4[:line] == 1
+    assert meta4[:column] == 3
+
+    assert message4 == "invalid or reserved Unicode code point \\u{110000}. Syntax error after: "
+    assert token4 == "\\u"
+  end
+
   test "string_to_quoted raises UnicodeConversionError for invalid UTF-8 in quoted atoms and function calls" do
     invalid_utf8_cases = [
       # Quoted atom