Handle more incomplete expressions in cursor_context_to_quoted

Author: josevalim

lib/elixir/src/elixir_tokenizer.erl

@@ -551,8 +551,10 @@ tokenize([$: | String] = Original, Line, Column, Scope, Tokens) ->
       NewScope = maybe_warn_for_ambiguous_bang_before_equals(atom, Unencoded, Rest, Line, Column, Scope),
       Token = {atom, {Line, Column, nil}, Atom},
       tokenize(Rest, Line, Column + 1 + Length, NewScope, [Token | Tokens]);
-    empty ->
+    empty when Scope#elixir_tokenizer.cursor_completion == false ->
       unexpected_token(Original, Line, Column, Scope, Tokens);
+    empty ->
+      tokenize([], Line, Column, Scope, Tokens);
     {error, Reason} ->
       error(Reason, Original, Scope, Tokens)
   end;
@@ -565,17 +567,24 @@ tokenize([H | T], Line, Column, Scope, Tokens) when ?is_digit(H) ->
   case tokenize_number(T, [H], 1, false) of
     {error, Reason, Original} ->
       error({Line, Column, Reason, Original}, T, Scope, Tokens);
-    {[I | _], _Number, Original, _Length} when ?is_upcase(I); ?is_downcase(I); I == $_ ->
-      Msg =
-        io_lib:format(
-          "invalid character ~ts after number ~ts. If you intended to write a number, "
-          "make sure to add the proper punctuation character after the number (space, comma, etc). "
-          "If you meant to write an identifier, note that identifiers in Elixir cannot start with numbers. "
-          "Unexpected token: ",
-          [[I], Original]
-        ),
-
-      error({Line, Column, Msg, [I]}, T, Scope, Tokens);
+    {[I | Rest], Number, Original, _Length} when ?is_upcase(I); ?is_downcase(I); I == $_ ->
+      if
+        Number == 0, (I =:= $x) orelse (I =:= $o) orelse (I =:= $b), Rest == [],
+        Scope#elixir_tokenizer.cursor_completion /= false ->
+          tokenize([], Line, Column, Scope, Tokens);
+
+        true ->
+          Msg =
+            io_lib:format(
+              "invalid character ~ts after number ~ts. If you intended to write a number, "
+              "make sure to add the proper punctuation character after the number (space, comma, etc). "
+              "If you meant to write an identifier, note that identifiers in Elixir cannot start with numbers. "
+              "Unexpected token: ",
+              [[I], Original]
+            ),
+
+          error({Line, Column, Msg, [I]}, T, Scope, Tokens)
+      end;
     {Rest, Number, Original, Length} when is_integer(Number) ->
       Token = {int, {Line, Column, Number}, Original},
       tokenize(Rest, Line, Column + Length, Scope, [Token | Tokens]);
@@ -679,9 +688,16 @@ tokenize(String, Line, Column, Scope, Tokens) ->
     {keyword, Atom, Type, Rest, Length} ->
       tokenize_keyword(Type, Rest, Line, Column, Atom, Length, Scope, Tokens);
 
-    empty ->
+    empty when Scope#elixir_tokenizer.cursor_completion == false ->
       unexpected_token(String, Line, Column, Scope, Tokens);
 
+    empty ->
+      case String of
+        [$~, L] when ?is_upcase(L); ?is_downcase(L) -> tokenize([], Line, Column, Scope, Tokens);
+        [$~] -> tokenize([], Line, Column, Scope, Tokens);
+        _ -> unexpected_token(String, Line, Column, Scope, Tokens)
+      end;
+
     {error, Reason} ->
       error(Reason, String, Scope, Tokens)
   end.

lib/elixir/test/elixir/code_fragment_test.exs

@@ -857,16 +857,14 @@ defmodule CodeFragmentTest do
     test "maps and structs" do
       assert cc2q("%") == s2q("__cursor__()")
       assert cc2q("%{") == s2q("%{__cursor__()}")
+      assert cc2q("%{bar:") == s2q("%{__cursor__()}")
       assert cc2q("%{bar: ") == s2q("%{bar: __cursor__()}")
       assert cc2q("%{bar: baz,") == s2q("%{bar: baz, __cursor__()}")
 
       assert cc2q("%Foo") == s2q("__cursor__()")
       assert cc2q("%Foo{") == s2q("%Foo{__cursor__()}")
       assert cc2q("%Foo{bar: ") == s2q("%Foo{bar: __cursor__()}")
       assert cc2q("%Foo{bar: baz,") == s2q("%Foo{bar: baz, __cursor__()}")
-
-      assert {:error, {_, "unexpected token: ", "\":\" (column 9, code point U+003A)"}} =
-               cc2q("%Foo{bar:")
     end
 
     test "removes tokens until opening" do
@@ -908,6 +906,15 @@ defmodule CodeFragmentTest do
       assert cc2q("foo(bar do :done end |>") == s2q("foo(__cursor__())")
     end
 
+    test "incomplete expressions" do
+      assert cc2q("foo(123, :") == s2q("foo(123, __cursor__())")
+      assert cc2q("foo(123, %") == s2q("foo(123, __cursor__())")
+      assert cc2q("foo(123, 0x") == s2q("foo(123, __cursor__())")
+      assert cc2q("foo(123, ~") == s2q("foo(123, __cursor__())")
+      assert cc2q("foo(123, ~r") == s2q("foo(123, __cursor__())")
+      assert cc2q("foo(123, ~r/") == s2q("foo(123, __cursor__())")
+    end
+
     test "options" do
       opts = [columns: true]
       assert cc2q("foo(", opts) == s2q("foo(__cursor__())", opts)