Support operators in container_cursor_to_quoted

Author: josevalim

lib/elixir/lib/code/fragment.ex

@@ -962,9 +962,8 @@ defmodule Code.Fragment do
   This function receives a string with an Elixir code fragment,
   representing a cursor position, and converts such string to
   AST with the inclusion of special `__cursor__()` node representing
-  the cursor position within its parent.
+  the cursor position within its container (i.e. its parent).
 
-  The parent node is any function call, tuple, list, map, and so on.
   For example, take this code, which would be given as input:
 
       max(some_value,
@@ -995,16 +994,7 @@ defmodule Code.Fragment do
   Calls without parenthesis are also supported, as we assume the
   brackets are implicit.
 
-  Operators and anonymous functions are not containers, and therefore
-  will be discarded. The following will all return the same AST:
-
-      max(some_value,
-      max(some_value, fn x -> x end
-      max(some_value, 1 + another_val
-      max(some_value, 1 |> some_fun() |> another_fun
-
-  On the other hand, tuples, lists, maps, and binaries all retain the
-  cursor position:
+  Tuples, lists, maps, and binaries all retain the cursor position:
 
       max(some_value, [1, 2,
 
@@ -1049,18 +1039,11 @@ defmodule Code.Fragment do
 
       iex> Code.Fragment.container_cursor_to_quoted("if(is_atom(var))")
       {:ok, {:__cursor__, [line: 1], []}}
-      iex> Code.Fragment.container_cursor_to_quoted("alias Foo.Bar")
-      {:ok, {:__cursor__, [line: 1], []}}
-
-  Operators are never considered containers:
-
-      iex> Code.Fragment.container_cursor_to_quoted("if(foo +")
-      {:ok, {:if, [line: 1], [{:__cursor__, [line: 1], []}]}}
 
-  with the exception of `::` inside binaries and `|` inside maps:
+  Operators are also included from Elixir v1.15:
 
-      iex> Code.Fragment.container_cursor_to_quoted("<<some::integer")
-      {:ok, {:<<>>, [line: 1], [{:"::", [line: 1], [{:some, [line: 1], nil}, {:__cursor__, [line: 1], []}]}]}}
+      iex> Code.Fragment.container_cursor_to_quoted("foo +")
+      {:ok, {:+, [line: 1], [{:foo, [line: 1], nil}, {:__cursor__, [line: 1], []}]}}
 
   ## Options
 

lib/elixir/src/elixir_tokenizer.erl

@@ -1784,10 +1784,13 @@ prune_tokens([{kw_identifier_safe, _, _} | _] = Tokens, [], Terminators) ->
   {Tokens, Terminators};
 prune_tokens([{kw_identifier_unsafe, _, _} | _] = Tokens, [], Terminators) ->
   {Tokens, Terminators};
-%%% we usually skip operators, except these contextual ones
-prune_tokens([{type_op, _, '::'} | _] = Tokens, [], [{'<<', _, _} | _] = Terminators) ->
-  {Tokens, Terminators};
-prune_tokens([{pipe_op, _, '|'} | _] = Tokens, [], [{'{', _, _} | _] = Terminators) ->
+prune_tokens([{OpType, _, _} | _] = Tokens, [], Terminators)
+  when OpType =:= comp_op; OpType =:= at_op; OpType =:= unary_op; OpType =:= and_op;
+       OpType =:= or_op; OpType =:= arrow_op; OpType =:= match_op; OpType =:= in_op;
+       OpType =:= in_match_op; OpType =:= type_op; OpType =:= dual_op; OpType =:= mult_op;
+       OpType =:= power_op; OpType =:= concat_op; OpType =:= range_op; OpType =:= xor_op;
+       OpType =:= pipe_op; OpType =:= stab_op; OpType =:= when_op; OpType =:= assoc_op;
+       OpType =:= rel_op; OpType =:= ternary_op ->
   {Tokens, Terminators};
 %%% or we traverse until the end.
 prune_tokens([_ | Tokens], Opener, Terminators) ->

lib/elixir/test/elixir/code_fragment_test.exs

@@ -1096,6 +1096,17 @@ defmodule CodeFragmentTest do
       assert cc2q(~S|"foo #{"bar #{{|) == s2q(~S|"foo #{"bar #{{__cursor__()}}"}"|)
     end
 
+    test "keeps operators" do
+      assert cc2q("1 + 2") == s2q("1 + __cursor__()")
+    end
+
+    test "keeps function calls without parens" do
+      # assert cc2q("alias foo") == s2q("alias __cursor__()")
+      # assert cc2q("alias Foo.Bar") == s2q("alias __cursor__()")
+      assert cc2q("alias Foo.Bar,") == s2q("alias Foo.Bar, __cursor__()")
+      assert cc2q("alias Foo.Bar, as: ") == s2q("alias Foo.Bar, as: __cursor__()")
+    end
+
     test "do-end blocks" do
       assert cc2q("foo do baz") == s2q("foo do __cursor__() end")
       assert cc2q("foo do bar; baz") == s2q("foo do bar; __cursor__() end")
@@ -1166,20 +1177,19 @@ defmodule CodeFragmentTest do
       assert cc2q("(123") == s2q("(__cursor__())")
       assert cc2q("[foo") == s2q("[__cursor__()]")
       assert cc2q("{'foo'") == s2q("{__cursor__()}")
-      assert cc2q("<<1+2") == s2q("<<__cursor__()>>")
       assert cc2q("foo do :atom") == s2q("foo do __cursor__() end")
       assert cc2q("foo(:atom") == s2q("foo(__cursor__())")
     end
 
     test "removes tokens until comma" do
       assert cc2q("[bar, 123") == s2q("[bar, __cursor__()]")
       assert cc2q("{bar, 'foo'") == s2q("{bar, __cursor__()}")
-      assert cc2q("<<bar, 1+2") == s2q("<<bar, __cursor__()>>")
+      assert cc2q("<<bar, \"sample\"") == s2q("<<bar, __cursor__()>>")
       assert cc2q("foo(bar, :atom") == s2q("foo(bar, __cursor__())")
       assert cc2q("foo bar, :atom") == s2q("foo(bar, __cursor__())")
     end
 
-    test "removes functions" do
+    test "removes anonymous functions" do
       assert cc2q("(fn") == s2q("(__cursor__())")
       assert cc2q("(fn x") == s2q("(__cursor__())")
       assert cc2q("(fn x ->") == s2q("(__cursor__())")
@@ -1194,20 +1204,20 @@ defmodule CodeFragmentTest do
     end
 
     test "removes closed terminators" do
-      assert cc2q("foo([1, 2, 3] |>") == s2q("foo(__cursor__())")
-      assert cc2q("foo({1, 2, 3} |>") == s2q("foo(__cursor__())")
-      assert cc2q("foo((1, 2, 3) |>") == s2q("foo(__cursor__())")
-      assert cc2q("foo(<<1, 2, 3>> |>") == s2q("foo(__cursor__())")
-      assert cc2q("foo(bar do :done end |>") == s2q("foo(__cursor__())")
+      assert cc2q("foo([1, 2, 3]") == s2q("foo(__cursor__())")
+      assert cc2q("foo({1, 2, 3}") == s2q("foo(__cursor__())")
+      assert cc2q("foo((1, 2, 3)") == s2q("foo(__cursor__())")
+      assert cc2q("foo(<<1, 2, 3>>") == s2q("foo(__cursor__())")
+      assert cc2q("foo(bar do :done end") == s2q("foo(__cursor__())")
     end
 
     test "incomplete expressions" do
-      # assert cc2q("foo(123, :") == s2q("foo(123, __cursor__())")
-      # assert cc2q("foo(123, %") == s2q("foo(123, __cursor__())")
-      # assert cc2q("foo(123, 0x") == s2q("foo(123, __cursor__())")
-      # assert cc2q("foo(123, ~") == s2q("foo(123, __cursor__())")
+      assert cc2q("foo(123, :") == s2q("foo(123, __cursor__())")
+      assert cc2q("foo(123, %") == s2q("foo(123, __cursor__())")
+      assert cc2q("foo(123, 0x") == s2q("foo(123, __cursor__())")
+      assert cc2q("foo(123, ~") == s2q("foo(123, __cursor__())")
       assert cc2q("foo(123, ~r") == s2q("foo(123, __cursor__())")
-      # assert cc2q("foo(123, ~r/") == s2q("foo(123, __cursor__())")
+      assert cc2q("foo(123, ~r/") == s2q("foo(123, __cursor__())")
     end
 
     test "no warnings" do