fixes to locals(), exec() and eval() so that various scoping scenarios work correctly

Author: craigbarratt

custom_components/pyscript/eval.py

@@ -93,8 +93,22 @@ async def eval_func(arg_str, eval_globals=None, eval_locals=None):
                 eval_ast.sym_table = eval_globals
         else:
             eval_ast.sym_table_stack = ast_ctx.sym_table_stack.copy()
-            eval_ast.sym_table = ast_ctx.sym_table
-        eval_ast.curr_func = ast_ctx.curr_func
+            if ast_ctx.sym_table == ast_ctx.global_sym_table:
+                eval_ast.sym_table = ast_ctx.sym_table
+            else:
+                eval_ast.sym_table = ast_ctx.sym_table.copy()
+                eval_ast.sym_table.update(ast_ctx.user_locals)
+                to_delete = set()
+                for var, value in eval_ast.sym_table.items():
+                    if isinstance(value, EvalLocalVar):
+                        if value.is_defined():
+                            eval_ast.sym_table[var] = value.get()
+                        else:
+                            to_delete.add(var)
+                for var in to_delete:
+                    del eval_ast.sym_table[var]
+
+        eval_ast.curr_func = None
         try:
             eval_result = await eval_ast.aeval(eval_ast.ast)
         except Exception as err:
@@ -106,7 +120,17 @@ async def eval_func(arg_str, eval_globals=None, eval_locals=None):
                 + eval_ast.exception_long
             )
             raise
-        ast_ctx.curr_func = eval_ast.curr_func
+        #
+        # save variables only in the locals scope
+        #
+        if eval_globals is None and eval_ast.sym_table != ast_ctx.sym_table:
+            for var, value in eval_ast.sym_table.items():
+                if var in ast_ctx.global_sym_table and value == ast_ctx.global_sym_table[var]:
+                    continue
+                if var not in ast_ctx.sym_table and (
+                    ast_ctx.curr_func is None or var not in ast_ctx.curr_func.local_names
+                ):
+                    ast_ctx.user_locals[var] = value
         return eval_result
 
     return eval_func
@@ -135,7 +159,20 @@ def ast_locals_factory(ast_ctx):
     """Generate a locals() function with given ast_ctx."""
 
     async def locals_func():
-        return ast_ctx.sym_table
+        if ast_ctx.sym_table == ast_ctx.global_sym_table:
+            return ast_ctx.sym_table
+        local_sym_table = ast_ctx.sym_table.copy()
+        local_sym_table.update(ast_ctx.user_locals)
+        to_delete = set()
+        for var, value in local_sym_table.items():
+            if isinstance(value, EvalLocalVar):
+                if value.is_defined():
+                    local_sym_table[var] = value.get()
+                else:
+                    to_delete.add(var)
+        for var in to_delete:
+            del local_sym_table[var]
+        return local_sym_table
 
     return locals_func
 
@@ -698,6 +735,8 @@ async def call(self, ast_ctx, *args, **kwargs):
         self.exception_obj = None
         self.exception_long = None
         prev_func = ast_ctx.curr_func
+        save_user_locals = ast_ctx.user_locals
+        ast_ctx.user_locals = {}
         ast_ctx.curr_func = self
         del args, kwargs
         for arg1 in self.func_def.body:
@@ -710,6 +749,7 @@ async def call(self, ast_ctx, *args, **kwargs):
             if ast_ctx.get_exception_obj():
                 break
         ast_ctx.curr_func = prev_func
+        ast_ctx.user_locals = save_user_locals
         ast_ctx.code_str, ast_ctx.code_list = code_str, code_list
         if prev_sym_table is not None:
             (
@@ -798,6 +838,7 @@ def __init__(self, name, global_ctx, logger_name=None):
         self.sym_table_stack = []
         self.sym_table = self.global_sym_table
         self.local_sym_table = {}
+        self.user_locals = {}
         self.curr_func = None
         self.filename = name
         self.code_str = None

docs/new_features.rst

@@ -57,6 +57,7 @@ Bug fixes since 1.2.1 include:
 - Timeouts that implement time triggers might infrequenctly occur a tiny time before the target time. A fix was added
   to do an additional short timeout when there is an early timeout, to make sure any time trigger occurs at or shortly
   after the target time (and never before).
+- Fixes to ``locals()``, ``exec()`` and ``eval()`` so that various scoping scenarios work correctly.
 - When exception text is created, ensure lineno is inside code_list[]; with lambda function or eval it might not be.
 - An exception is raised when a function is called with unexpected keyword parameters that don't have corresponding
   keyword arguments (however, the trigger parameter names are excluded from this check, since trigger functions

tests/test_unit_eval.py

@@ -198,6 +198,98 @@
     ["eval('1+2')", 3],
     ["x = 5; eval('2 * x')", 10],
     ["x = 5; exec('x = 2 * x'); x", 10],
+    [
+        """
+def func():
+    x = 5
+    exec('x = 2 * x')
+    return x
+func()
+""",
+        5,
+    ],
+    ["x = 5; locals()['x'] = 10; x", 10],
+    ["x = 5; globals()['x'] = 10; x", 10],
+    [
+        """
+def func():
+    x = 5
+    locals()['x'] = 10
+    return x
+func()
+""",
+        5,
+    ],
+    [
+        """
+bar = 100
+bar2 = 50
+bar3 = [0]
+def func(bar=6):
+    def foo(bar=6):
+        bar += 2
+        bar5 = 100
+        exec("bar2 += 1; bar += 10; bar3[0] = 1234 + bar + bar2; bar4 = 123; bar5 += 10")
+        bar += 2
+        del bar5
+        return [bar, bar2, bar3, eval('bar2'), eval('bar4'), locals()]
+    return foo(bar)
+[func(), func(5), bar, bar2]
+    """,
+        [
+            [10, 50, [1302], 51, 123, {"bar": 10, "bar2": 51, "bar4": 123}],
+            [9, 50, [1302], 51, 123, {"bar": 9, "bar2": 51, "bar4": 123}],
+            100,
+            50,
+        ],
+    ],
+    [
+        """
+bar = 100
+bar2 = 50
+bar3 = [0]
+def func(bar=6):
+    bar2 = 10
+    def foo(bar=6):
+        nonlocal bar2
+        bar += 2
+        exec("bar2 += 1; bar += 10; bar3[0] = 1234 + bar2")
+        return [bar, bar2, bar3, eval('bar2'), locals()]
+    return foo(bar)
+[func(), func(5), bar, bar2]
+""",
+        [[8, 10, [1245], 10, {"bar": 8, "bar2": 10}], [7, 10, [1245], 10, {"bar": 7, "bar2": 10}], 100, 50],
+    ],
+    [
+        """
+x = 10
+def foo():
+    x = 5
+    del x
+    return eval("x")
+foo()
+""",
+        10,
+    ],
+    [
+        """
+bar = 100
+bar2 = 50
+bar3 = [0]
+def func(bar=6):
+    bar2 = 10
+    def foo(bar=6):
+        nonlocal bar2
+        bar += 2
+        del bar
+        exec("bar2 += 1; bar += 10; bar3[0] = 1234 + bar2 + bar")
+        return [bar3, eval('bar'), eval('bar2'), locals()]
+    del bar2
+    return foo(bar)
+[func(), func(5), bar, bar2]
+""",
+        [[[1395], 100, 50, {}], [[1395], 100, 50, {}], 100, 50],
+    ],
     ["eval('xyz', {'xyz': 10})", 10],
     ["g = {'xyz': 10}; eval('xyz', g, {})", 10],
     ["g = {'xyz': 10}; eval('xyz', {}, g)", 10],
@@ -352,20 +444,19 @@ def foo(bar=6):
 """,
         [8, 7, 100],
     ],
-    # eval()/exec() scoping is broken; remove test until fixed
-    #    [
-    #        """
-    # bar = 100
-    # def foo(bar=6):
-    #    bar += 2
-    #    del bar
-    #    return eval('bar')
-    #    bar += 5
-    #    return 1000
-    # [foo(), foo(5), bar]
-    # """,
-    #        [100, 100, 100],
-    #    ],
+    [
+        """
+bar = 100
+def foo(bar=6):
+   bar += 2
+   del bar
+   return eval('bar')
+   bar += 5
+   return 1000
+[foo(), foo(5), bar]
+""",
+        [100, 100, 100],
+    ],
     [
         """
 bar = 100
@@ -1371,7 +1462,7 @@ def func():
     eval('1 + y')
 func()
 """,
-        "Exception in test line 4 column 9: Exception in func(), eval() line 1 column 4: name 'y' is not defined",
+        "Exception in test line 4 column 9: Exception in eval() line 1 column 4: name 'y' is not defined",
     ],
     [
         """