Bytecode DSL: async generators

* implement async generators
* various fixes in async support
* avoid races when creating root node from code unit in Bytecode DSL MakeFunction operation

Author: steve-s

graalpython/com.oracle.graal.python.test/src/tests/test_interop.py

@@ -959,8 +959,6 @@ def l(*elements):
         assert [e for e in reversed(al)] == [3,2,1]
         assert [e for e in al] == [1,2,3]
 
-    # AssertionError: "descriptor requires a 'dict' object but received a 'ForeignDict'" does not match "object.__new__(ForeignDict) is not safe, use ForeignDict.__new__()"
-    @unittest.skipIf(os.environ.get('BYTECODE_DSL_INTERPRETER'), "TODO: bug in comment above")
     def test_java_map(self):
         from java.util import HashMap
         h = HashMap()

graalpython/com.oracle.graal.python.test/src/tests/test_traceback.py

@@ -104,7 +104,6 @@ def test():
     )
 
 
-@unittest.skipIf(os.environ.get('BYTECODE_DSL_INTERPRETER'), "TODO: correct generator tracebacks")
 def test_basic_traceback_generator():
     def foo():
         yield 1
@@ -141,7 +140,6 @@ def reraise():
     )
 
 
-@unittest.skipIf(os.environ.get('BYTECODE_DSL_INTERPRETER'), "TODO: bug in comment above")
 def test_reraise_direct_generator():
     def reraise():
         try:
@@ -401,7 +399,6 @@ def reraise_from_finally():
     )
 
 
-@unittest.skipIf(os.environ.get('BYTECODE_DSL_INTERPRETER'), "TODO: bug in comment above")
 def test_finally_generator():
     def test():
         try:
@@ -426,7 +423,6 @@ def test():
     )
 
 
-@unittest.skipIf(os.environ.get('BYTECODE_DSL_INTERPRETER'), "TODO: bug in comment above")
 def test_reraise_from_finally_generator():
     def reraise_from_finally():
         try:
@@ -491,7 +487,6 @@ def test():
     )
 
 
-@unittest.skipIf(os.environ.get('BYTECODE_DSL_INTERPRETER'), "TODO: correct generator tracebacks")
 def test_generator_throw_existing():
     try:
         raise OverflowError
@@ -514,7 +509,6 @@ def test():
     )
 
 
-@unittest.skipIf(os.environ.get('BYTECODE_DSL_INTERPRETER'), "TODO: correct generator tracebacks")
 def test_generator_throw_with_traceback():
     try:
         raise NameError

graalpython/com.oracle.graal.python.test/src/tests/test_yield_from.py

@@ -976,7 +976,6 @@ def g():
     #     for stack in spam(eggs(gen())):
     #         self.assertTrue('spam' in stack and 'eggs' in stack)
 
-    @unittest.skipIf(os.environ.get('BYTECODE_DSL_INTERPRETER'), "TODO: bug in comment above")
     def test_custom_iterator_return(self):
         # See issue #15568
         class MyIter:

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags_bytecode_dsl/test_asyncgen.txt

@@ -1,6 +1,50 @@
 test.test_asyncgen.AsyncGenAsyncioTest.test_aiter_bad_args @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_aiter_idempotent @ linux-x86_64
 test.test_asyncgen.AsyncGenAsyncioTest.test_anext_bad_args @ linux-x86_64
 test.test_asyncgen.AsyncGenAsyncioTest.test_anext_return_generator @ linux-x86_64
 test.test_asyncgen.AsyncGenAsyncioTest.test_anext_return_iterator @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_aclose_after_exhaustion @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_aclose_compatible_with_get_stack @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_aclose_twice_with_different_coros @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_aiter @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_aiter_class @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_01 @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_02 @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_03 @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_aclose_10 @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_aclose_11 @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_anext_04 @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_anext_05 @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_anext_06 @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_anext_stopiteration @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_anext_tuple @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_asend_01 @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_asend_02 @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_asend_03 @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_athrow_01 @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_athrow_02 @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_athrow_03 @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_athrow_stopiteration @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_athrow_tuple @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_shutdown_01 @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_shutdown_02 @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_shutdown_exception_01 @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_await_same_aclose_coro_twice @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_await_same_anext_coro_twice @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_expression_01 @ linux-x86_64
+test.test_asyncgen.AsyncGenAsyncioTest.test_asyncgen_nonstarted_hooks_are_cancellable @ linux-x86_64
 test.test_asyncgen.AsyncGenSyntaxTest.test_async_gen_syntax_01 @ linux-x86_64
 test.test_asyncgen.AsyncGenSyntaxTest.test_async_gen_syntax_02 @ linux-x86_64
+test.test_asyncgen.AsyncGenSyntaxTest.test_async_gen_syntax_03 @ linux-x86_64
+test.test_asyncgen.AsyncGenSyntaxTest.test_async_gen_syntax_04 @ linux-x86_64
+test.test_asyncgen.AsyncGenSyntaxTest.test_async_gen_syntax_05 @ linux-x86_64
+test.test_asyncgen.AsyncGenTest.test_async_gen_exception_03 @ linux-x86_64
+test.test_asyncgen.AsyncGenTest.test_async_gen_exception_04 @ linux-x86_64
+test.test_asyncgen.AsyncGenTest.test_async_gen_exception_05 @ linux-x86_64
+test.test_asyncgen.AsyncGenTest.test_async_gen_exception_07 @ linux-x86_64
+test.test_asyncgen.AsyncGenTest.test_async_gen_exception_08 @ linux-x86_64
+test.test_asyncgen.AsyncGenTest.test_async_gen_exception_09 @ linux-x86_64
+test.test_asyncgen.AsyncGenTest.test_async_gen_exception_11 @ linux-x86_64
+test.test_asyncgen.AsyncGenTest.test_async_gen_exception_12 @ linux-x86_64
+test.test_asyncgen.AsyncGenTest.test_async_gen_iteration_01 @ linux-x86_64
+test.test_asyncgen.AsyncGenTest.test_async_gen_iteration_02 @ linux-x86_64

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags_bytecode_dsl/test_asyncio.txt

@@ -354,6 +354,7 @@ test.test_asyncio.test_pep492.CoroutineTests.test_task_print_stack @ linux-x86_6
 test.test_asyncio.test_pep492.CoroutineTests.test_types_coroutine @ linux-x86_64
 test.test_asyncio.test_pep492.LockTests.test_context_manager_async_with @ linux-x86_64
 test.test_asyncio.test_pep492.LockTests.test_context_manager_with_await @ linux-x86_64
+test.test_asyncio.test_pep492.StreamReaderTests.test_readline @ linux-x86_64
 test.test_asyncio.test_proactor_events.BaseProactorEventLoopTests.test_close @ linux-x86_64
 test.test_asyncio.test_proactor_events.BaseProactorEventLoopTests.test_close_self_pipe @ linux-x86_64
 test.test_asyncio.test_proactor_events.BaseProactorEventLoopTests.test_create_server @ linux-x86_64
@@ -481,6 +482,7 @@ test.test_asyncio.test_queues.QueuePutTests.test_put_cancelled_race @ linux-x86_
 test.test_asyncio.test_queues.QueuePutTests.test_put_with_waiting_getters @ linux-x86_64
 test.test_asyncio.test_queues.QueuePutTests.test_why_are_putters_waiting @ linux-x86_64
 test.test_asyncio.test_runners.RunTests.test_asyncio_run_cancels_hanging_tasks @ linux-x86_64
+test.test_asyncio.test_runners.RunTests.test_asyncio_run_closes_gens_after_hanging_tasks_errors @ linux-x86_64
 test.test_asyncio.test_runners.RunTests.test_asyncio_run_debug @ linux-x86_64
 test.test_asyncio.test_runners.RunTests.test_asyncio_run_from_running_loop @ linux-x86_64
 test.test_asyncio.test_runners.RunTests.test_asyncio_run_loop_factory @ linux-x86_64
@@ -820,6 +822,7 @@ test.test_asyncio.test_subprocess.SubprocessThreadedWatcherTests.test_subprocess
 test.test_asyncio.test_subprocess.SubprocessThreadedWatcherTests.test_terminate @ linux-x86_64
 test.test_asyncio.test_subprocess.SubprocessTransportTests.test_proc_exited @ linux-x86_64
 test.test_asyncio.test_subprocess.SubprocessTransportTests.test_subprocess_repr @ linux-x86_64
+test.test_asyncio.test_taskgroups.TestTaskGroup.test_cancel_children_on_child_error @ linux-x86_64
 test.test_asyncio.test_taskgroups.TestTaskGroup.test_cancellation @ linux-x86_64
 test.test_asyncio.test_taskgroups.TestTaskGroup.test_exception_refcycles_base_error @ linux-x86_64
 test.test_asyncio.test_taskgroups.TestTaskGroup.test_exception_refcycles_direct @ linux-x86_64
@@ -829,7 +832,13 @@ test.test_asyncio.test_taskgroups.TestTaskGroup.test_taskgroup_02 @ linux-x86_64
 test.test_asyncio.test_taskgroups.TestTaskGroup.test_taskgroup_03 @ linux-x86_64
 test.test_asyncio.test_taskgroups.TestTaskGroup.test_taskgroup_04 @ linux-x86_64
 test.test_asyncio.test_taskgroups.TestTaskGroup.test_taskgroup_07 @ linux-x86_64
+test.test_asyncio.test_taskgroups.TestTaskGroup.test_taskgroup_08 @ linux-x86_64
+test.test_asyncio.test_taskgroups.TestTaskGroup.test_taskgroup_09 @ linux-x86_64
+test.test_asyncio.test_taskgroups.TestTaskGroup.test_taskgroup_10 @ linux-x86_64
+test.test_asyncio.test_taskgroups.TestTaskGroup.test_taskgroup_11 @ linux-x86_64
+test.test_asyncio.test_taskgroups.TestTaskGroup.test_taskgroup_12 @ linux-x86_64
 test.test_asyncio.test_taskgroups.TestTaskGroup.test_taskgroup_13 @ linux-x86_64
+test.test_asyncio.test_taskgroups.TestTaskGroup.test_taskgroup_14 @ linux-x86_64
 test.test_asyncio.test_taskgroups.TestTaskGroup.test_taskgroup_15 @ linux-x86_64
 test.test_asyncio.test_taskgroups.TestTaskGroup.test_taskgroup_16 @ linux-x86_64
 test.test_asyncio.test_taskgroups.TestTaskGroup.test_taskgroup_17 @ linux-x86_64
@@ -937,11 +946,13 @@ test.test_asyncio.test_tasks.PyTask_PyFuture_SubclassTests.test_create_task_with
 test.test_asyncio.test_tasks.PyTask_PyFuture_SubclassTests.test_create_task_with_noncoroutine @ linux-x86_64
 test.test_asyncio.test_tasks.PyTask_PyFuture_SubclassTests.test_current_task @ linux-x86_64
 test.test_asyncio.test_tasks.PyTask_PyFuture_SubclassTests.test_current_task_with_interleaving_tasks @ linux-x86_64
+test.test_asyncio.test_tasks.PyTask_PyFuture_SubclassTests.test_ensure_future_awaitable @ linux-x86_64
 test.test_asyncio.test_tasks.PyTask_PyFuture_SubclassTests.test_ensure_future_coroutine @ linux-x86_64
 test.test_asyncio.test_tasks.PyTask_PyFuture_SubclassTests.test_ensure_future_error_msg @ linux-x86_64
 test.test_asyncio.test_tasks.PyTask_PyFuture_SubclassTests.test_ensure_future_future @ linux-x86_64
 test.test_asyncio.test_tasks.PyTask_PyFuture_SubclassTests.test_ensure_future_neither @ linux-x86_64
 test.test_asyncio.test_tasks.PyTask_PyFuture_SubclassTests.test_ensure_future_task @ linux-x86_64
+test.test_asyncio.test_tasks.PyTask_PyFuture_SubclassTests.test_ensure_future_task_awaitable @ linux-x86_64
 test.test_asyncio.test_tasks.PyTask_PyFuture_SubclassTests.test_exception_chaining_after_await @ linux-x86_64
 test.test_asyncio.test_tasks.PyTask_PyFuture_SubclassTests.test_exception_chaining_after_await_with_context_cycle @ linux-x86_64
 test.test_asyncio.test_tasks.PyTask_PyFuture_SubclassTests.test_exception_traceback @ linux-x86_64
@@ -1042,11 +1053,13 @@ test.test_asyncio.test_tasks.PyTask_PyFuture_Tests.test_create_task_with_asyncli
 test.test_asyncio.test_tasks.PyTask_PyFuture_Tests.test_create_task_with_noncoroutine @ linux-x86_64
 test.test_asyncio.test_tasks.PyTask_PyFuture_Tests.test_current_task @ linux-x86_64
 test.test_asyncio.test_tasks.PyTask_PyFuture_Tests.test_current_task_with_interleaving_tasks @ linux-x86_64
+test.test_asyncio.test_tasks.PyTask_PyFuture_Tests.test_ensure_future_awaitable @ linux-x86_64
 test.test_asyncio.test_tasks.PyTask_PyFuture_Tests.test_ensure_future_coroutine @ linux-x86_64
 test.test_asyncio.test_tasks.PyTask_PyFuture_Tests.test_ensure_future_error_msg @ linux-x86_64
 test.test_asyncio.test_tasks.PyTask_PyFuture_Tests.test_ensure_future_future @ linux-x86_64
 test.test_asyncio.test_tasks.PyTask_PyFuture_Tests.test_ensure_future_neither @ linux-x86_64
 test.test_asyncio.test_tasks.PyTask_PyFuture_Tests.test_ensure_future_task @ linux-x86_64
+test.test_asyncio.test_tasks.PyTask_PyFuture_Tests.test_ensure_future_task_awaitable @ linux-x86_64
 test.test_asyncio.test_tasks.PyTask_PyFuture_Tests.test_exception_chaining_after_await @ linux-x86_64
 test.test_asyncio.test_tasks.PyTask_PyFuture_Tests.test_exception_chaining_after_await_with_context_cycle @ linux-x86_64
 test.test_asyncio.test_tasks.PyTask_PyFuture_Tests.test_exception_traceback @ linux-x86_64

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags_bytecode_dsl/test_collections.txt

@@ -83,6 +83,8 @@ test.test_collections.TestNamedTuple.test_pickle @ linux-x86_64
 test.test_collections.TestNamedTuple.test_readonly @ linux-x86_64
 test.test_collections.TestNamedTuple.test_repr @ linux-x86_64
 test.test_collections.TestNamedTuple.test_tupleness @ linux-x86_64
+test.test_collections.TestOneTrickPonyABCs.test_AsyncIterable @ linux-x86_64
+test.test_collections.TestOneTrickPonyABCs.test_AsyncIterator @ linux-x86_64
 test.test_collections.TestOneTrickPonyABCs.test_Callable @ linux-x86_64
 test.test_collections.TestOneTrickPonyABCs.test_Collection @ linux-x86_64
 test.test_collections.TestOneTrickPonyABCs.test_Container @ linux-x86_64

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags_bytecode_dsl/test_compile.txt

@@ -51,8 +51,6 @@ test.test_compile.TestSpecifics.test_stack_overflow @ linux-x86_64
 test.test_compile.TestSpecifics.test_subscripts @ linux-x86_64
 test.test_compile.TestSpecifics.test_syntax_error @ linux-x86_64
 test.test_compile.TestSpecifics.test_unary_minus @ linux-x86_64
-test.test_compile.TestStackSizeStability.test_async_for @ linux-x86_64
-test.test_compile.TestStackSizeStability.test_async_for_else @ linux-x86_64
 test.test_compile.TestStackSizeStability.test_if @ linux-x86_64
 test.test_compile.TestStackSizeStability.test_if_else @ linux-x86_64
 test.test_compile.TestStackSizeStability.test_while_else @ linux-x86_64

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags_bytecode_dsl/test_contextlib_async.txt

@@ -1,8 +1,27 @@
 test.test_contextlib_async.AclosingTestCase.test_aclosing @ linux-x86_64
+test.test_contextlib_async.AclosingTestCase.test_aclosing_bpo41229 @ linux-x86_64
 test.test_contextlib_async.AclosingTestCase.test_aclosing_error @ linux-x86_64
 test.test_contextlib_async.AclosingTestCase.test_instance_docs @ linux-x86_64
 test.test_contextlib_async.AsyncContextManagerTestCase.test_contextmanager_attribs @ linux-x86_64
 test.test_contextlib_async.AsyncContextManagerTestCase.test_contextmanager_doc_attrib @ linux-x86_64
+test.test_contextlib_async.AsyncContextManagerTestCase.test_contextmanager_except @ linux-x86_64
+test.test_contextlib_async.AsyncContextManagerTestCase.test_contextmanager_except_stopiter @ linux-x86_64
+test.test_contextlib_async.AsyncContextManagerTestCase.test_contextmanager_finally @ linux-x86_64
+test.test_contextlib_async.AsyncContextManagerTestCase.test_contextmanager_no_reraise @ linux-x86_64
+test.test_contextlib_async.AsyncContextManagerTestCase.test_contextmanager_non_normalised @ linux-x86_64
+test.test_contextlib_async.AsyncContextManagerTestCase.test_contextmanager_plain @ linux-x86_64
+test.test_contextlib_async.AsyncContextManagerTestCase.test_contextmanager_traceback @ linux-x86_64
+test.test_contextlib_async.AsyncContextManagerTestCase.test_contextmanager_trap_no_yield @ linux-x86_64
+test.test_contextlib_async.AsyncContextManagerTestCase.test_contextmanager_trap_second_yield @ linux-x86_64
+test.test_contextlib_async.AsyncContextManagerTestCase.test_contextmanager_trap_yield_after_throw @ linux-x86_64
+test.test_contextlib_async.AsyncContextManagerTestCase.test_contextmanager_wrap_runtimeerror @ linux-x86_64
+test.test_contextlib_async.AsyncContextManagerTestCase.test_decorating_method @ linux-x86_64
+test.test_contextlib_async.AsyncContextManagerTestCase.test_decorator @ linux-x86_64
+test.test_contextlib_async.AsyncContextManagerTestCase.test_decorator_with_exception @ linux-x86_64
+test.test_contextlib_async.AsyncContextManagerTestCase.test_instance_docstring_given_cm_docstring @ linux-x86_64
+test.test_contextlib_async.AsyncContextManagerTestCase.test_keywords @ linux-x86_64
+test.test_contextlib_async.AsyncContextManagerTestCase.test_recursive @ linux-x86_64
+test.test_contextlib_async.TestAbstractAsyncContextManager.test_async_gen_propagates_generator_exit @ linux-x86_64
 test.test_contextlib_async.TestAbstractAsyncContextManager.test_enter @ linux-x86_64
 test.test_contextlib_async.TestAbstractAsyncContextManager.test_exit_is_abstract @ linux-x86_64
 test.test_contextlib_async.TestAbstractAsyncContextManager.test_structural_subclassing @ linux-x86_64