Add no-GIL interpreter support

.github/workflows/tests.yml

@@ -22,7 +22,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: [3.9, "3.10", "3.11", "3.12", "3.13", "3.14"]
+        python-version: [3.9, "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t", "3.15", "3.15t"]
         # Recall the macOS builds upload built wheels so all supported versions
         # need to run on mac.
         os: [ubuntu-latest, macos-latest]

docs/development.rst

@@ -11,6 +11,17 @@ Github
 The primary development location for greenlet is GitHub:
 https://github.com/python-greenlet/greenlet/
 
+Compilation database
+====================
+
+To get ``compile_commands.json`` (used by ``clang-tidy``, for example), 
+install ``bear``, then run:
+
+.. code-block:: shell
+
+   $ bear -- python setup.py
+
+
 Releases
 ========
 

src/greenlet/PyModule.cpp

@@ -144,7 +144,7 @@ PyDoc_STRVAR(mod_get_clocks_used_doing_optional_cleanup_doc,
 static PyObject*
 mod_get_clocks_used_doing_optional_cleanup(PyObject* UNUSED(module))
 {
-    std::clock_t& clocks = ThreadState::clocks_used_doing_gc();
+    std::clock_t clocks = ThreadState::clocks_used_doing_gc();
 
     if (clocks == std::clock_t(-1)) {
         Py_RETURN_NONE;
@@ -168,15 +168,15 @@ mod_enable_optional_cleanup(PyObject* UNUSED(module), PyObject* flag)
         return nullptr;
     }
 
-    std::clock_t& clocks = ThreadState::clocks_used_doing_gc();
     if (is_true) {
+        std::clock_t clocks = ThreadState::clocks_used_doing_gc();
         // If we already have a value, we don't want to lose it.
         if (clocks == std::clock_t(-1)) {
-            clocks = 0;
+            ThreadState::set_clocks_used_doing_gc(0);
         }
     }
     else {
-        clocks = std::clock_t(-1);
+        ThreadState::set_clocks_used_doing_gc(std::clock_t(-1));
     }
     Py_RETURN_NONE;
 }

src/greenlet/TGreenlet.cpp

@@ -611,7 +611,104 @@ bool Greenlet::is_currently_running_in_some_thread() const
     return this->stack_state.active() && !this->python_state.top_frame();
 }
 
-#if GREENLET_PY312
+#if GREENLET_PY315
+void GREENLET_NOINLINE(Greenlet::expose_frames)()
+{
+    if (!this->python_state.top_frame()) {
+        return;
+    }
+
+    _PyInterpreterFrame* last_complete_iframe = nullptr;
+    _PyInterpreterFrame* iframe = this->python_state.top_frame()->f_frame;
+    while (iframe) {
+        // We must make a copy before looking at the iframe contents,
+        // since iframe might point to a portion of the greenlet's C stack
+        // that was spilled when switching greenlets.
+        _PyInterpreterFrame iframe_copy;
+        this->stack_state.copy_from_stack(&iframe_copy, iframe, sizeof(*iframe));
+        if (!_PyFrame_IsIncomplete(&iframe_copy)) {
+            // If the iframe were OWNED_BY_INTERPRETER then it would always be
+            // incomplete. Since it's not incomplete, it's not on the C stack
+            // and we can access it through the original `iframe` pointer
+            // directly.  This is important since GetFrameObject might
+            // lazily _create_ the frame object and we don't want the
+            // interpreter to lose track of it.
+            assert(iframe_copy.owner != FRAME_OWNED_BY_INTERPRETER);
+
+            // We really want to just write:
+            //     PyFrameObject* frame = _PyFrame_GetFrameObject(iframe);
+            // but _PyFrame_GetFrameObject calls _PyFrame_MakeAndSetFrameObject
+            // which is not a visible symbol in libpython. The easiest
+            // way to get a public function to call it is using
+            // PyFrame_GetBack, which is defined as follows:
+            //     assert(frame != NULL);
+            //     assert(!_PyFrame_IsIncomplete(frame->f_frame));
+            //     PyFrameObject *back = frame->f_back;
+            //     if (back == NULL) {
+            //         _PyInterpreterFrame *prev = frame->f_frame->previous;
+            //         prev = _PyFrame_GetFirstComplete(prev);
+            //         if (prev) {
+            //             back = _PyFrame_GetFrameObject(prev);
+            //         }
+            //     }
+            //     return (PyFrameObject*)Py_XNewRef(back);
+            if (!iframe->frame_obj) {
+                PyFrameObject dummy_frame;
+                _PyInterpreterFrame dummy_iframe;
+                dummy_frame.f_back = nullptr;
+                dummy_frame.f_frame = &dummy_iframe;
+                // force the iframe to be considered complete without
+                // needing to check its code object:
+                dummy_iframe.owner = FRAME_OWNED_BY_GENERATOR;
+                dummy_iframe.previous = iframe;
+                assert(!_PyFrame_IsIncomplete(&dummy_iframe));
+                // Drop the returned reference immediately; the iframe
+                // continues to hold a strong reference
+                Py_XDECREF(PyFrame_GetBack(&dummy_frame));
+                assert(iframe->frame_obj);
+            }
+
+            // This is a complete frame, so make the last one of those we saw
+            // point at it, bypassing any incomplete frames (which may have
+            // been on the C stack) in between the two. We're overwriting
+            // last_complete_iframe->previous and need that to be reversible,
+            // so we store the original previous ptr in the frame object
+            // (which we must have created on a previous iteration through
+            // this loop). The frame object has a bunch of storage that is
+            // only used when its iframe is OWNED_BY_FRAME_OBJECT, which only
+            // occurs when the frame object outlives the frame's execution,
+            // which can't have happened yet because the frame is currently
+            // executing as far as the interpreter is concerned. So, we can
+            // reuse it for our own purposes.
+            assert(iframe->owner == FRAME_OWNED_BY_THREAD
+                   || iframe->owner == FRAME_OWNED_BY_GENERATOR);
+            if (last_complete_iframe) {
+                assert(last_complete_iframe->frame_obj);
+                memcpy(&last_complete_iframe->frame_obj->_f_frame_data[0],
+                       &last_complete_iframe->previous, sizeof(void *));
+                last_complete_iframe->previous = iframe;
+            }
+            last_complete_iframe = iframe;
+        }
+        // Frames that are OWNED_BY_FRAME_OBJECT are linked via the
+        // frame's f_back while all others are linked via the iframe's
+        // previous ptr. Since all the frames we traverse are running
+        // as far as the interpreter is concerned, we don't have to
+        // worry about the OWNED_BY_FRAME_OBJECT case.
+        iframe = iframe_copy.previous;
+    }
+
+    // Give the outermost complete iframe a null previous pointer to
+    // account for any potential incomplete/C-stack iframes between it
+    // and the actual top-of-stack
+    if (last_complete_iframe) {
+        assert(last_complete_iframe->frame_obj);
+        memcpy(&last_complete_iframe->frame_obj->_f_frame_data[0],
+               &last_complete_iframe->previous, sizeof(void *));
+        last_complete_iframe->previous = nullptr;
+    }
+}
+#elif GREENLET_PY312
 void GREENLET_NOINLINE(Greenlet::expose_frames)()
 {
     if (!this->python_state.top_frame()) {
@@ -633,7 +730,7 @@ void GREENLET_NOINLINE(Greenlet::expose_frames)()
             // directly.  This is important since GetFrameObject might
             // lazily _create_ the frame object and we don't want the
             // interpreter to lose track of it.
-            assert(iframe_copy.owner != FRAME_OWNED_BY_CSTACK);
+            static_assert(iframe_copy.owner != FRAME_OWNED_BY_CSTACK);
 
             // We really want to just write:
             //     PyFrameObject* frame = _PyFrame_GetFrameObject(iframe);

src/greenlet/TMainGreenlet.cpp

@@ -14,11 +14,18 @@
 
 #include "TGreenlet.hpp"
 
+#ifdef Py_GIL_DISABLED
+#include <atomic>
+#endif
 
-
-// Protected by the GIL. Incremented when we create a main greenlet,
-// in a new thread, decremented when it is destroyed.
+// Incremented when we create a main greenlet, in a new thread, decremented
+// when it is destroyed.
+#ifdef Py_GIL_DISABLED
+static std::atomic<Py_ssize_t> G_TOTAL_MAIN_GREENLETS(0);
+#else
+// Protected by the GIL.
 static Py_ssize_t G_TOTAL_MAIN_GREENLETS;
+#endif
 
 namespace greenlet {
 greenlet::PythonAllocator<MainGreenlet> MainGreenlet::allocator;

src/greenlet/TThreadState.hpp

@@ -3,6 +3,7 @@
 
 #include <ctime>
 #include <stdexcept>
+#include <atomic>
 
 #include "greenlet_internal.hpp"
 #include "greenlet_refs.hpp"
@@ -118,7 +119,11 @@ class ThreadState {
     void* exception_state;
 #endif
 
+#ifdef Py_GIL_DISABLED
+    static std::atomic<std::clock_t> _clocks_used_doing_gc;
+#else
     static std::clock_t _clocks_used_doing_gc;
+#endif
     static ImmortalString get_referrers_name;
     static PythonAllocator<ThreadState> allocator;
 
@@ -160,7 +165,7 @@ class ThreadState {
     static void init()
     {
         ThreadState::get_referrers_name = "get_referrers";
-        ThreadState::_clocks_used_doing_gc = 0;
+        ThreadState::set_clocks_used_doing_gc(0);
     }
 
     ThreadState()
@@ -349,9 +354,31 @@ class ThreadState {
     /**
      * Set to std::clock_t(-1) to disable.
      */
-    inline static std::clock_t& clocks_used_doing_gc()
+    inline static std::clock_t clocks_used_doing_gc()
     {
+#ifdef Py_GIL_DISABLED
+        return ThreadState::_clocks_used_doing_gc.load(std::memory_order_relaxed);
+#else
         return ThreadState::_clocks_used_doing_gc;
+#endif
+    }
+
+    inline static void set_clocks_used_doing_gc(std::clock_t value)
+    {
+#ifdef Py_GIL_DISABLED
+        ThreadState::_clocks_used_doing_gc.store(value, std::memory_order_relaxed);
+#else
+        ThreadState::_clocks_used_doing_gc = value;
+#endif
+    }
+
+    inline static void add_clocks_used_doing_gc(std::clock_t value)
+    {
+#ifdef Py_GIL_DISABLED
+        ThreadState::_clocks_used_doing_gc.fetch_add(value, std::memory_order_relaxed);
+#else
+        ThreadState::_clocks_used_doing_gc += value;
+#endif
     }
 
     ~ThreadState()
@@ -390,7 +417,7 @@ class ThreadState {
             PyGreenlet* old_main_greenlet = this->main_greenlet.borrow();
             Py_ssize_t cnt = this->main_greenlet.REFCNT();
             this->main_greenlet.CLEAR();
-            if (ThreadState::_clocks_used_doing_gc != std::clock_t(-1)
+            if (ThreadState::clocks_used_doing_gc() != std::clock_t(-1)
                 && cnt == 2 && Py_REFCNT(old_main_greenlet) == 1) {
                 // Highly likely that the reference is somewhere on
                 // the stack, not reachable by GC. Verify.
@@ -444,7 +471,7 @@ class ThreadState {
                         }
                     }
                     std::clock_t end = std::clock();
-                    ThreadState::_clocks_used_doing_gc += (end - begin);
+                    ThreadState::add_clocks_used_doing_gc(end - begin);
                 }
             }
         }
@@ -486,7 +513,11 @@ class ThreadState {
 
 ImmortalString ThreadState::get_referrers_name(nullptr);
 PythonAllocator<ThreadState> ThreadState::allocator;
+#ifdef Py_GIL_DISABLED
+std::atomic<std::clock_t> ThreadState::_clocks_used_doing_gc(0);
+#else
 std::clock_t ThreadState::_clocks_used_doing_gc(0);
+#endif
 
 
 

src/greenlet/greenlet.cpp

@@ -213,6 +213,10 @@ greenlet_internal_mod_init() noexcept
     try {
         CreatedModule m(greenlet_module_def);
 
+#ifdef Py_GIL_DISABLED
+        PyUnstable_Module_SetGIL(m.borrow(), Py_MOD_GIL_NOT_USED);
+#endif
+
         Require(PyType_Ready(&PyGreenlet_Type));
         Require(PyType_Ready(&PyGreenletUnswitchable_Type));
 
@@ -291,6 +295,9 @@ greenlet_internal_mod_init() noexcept
         //      << "\n\tPyGreenlet     : " << sizeof(PyGreenlet)
         //      << endl;
 
+#ifdef Py_GIL_DISABLED
+        PyUnstable_Module_SetGIL(m.borrow(), Py_MOD_GIL_NOT_USED);
+#endif
         return m.borrow(); // But really it's the main reference.
     }
     catch (const LockInitError& e) {

src/greenlet/greenlet_cpython_compat.hpp

@@ -67,6 +67,12 @@ Greenlet won't compile on anything older than Python 3.11 alpha 4 (see
 #    define GREENLET_PY314 0
 #endif
 
+#if PY_VERSION_HEX >= 0x30F0000
+#    define GREENLET_PY315 1
+#else
+#    define GREENLET_PY315 0
+#endif
+
 #ifndef Py_SET_REFCNT
 /* Py_REFCNT and Py_SIZE macros are converted to functions
 https://bugs.python.org/issue39573 */

src/greenlet/greenlet_slp_switch.hpp

@@ -36,7 +36,11 @@
 // running this code, the thread isn't exiting. This also nets us a
 // 10-12% speed improvement.
 
+#if Py_GIL_DISABLED
+thread_local greenlet::Greenlet* switching_thread_state = nullptr;
+#else
 static greenlet::Greenlet* volatile switching_thread_state = nullptr;
+#endif
 
 
 extern "C" {

src/greenlet/tests/__init__.py

@@ -29,6 +29,7 @@
 # XXX: First tested on 3.14a7. Revisit all uses of this on later versions to ensure they
 # are still valid.
 PY314 = sys.version_info[:2] >= (3, 14)
+PY315 = sys.version_info[:2] >= (3, 15)
 
 WIN = sys.platform.startswith("win")
 RUNNING_ON_GITHUB_ACTIONS = os.environ.get('GITHUB_ACTIONS')

src/greenlet/tests/_test_extension.c

@@ -251,5 +251,8 @@ PyInit__test_extension(void)
     }
 
     PyGreenlet_Import();
+#ifdef Py_GIL_DISABLED
+    PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED);
+#endif
     return module;
 }

src/greenlet/tests/_test_extension_cpp.cpp

@@ -221,6 +221,13 @@ PyInit__test_extension_cpp(void)
     p_test_exception_throw_nonstd = test_exception_throw_nonstd;
     p_test_exception_throw_std = test_exception_throw_std;
     p_test_exception_switch_recurse = test_exception_switch_recurse;
+#ifdef Py_GIL_DISABLED
+    PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED);
+#endif
+
+#ifdef Py_GIL_DISABLED
+    PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED);
+#endif
 
     return module;
 }

tox.ini

@@ -1,11 +1,11 @@
 [tox]
 envlist =
-    py{37,38,39,310,311,312,313,314},py{310,311,312,313,314}-ns,docs
+    py{37,38,39,310,311,312,313,314,315},py{310,311,312,313,314,315}-ns,docs,py314t,tsan-314#,tsan-314t
 
 [testenv]
 commands =
     python -c 'import greenlet._greenlet as G; assert G.GREENLET_USE_STANDARD_THREADING'
-    python -m unittest discover -v greenlet.tests
+    python -m unittest discover greenlet.tests
     sphinx-build -b doctest -d docs/_build/doctrees-{envname} docs docs/_build/doctest-{envname}
 sitepackages = False
 extras =
@@ -21,3 +21,13 @@ commands =
     sphinx-build -b html -d docs/_build/doctrees docs docs/_build/html
     sphinx-build -b doctest -d docs/_build/doctrees docs docs/_build/doctest
 extras = docs
+
+[testenv:tsan-314t]
+basepython = /usr/local/python-builds/tsan/bin/python3.14t
+passenv =
+    TSAN_OPTIONS
+
+[testenv:tsan-314]
+basepython = /usr/local/python-builds/default-tsan/bin/python3.14
+passenv =
+    TSAN_OPTIONS