Update to greenlet 2.0rc1

Fixes #1909

We'll need to release a final greenlet 2.0 and remove the rc qualifiers here before releasing gevent, because pre-release qualifiers are contagious.

deps/greenlet/greenlet.h

@@ -5,6 +5,7 @@
 #ifndef Py_GREENLETOBJECT_H
 #define Py_GREENLETOBJECT_H
 
+
 #include <Python.h>
 
 #ifdef __cplusplus
@@ -14,60 +15,24 @@ extern "C" {
 /* This is deprecated and undocumented. It does not change. */
 #define GREENLET_VERSION "1.0.0"
 
-#if PY_VERSION_HEX >= 0x30B00A6
-#  define GREENLET_PY311 1
-   /* _PyInterpreterFrame moved to the internal C API in Python 3.11 */
-#  include <internal/pycore_frame.h>
-#else
-#  define GREENLET_PY311 0
-#  define _PyCFrame CFrame
+#ifndef GREENLET_MODULE
+#define implementation_ptr_t void*
 #endif
 
 typedef struct _greenlet {
     PyObject_HEAD
-    char* stack_start;
-    char* stack_stop;
-    char* stack_copy;
-    intptr_t stack_saved;
-    struct _greenlet* stack_prev;
-    struct _greenlet* parent;
-    PyObject* run_info;
-    struct _frame* top_frame;
-    int recursion_depth;
-#if GREENLET_PY311
-    _PyInterpreterFrame *current_frame;
-    _PyStackChunk *datastack_chunk;
-    PyObject **datastack_top;
-    PyObject **datastack_limit;
-#endif
     PyObject* weakreflist;
-#if PY_VERSION_HEX >= 0x030700A3
-    _PyErr_StackItem* exc_info;
-    _PyErr_StackItem exc_state;
-#else
-    PyObject* exc_type;
-    PyObject* exc_value;
-    PyObject* exc_traceback;
-#endif
     PyObject* dict;
-#if PY_VERSION_HEX >= 0x030700A3
-    PyObject* context;
-#endif
-#if PY_VERSION_HEX >= 0x30A00B1
-    _PyCFrame* cframe;
-#endif
+    implementation_ptr_t pimpl;
 } PyGreenlet;
 
-#define PyGreenlet_Check(op) PyObject_TypeCheck(op, &PyGreenlet_Type)
-#define PyGreenlet_MAIN(op) (((PyGreenlet*)(op))->stack_stop == (char*)-1)
-#define PyGreenlet_STARTED(op) (((PyGreenlet*)(op))->stack_stop != NULL)
-#define PyGreenlet_ACTIVE(op) (((PyGreenlet*)(op))->stack_start != NULL)
-#define PyGreenlet_GET_PARENT(op) (((PyGreenlet*)(op))->parent)
+#define PyGreenlet_Check(op) (op && PyObject_TypeCheck(op, &PyGreenlet_Type))
+
 
 /* C API functions */
 
 /* Total number of symbols that are exported */
-#define PyGreenlet_API_pointers 8
+#define PyGreenlet_API_pointers 12
 
 #define PyGreenlet_Type_NUM 0
 #define PyExc_GreenletError_NUM 1
@@ -79,6 +44,11 @@ typedef struct _greenlet {
 #define PyGreenlet_Switch_NUM 6
 #define PyGreenlet_SetParent_NUM 7
 
+#define PyGreenlet_MAIN_NUM 8
+#define PyGreenlet_STARTED_NUM 9
+#define PyGreenlet_ACTIVE_NUM 10
+#define PyGreenlet_GET_PARENT_NUM 11
+
 #ifndef GREENLET_MODULE
 /* This section is used by modules that uses the greenlet C API */
 static void** _PyGreenlet_API = NULL;
@@ -144,6 +114,39 @@ static void** _PyGreenlet_API = NULL;
         (*(int (*)(PyGreenlet * greenlet, PyGreenlet * nparent)) \
              _PyGreenlet_API[PyGreenlet_SetParent_NUM])
 
+/*
+ * PyGreenlet_GetParent(PyObject* greenlet)
+ *
+ * return greenlet.parent;
+ *
+ * This could return NULL even if there is no exception active.
+ * If it does not return NULL, you are responsible for decrementing the
+ * reference count.
+ */
+#     define PyGreenlet_GetParent                                    \
+    (*(PyGreenlet* (*)(PyGreenlet*))                                 \
+     _PyGreenlet_API[PyGreenlet_GET_PARENT_NUM])
+
+/*
+ * deprecated, undocumented alias.
+ */
+#     define PyGreenlet_GET_PARENT PyGreenlet_GetParent
+
+#     define PyGreenlet_MAIN                                         \
+    (*(int (*)(PyGreenlet*))                                         \
+     _PyGreenlet_API[PyGreenlet_MAIN_NUM])
+
+#     define PyGreenlet_STARTED                                      \
+    (*(int (*)(PyGreenlet*))                                         \
+     _PyGreenlet_API[PyGreenlet_STARTED_NUM])
+
+#     define PyGreenlet_ACTIVE                                       \
+    (*(int (*)(PyGreenlet*))                                         \
+     _PyGreenlet_API[PyGreenlet_ACTIVE_NUM])
+
+
+
+
 /* Macro that imports greenlet and initializes C API */
 /* NOTE: This has actually moved to ``greenlet._greenlet._C_API``, but we
    keep the older definition to be sure older code that might have a copy of

docs/changes/1909.bugfix

+Update to greenlet 2.0. This fixes a deallocation issue that required
+a change in greenlet's ABI. The design of greenlet 2.0 is intended to
+prevent future fixes and enhancements from requiring an ABI change,
+making it easier to update gevent and greenlet independently.
+
+.. caution::
+
+   greenlet 2.0 requires a modern-ish C++ compiler. This may mean
+   certain older platforms are no longer supported.

pyproject.toml

@@ -23,8 +23,10 @@ requires = [
      # Python 3.7 requires at least 0.4.14, which is ABI incompatible with earlier
      # releases. Python 3.9 and 3.10 require 0.4.16;
      # 0.4.17 is ABI incompatible with earlier releases, but compatible with 1.0
-     # 1.1.3 is needed for CPython 3.11
-     "greenlet >= 1.1.3, < 2.0 ; platform_python_implementation == 'CPython'",
+     # 1.1.3 is needed for CPython 3.11.
+     # 2.0 is not ABI compatible with earlier releases, but with luck it won't
+     # have to break the ABI again.
+     "greenlet >= 2.0.0rc1 ; platform_python_implementation == 'CPython'",
 ]
 
 [tool.towncrier]

setup.py

@@ -213,7 +213,9 @@ greenlet_requires = [
     # so we can add an upper bound).
     # 1.1.0 is required for 3.10; it has a new ABI, but only on 1.1.0.
     # 1.1.3 is needed for 3.11, and supports everything 1.1.0 did.
-    'greenlet >= 1.1.3, < 2.0; platform_python_implementation=="CPython"',
+    # 2.0.0 supports everything 1.1.3 did, but breaks the ABI in a way that hopefully
+    # won't break again.
+    'greenlet >= 2.0.0rc1 ; platform_python_implementation=="CPython"',
 ]
 
 # Note that we don't add cffi to install_requires, it's

src/gevent/_gevent_cgreenlet.pxd

 # cython: auto_pickle=False
 
 cimport cython
+from cpython.ref cimport Py_DECREF
+
 from gevent._gevent_c_ident cimport IdentRegistry
 from gevent._gevent_c_hub_local cimport get_hub_noargs as get_hub
 from gevent._gevent_c_waiter cimport Waiter
@@ -21,12 +23,14 @@ cdef extern from "greenlet/greenlet.h":
         # properly handle the case that it can be NULL. So instead we inline a getparent
         # function that does the same thing as the green_getparent accessor but without
         # going through the overhead of generic attribute lookup.
-        cdef void* parent
+        #cdef void* parent
+        pass
 
     # These are actually macros and so must be included
     # (defined) in each .pxd, as are the two functions
     # that call them.
     greenlet PyGreenlet_GetCurrent()
+    void* PyGreenlet_GetParent(greenlet)
     void PyGreenlet_Import()
 
 @cython.final
@@ -36,13 +40,26 @@ cdef inline greenlet getcurrent():
 cdef inline object get_generic_parent(greenlet s):
     # We don't use any typed functions on the return of this,
     # so save the type check by making it just an object.
-    if s.parent != NULL:
-        return <object>s.parent
+    cdef object result
+    cdef void* parent = PyGreenlet_GetParent(s)
+    if parent != NULL:
+        # The cast will perform an incref; but the GetParent
+        # function already did an incref if we got it (and not NULL).
+        # Therefore, we must DECREF immediately.
+        result = <object>parent
+        Py_DECREF(result)
+        return result
 
 cdef inline SwitchOutGreenletWithLoop get_my_hub(greenlet s):
+    # This one we do want type checked on the return value.
     # Must not be called with s = None
-    if s.parent != NULL:
-        return <object>s.parent
+    cdef object result
+    cdef void* parent = PyGreenlet_GetParent(s)
+    if parent != NULL:
+        result = <object>parent
+        # See above
+        Py_DECREF(result)
+        return result
 
 cdef bint _greenlet_imported
 

src/gevent/tests/test__memleak.py

@@ -26,6 +26,16 @@ class TestQueue(TestCase): # pragma: no cover
             refcounts.append(sys.gettotalrefcount())
 
         # Refcounts may go down, but not up
+        # XXX: JAM: I think this may just be broken. Each time we add
+        # a new integer to our list of refcounts, we'll be
+        # creating a new reference. This makes sense when we see the list
+        # go up by one each iteration:
+        #
+        #   AssertionError: 530631 not less than or equal to 530630
+        #     : total refcount mismatch:
+        #      [530381, 530618, 530619, 530620, 530621,
+        #       530622, 530623, 530624, 530625, 530626,
+        #       530627, 530628, 530629, 530630, 530631]
         final = refcounts[-1]
         previous = refcounts[-2]
         self.assertLessEqual(