Use Py_SET_SIZE() and Py_SET_REFCNT() on Python 3.9.0a4 and newer (GH-3639)

* Add __Pyx_SET_SIZE() function: use Py_SET_SIZE() on Python 3.9.0a4
and newer, or use Py_SIZE() as an l-value on older Python versions.

Py_SIZE() must not be used as an l-value anymore in Python 3.9:
Py_SET_SIZE() must be used instead:

* https://bugs.python.org/issue39573
* https://docs.python.org/dev/c-api/structures.html#c.Py_SET_SIZE

* Add __Pyx_SET_REFCNT() function: use Py_SET_REFCNT() on Python 3.9.0a4
and newer, or use Py_REFCNT() as an l-value on older Python versions.

Py_REFCNT() must not be used as an l-value anymore in Python 3.9:
Py_SET_REFCNT() must be used instead:

* https://bugs.python.org/issue39573
* https://docs.python.org/dev/c-api/structures.html#c.Py_SET_REFCNT

Use it in ModuleNode.generate_usr_dealloc_call():

* Replace ++Py_REFCNT(o) with __Pyx_SET_REFCNT(o, Py_REFCNT(o) + 1)
* Replace --Py_REFCNT(o) with __Pyx_SET_REFCNT(o, Py_REFCNT(o) - 1)

Cython/Compiler/ModuleNode.py

@@ -1557,9 +1557,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("{")
         code.putln("PyObject *etype, *eval, *etb;")
         code.putln("PyErr_Fetch(&etype, &eval, &etb);")
-        code.putln("++Py_REFCNT(o);")
+        # increase the refcount while we are calling into user code
+        # to prevent recursive deallocation
+        code.putln("__Pyx_SET_REFCNT(o, Py_REFCNT(o) + 1);")
         code.putln("%s(o);" % entry.func_cname)
-        code.putln("--Py_REFCNT(o);")
+        code.putln("__Pyx_SET_REFCNT(o, Py_REFCNT(o) - 1);")
         code.putln("PyErr_Restore(etype, eval, etb);")
         code.putln("}")
 

Cython/Utility/Builtins.c

@@ -279,7 +279,8 @@ static PyObject *__Pyx_PyLong_AbsNeg(PyObject *n) {
     {
         PyObject *copy = _PyLong_Copy((PyLongObject*)n);
         if (likely(copy)) {
-            Py_SIZE(copy) = -(Py_SIZE(copy));
+            // negate the size to swap the sign
+            __Pyx_SET_SIZE(copy, -Py_SIZE(copy));
         }
         return copy;
     }

Cython/Utility/ModuleSetupCode.c

@@ -609,6 +609,15 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) {
   #define PySet_CheckExact(obj)        (Py_TYPE(obj) == &PySet_Type)
 #endif
 
+
+#if PY_VERSION_HEX >= 0x030900A4
+  #define __Pyx_SET_REFCNT(obj, refcnt) Py_SET_REFCNT(obj, refcnt)
+  #define __Pyx_SET_SIZE(obj, size) Py_SET_SIZE(obj, size)
+#else
+  #define __Pyx_SET_REFCNT(obj, refcnt) Py_REFCNT(obj) = (refcnt)
+  #define __Pyx_SET_SIZE(obj, size) Py_SIZE(obj) = (size)
+#endif
+
 #if CYTHON_ASSUME_SAFE_MACROS
   #define __Pyx_PySequence_SIZE(seq)  Py_SIZE(seq)
 #else

Cython/Utility/Optimize.c

@@ -35,7 +35,7 @@ static CYTHON_INLINE int __Pyx_PyList_Append(PyObject* list, PyObject* x) {
     if (likely(L->allocated > len) & likely(len > (L->allocated >> 1))) {
         Py_INCREF(x);
         PyList_SET_ITEM(list, len, x);
-        Py_SIZE(list) = len+1;
+        __Pyx_SET_SIZE(list, len + 1);
         return 0;
     }
     return PyList_Append(list, x);
@@ -53,7 +53,7 @@ static CYTHON_INLINE int __Pyx_ListComp_Append(PyObject* list, PyObject* x) {
     if (likely(L->allocated > len)) {
         Py_INCREF(x);
         PyList_SET_ITEM(list, len, x);
-        Py_SIZE(list) = len+1;
+        __Pyx_SET_SIZE(list, len + 1);
         return 0;
     }
     return PyList_Append(list, x);
@@ -104,7 +104,7 @@ static CYTHON_INLINE PyObject* __Pyx__PyObject_Pop(PyObject* L) {
 static CYTHON_INLINE PyObject* __Pyx_PyList_Pop(PyObject* L) {
     /* Check that both the size is positive and no reallocation shrinking needs to be done. */
     if (likely(PyList_GET_SIZE(L) > (((PyListObject*)L)->allocated >> 1))) {
-        Py_SIZE(L) -= 1;
+        __Pyx_SET_SIZE(L, Py_SIZE(L) - 1);
         return PyList_GET_ITEM(L, PyList_GET_SIZE(L));
     }
     return CALL_UNBOUND_METHOD(PyList_Type, "pop", L);
@@ -167,7 +167,7 @@ static PyObject* __Pyx__PyList_PopIndex(PyObject* L, PyObject* py_ix, Py_ssize_t
         }
         if (likely(__Pyx_is_valid_index(cix, size))) {
             PyObject* v = PyList_GET_ITEM(L, cix);
-            Py_SIZE(L) -= 1;
+            __Pyx_SET_SIZE(L, Py_SIZE(L) - 1);
             size -= 1;
             memmove(&PyList_GET_ITEM(L, cix), &PyList_GET_ITEM(L, cix+1), (size_t)(size-cix)*sizeof(PyObject*));
             return v;

tests/run/verbatiminclude.pyx

@@ -25,9 +25,9 @@ cdef class C:
 
 cdef extern from "Python.h":
     """
-    #define Py_SET_SIZE(obj, size)  Py_SIZE((obj)) = (size)
+    #define my_SET_SIZE(obj, size)  __Pyx_SET_SIZE(obj, size)
     """
-    void Py_SET_SIZE(object, Py_ssize_t)
+    void my_SET_SIZE(object, Py_ssize_t)
 
 
 def test_square(x):
@@ -59,4 +59,4 @@ def test_class():
 def test_set_size(x, size):
     # This function manipulates Python objects in a bad way, so we
     # do not call it. The real test is that it compiles.
-    Py_SET_SIZE(x, size)
+    my_SET_SIZE(x, size)