Change cypclass reference counting convention

This changes the reference counting convention for passing cyobjects
as arguments in a function call.

Before this commit, cyobjects used the same convention as pyobjects:
- The function borrows a reference on the argument from the caller
  and the caller keeps ownership of the object passed as argument,
  and must eventually decref it, even if it is a temporary rvalue
  that will not be reachable in the caller's scope after the call.
- If the function needs to take ownership of the argument, e.g. to
  store it, it must increment its reference count first, at which
  point the caller and callee both own a reference to the object.
- If the callee does not take ownership of the argument, it should
  not decrement its reference count at any point.

After this commit, the convention for cyobjects is as follows:
- The function steals the reference on the argument from the caller,
  and the caller should not decrement its reference count after the
  call.
- If the object will still be reachable in the caller's scope after
  the call the caller must increment its reference count __before__
  the call to retain ownership of its reference after the call.
- The function has ownership of the reference received as argument,
  and must decref it when if goes out of scope.

The main reason for this change is to make it possible to 'consume'
an argument from within a function. Before this change there was no
reliable way to determine from within the function whether the caller
would retain a reference on the argument after the call, which is
required when the 'consume' operation needs a runtime check.

The 'self' argument in a cypclass method is currently still an
exception to this change: it still follows the previous convention.

Cython/Compiler/Code.py

@@ -849,7 +849,7 @@ class FunctionState(object):
         elif type.is_cfunction:
             from . import PyrexTypes
             type = PyrexTypes.c_ptr_type(type)  # A function itself isn't an l-value
-        if not type.is_pyobject and not type.is_memoryviewslice and not type.is_cyp_class:
+        if not type.is_pyobject and not type.is_memoryviewslice:
             # Make manage_ref canonical, so that manage_ref will always mean
             # a decref is needed.
             manage_ref = False

Cython/Compiler/ModuleNode.py

@@ -1061,9 +1061,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             code.putln("this->%s->push(message);" % queue_attr_cname)
             code.putln("Cy_UNWLOCK(%s);" % queue_attr_cname)
             code.putln("} else {")
+            code.putln("Cy_DECREF(message);")
             code.putln('fprintf(stderr, "Acthon error: No queue to push to for %s remote call !\\n");' % reified_function_entry.name)
             code.putln("}")
-            code.putln("Cy_DECREF(message);")
             # Return result object
             code.putln("return result_object;")
             code.putln("}")

Cython/Compiler/Nodes.py

@@ -2199,10 +2199,12 @@ class FuncDefNode(StatNode, BlockNode):
         is_cdef = isinstance(self, CFuncDefNode)
         for entry in lenv.arg_entries:
             if not entry.type.is_memoryviewslice:
-                if (acquire_gil or entry.cf_is_reassigned) and not entry.in_closure:
-                    if entry.type.is_cyp_class and entry.is_self_arg:
-                        pass
-                    else:
+                if entry.type.is_cyp_class:
+                    # CyObjects use another refcounting convention.
+                    # Except for the 'self' argument.
+                    if entry.is_self_arg and (entry.is_consumed or entry.cf_is_reassigned) and not entry.in_closure:
+                        code.put_var_incref(entry)
+                elif (acquire_gil or entry.cf_is_reassigned) and not entry.in_closure:
                     code.put_var_incref(entry)
             # Note: defaults are always incref-ed. For def functions, we
             #       we acquire arguments from object conversion, so we have
@@ -2418,6 +2420,11 @@ class FuncDefNode(StatNode, BlockNode):
                 # functions, but not borrowed slices from cdef functions.
                 if is_cdef and not entry.cf_is_reassigned:
                     continue
+            elif entry.type.is_cyp_class:
+                # CyObject arguments are systematically decrefed.
+                # Except for the 'self' argument when it has not been reassigned or consumed.
+                if entry.is_self_arg and not (entry.is_consumed or entry.cf_is_reassigned):
+                    continue
             else:
                 if entry.in_closure:
                     continue
@@ -2425,9 +2432,6 @@ class FuncDefNode(StatNode, BlockNode):
                     continue
                 if entry.type.needs_refcounting:
                     assure_gil('success')
-            if entry.type.is_cyp_class and entry.is_self_arg:
-                pass
-            else:
             # FIXME use entry.xdecref_cleanup - del arg seems to be the problem
             code.put_var_xdecref(entry, have_gil=gil_owned['success'])
         if self.needs_closure:
@@ -3782,19 +3786,11 @@ class DefNodeWrapper(FuncDefNode):
         # ----- Non-error return cleanup
         code.put_label(code.return_label)
         for entry in lenv.var_entries:
-            if entry.is_arg:
-                # The conversion from PyObject to CyObject always creates a new CyObject reference.
-                # This decrements all arguments-as-variables converted straight from an actual argument.
-                # This includes CyObjects converted directly from a corresponding PyObject argument.
-                if entry.xdecref_cleanup or entry.type.is_cyp_class:
+            if entry.is_arg and not entry.type.is_cyp_class:
+                if entry.xdecref_cleanup:
                     code.put_var_xdecref(entry)
                 else:
                     code.put_var_decref(entry)
-        for entry in lenv.arg_entries:
-            if entry.type.is_cyp_class:
-                # The conversion from PyObject to CyObject always creates a new CyObject reference.
-                # This decrements CyObjects converted from generic PyObject args passed via tuple and kw dict.
-                code.put_var_xdecref(entry)
 
         code.put_finish_refcount_context()
         if not self.return_type.is_void:
@@ -6580,7 +6576,7 @@ class DelStatNode(StatNode):
                 arg.free_temps(code)
             elif arg.type.is_cyp_class:
                 arg.generate_evaluation_code(code)
-                code.put_decref_clear(arg.result(), arg.type)
+                code.put_xdecref_clear(arg.result(), arg.type)
                 arg.generate_disposal_code(code)
                 arg.free_temps(code)
             # else error reported earlier

Cython/Compiler/PyrexTypes.py

@@ -3796,12 +3796,10 @@ class CFuncTypeArg(BaseType):
     accept_none = True
     accept_builtin_subtypes = False
     annotation = None
-    original_template = None
-    has_cypclass_specialisation = False
 
     subtypes = ['type']
 
-    def __init__(self, name, type, pos, cname=None, annotation=None, original_template=None):
+    def __init__(self, name, type, pos, cname=None, annotation=None):
         self.name = name
         if cname is not None:
             self.cname = cname
@@ -3812,8 +3810,6 @@ class CFuncTypeArg(BaseType):
         self.type = type
         self.pos = pos
         self.needs_type_test = False  # TODO: should these defaults be set in analyse_types()?
-        self.original_template = original_template or self
-        self.original_template.has_cypclass_specialisation = type.is_cyp_class
 
     def __repr__(self):
         return "%s:%s" % (self.name, repr(self.type))
@@ -3821,14 +3817,10 @@ class CFuncTypeArg(BaseType):
     def declaration_code(self, for_display = 0, pyrex = 0, cname = None):
         if cname is None:
             cname = self.cname
-        if self.type.is_template_typename and self.has_cypclass_specialisation:
-            base_code = "Cy_Raw<%s>" % self.type.empty_declaration_code()
-            return self.base_declaration_code(base_code, self.cname)
         return self.type.declaration_code(cname, for_display, pyrex)
 
     def specialize(self, values):
-        return CFuncTypeArg(self.name, self.type.specialize(values), self.pos, self.cname,
-            None, self.original_template)
+        return CFuncTypeArg(self.name, self.type.specialize(values), self.pos, self.cname)
 
 
 class ToPyStructUtilityCode(object):
@@ -4725,15 +4717,13 @@ class CypClassType(CppClassType):
         code.putln("Cy_DECREF(%s);" % cname)
 
     def generate_decref_clear(self, code, cname, **kwds):
-        code.putln("Cy_DECREF(%s);" % cname)
-        code.putln("%s = NULL;" % cname)
+        code.putln("Cy_DECREF(%s); %s = NULL;" % (cname, cname))
 
     def generate_xdecref(self, code, cname, **kwds):
         code.putln("Cy_XDECREF(%s);" % cname)
 
     def generate_xdecref_clear(self, code, cname, **kwds):
-        code.putln("Cy_XDECREF(%s);" % cname)
-        code.putln("%s = NULL;" % cname)
+        code.putln("Cy_XDECREF(%s); %s = NULL;" % (cname, cname))
 
     def generate_gotref(self, code, cname):
         code.putln("Cy_GOTREF(%s);" % cname)

Cython/Compiler/Symtab.py

@@ -180,6 +180,8 @@ class Entry(object):
     # original_name    string     The original name of a cpp or cypclass method
     #
     # active_entry     Entry      Entry for the active version of an asyncable cypclass method
+    #
+    # is_consumed      boolean    The entry is the operand of a 'consume' expression.
 
     # TODO: utility_code and utility_code_definition serves the same purpose...
 
@@ -258,6 +260,7 @@ class Entry(object):
     static_cname = None
     original_name = None
     active_entry = None
+    is_consumed = False
 
     def __init__(self, name, cname, type, pos = None, init = None):
         self.name = name

Cython/Utility/CyObjects.cpp

@@ -141,13 +141,7 @@
 
             constexpr Cy_Ref_impl() noexcept = default;
 
-            // constexpr Cy_Ref_impl(std::nullptr_t null) noexcept : uobj(null) {}
-
-            Cy_Ref_impl(T* const& uobj) : uobj(uobj) {
-                if (uobj != nullptr) {
-                    uobj->CyObject_INCREF();
-                }
-            }
+            constexpr Cy_Ref_impl(T* const& uobj) noexcept : uobj(uobj) {}
 
             constexpr Cy_Ref_impl(T* && uobj) noexcept : uobj(uobj) {}
 
@@ -280,6 +274,7 @@
             }
             template <typename U = T, typename std::enable_if<Cy_has_equality<U>::value, int>::type = 0>
             bool operator()(const Cy_Ref_impl<T>& lhs, const Cy_Ref_impl<T>& rhs) const {
+                Cy_INCREF(rhs.uobj);
                 return lhs.uobj->operator==(rhs.uobj);
             }
         };
@@ -420,6 +415,12 @@
             return ob->CyObject_GETREF();
         }
 
+        static inline int Cy_GETREF(const CyObject *ob) {
+            int refcnt = ob->CyObject_GETREF();
+            ob->CyObject_DECREF();
+            return refcnt;
+        }
+
         static inline void _Cy_RLOCK(const CyObject *ob, const char *context) {
             if (ob != NULL) {
                 ob->CyObject_RLOCK(context);
@@ -473,7 +474,9 @@
         template <typename T, typename O>
         static inline int isinstanceof(O ob) {
             static_assert(std::is_convertible<T, CyObject *>::value, "wrong type 'T' for isinstanceof[T]");
-            return dynamic_cast<const typename std::remove_pointer<T>::type *>(ob) != NULL;
+            bool result = dynamic_cast<const typename std::remove_pointer<T>::type *>(ob) != NULL;
+            Cy_DECREF(ob);
+            return result;
         }
 
         /*
@@ -482,7 +485,6 @@
         template <typename T>
         static inline T * activate(T * ob) {
             static_assert(std::is_convertible<T *, ActhonActivableClass *>::value, "wrong type for activate");
-            Cy_INCREF(ob);
             return ob;
         }
 
@@ -608,7 +610,6 @@
         /* Cast argument to CyObject* type. */
         #define _CyObject_CAST(ob) ob
 
-        #define Cy_GETREF(ob) (_Cy_GETREF(_CyObject_CAST(ob)))
         #define Cy_GOTREF(ob)
         #define Cy_XGOTREF(ob)
         #define Cy_GIVEREF(ob)