Actually implement py3 style metaclasses with support for __prepare__ and namespaces

Cython/Compiler/ExprNodes.py

@@ -4540,23 +4540,14 @@ class ClassNode(ExprNode, ModuleNameMixin):
     #  dict         ExprNode           Class dict (not owned by this node)
     #  doc          ExprNode or None   Doc string
     #  module_name  EncodedString      Name of defining module
-    #  keyword_args ExprNode or None   Py3 Dict of keyword arguments, passed to __new__
-    #  starstar_arg ExprNode or None   Py3 Dict of extra keyword args, same here
     
-    subexprs = ['bases', 'keyword_args', 'starstar_arg', 'doc']
+    subexprs = ['bases', 'doc']
 
     def analyse_types(self, env):
         self.bases.analyse_types(env)
         if self.doc:
             self.doc.analyse_types(env)
             self.doc = self.doc.coerce_to_pyobject(env)
-        if self.keyword_args:
-            self.keyword_args.analyse_types(env)
-        if self.starstar_arg:
-            self.starstar_arg.analyse_types(env)
-            # make sure we have a Python object as **kwargs mapping
-            self.starstar_arg = \
-                self.starstar_arg.coerce_to_pyobject(env)
         self.type = py_object_type
         self.is_temp = 1
         env.use_utility_code(create_class_utility_code);
@@ -4570,18 +4561,6 @@ class ClassNode(ExprNode, ModuleNameMixin):
 
     def generate_result_code(self, code):
         cname = code.intern_identifier(self.name)
-        if self.keyword_args and self.starstar_arg:
-            code.put_error_if_neg(self.pos,
-                "PyDict_Update(%s, %s)" % (
-                    self.keyword_args.py_result(),
-                    self.starstar_arg.py_result()))
-            keyword_code = self.keyword_args.py_result()
-        elif self.keyword_args:
-            keyword_code = self.keyword_args.py_result()
-        elif self.starstar_arg:
-            keyword_code = self.starstar_arg.py_result()
-        else:
-            keyword_code = 'NULL'
 
         if self.doc:
             code.put_error_if_neg(self.pos, 
@@ -4590,17 +4569,180 @@ class ClassNode(ExprNode, ModuleNameMixin):
                     self.doc.py_result()))
         py_mod_name = self.get_py_mod_name(code)
         code.putln(
-            '%s = __Pyx_CreateClass(%s, %s, %s, %s, %s); %s' % (
+            '%s = __Pyx_CreateClass(%s, %s, %s, %s); %s' % (
                 self.result(),
                 self.bases.py_result(),
                 self.dict.py_result(),
                 cname,
                 py_mod_name,
-                keyword_code,
                 code.error_goto_if_null(self.result(), self.pos)))
         code.put_gotref(self.py_result())
 
 
+class Py3ClassNode(ExprNode):
+    #  Helper class used in the implementation of Python3+
+    #  class definitions. Constructs a class object given
+    #  a name, tuple of bases and class dictionary.
+    #
+    #  name         EncodedString      Name of the class
+    #  dict         ExprNode           Class dict (not owned by this node)
+    #  module_name  EncodedString      Name of defining module
+
+    subexprs = []
+
+    def analyse_types(self, env):
+        self.type = py_object_type
+        self.is_temp = 1
+        env.use_utility_code(create_py3class_utility_code);
+
+    def may_be_none(self):
+        return True
+
+    gil_message = "Constructing Python3+ class"
+
+    def generate_result_code(self, code):
+        cname = code.intern_identifier(self.name)
+        code.putln(
+            '%s = __Pyx_Py3ClassCreate(%s, %s, %s, %s, %s); %s' % (
+                self.result(),
+                self.metaclass.result(),
+                cname,
+                self.bases.py_result(),
+                self.dict.py_result(),
+                self.mkw.py_result(),
+                code.error_goto_if_null(self.result(), self.pos)))
+        code.put_gotref(self.py_result())
+
+class KeywordArgsNode(ExprNode):
+    # Helper class for keyword arguments
+    #
+    # keyword_args ExprNode or None     Keyword arguments
+    # starstar_arg ExprNode or None     Extra arguments
+
+    subexprs = ['keyword_args', 'starstar_arg']
+
+    def analyse_types(self, env):
+        if self.keyword_args:
+            self.keyword_args.analyse_types(env)
+        if self.starstar_arg:
+            self.starstar_arg.analyse_types(env)
+            # make sure we have a Python object as **kwargs mapping
+            self.starstar_arg = \
+                self.starstar_arg.coerce_to_pyobject(env)
+        self.type = py_object_type
+        self.is_temp = 1
+
+    gil_message = "Constructing Keyword Args"
+
+    def generate_result_code(self, code):
+        if self.keyword_args and self.starstar_arg:
+            code.put_error_if_neg(self.pos,
+                "PyDict_Update(%s, %s)" % (
+                    self.keyword_args.py_result(),
+                    self.starstar_arg.py_result()))
+        if self.keyword_args:
+            code.putln("%s = %s;" % (self.result(), self.keyword_args.result()))
+            code.put_incref(self.keyword_args.result(), self.keyword_args.ctype())
+            code.put_giveref(self.keyword_args.result())
+        elif self.starstar_arg:
+            code.putln(
+                "%s = PyDict_Copy(%s); %s" % (
+                    self.result(),
+                    self.starstar_arg.py_result(),
+                    code.error_goto_if_null(self.result(), self.pos)))
+        else:
+            code.putln(
+                "%s = PyDict_New(); %s" % (
+                    self.result(),
+                    code.error_goto_if_null(self.result(), self.pos)))
+        code.put_gotref(self.py_result())
+
+class PyClassBasesNode(ExprNode):
+    # Helper class that holds bases for python3 class
+    # Actually hack to make `bases` visible across other nondes
+    #
+    # bases         ExprNode           Base class tuple
+
+    subexprs = ['bases']
+    type = py_object_type
+    is_temp = True
+
+    def analyse_types(self, env):
+        self.bases.analyse_types(env)
+
+    def generate_result_code(self, code):
+        code.putln("%s = %s;" % (self.result(), self.bases.result()))
+        code.put_incref(self.bases.result(), self.bases.ctype())
+        code.put_giveref(self.bases.result())
+        code.put_gotref(self.py_result())
+
+class PyClassMetaclassNode(ExprNode):
+    # Helper class holds Python3 metaclass object
+    #
+    #  bases        ExprNode           Base class tuple (not owned by this node)
+    #  mkw          ExprNode           Class keyword arguments (not owned by this node)
+
+    subexprs = []
+
+    def analyse_types(self, env):
+        self.type = py_object_type
+        self.is_temp = True
+
+    def may_be_none(self):
+        return True
+
+    def generate_result_code(self, code):
+        code.putln(
+            "%s = __Pyx_Py3MetaclassGet(%s, %s); %s" % (
+                self.result(),
+                self.bases.result(),
+                self.mkw.result(),
+                code.error_goto_if_null(self.result(), self.pos)))
+        code.put_gotref(self.py_result())
+
+class PyClassNamespaceNode(ExprNode, ModuleNameMixin):
+    # Helper class holds Python3 namespace object
+    #
+    # All this are not owned by this node
+    #  metaclass    ExprNode           Metaclass object
+    #  bases        ExprNode           Base class tuple
+    #  mkw          ExprNode           Class keyword arguments
+    #  doc          ExprNode or None   Doc string (owned)
+
+    subexprs = ['doc']
+
+    def analyse_types(self, env):
+        self.bases.analyse_types(env)
+        if self.doc:
+            self.doc.analyse_types(env)
+            self.doc = self.doc.coerce_to_pyobject(env)
+        self.type = py_object_type
+        self.is_temp = 1
+        #TODO(craig,haoyu) This should be moved to a better place
+        self.set_mod_name(env)
+
+    def may_be_none(self):
+        return True
+
+    def generate_result_code(self, code):
+        cname = code.intern_identifier(self.name)
+        py_mod_name = self.get_py_mod_name(code)
+        if self.doc:
+            doc_code = self.doc.result()
+        else:
+            doc_code = '(PyObject *) NULL'
+        code.putln(
+            "%s = __Pyx_Py3MetaclassPrepare(%s, %s, %s, %s, %s, %s); %s" % (
+                self.result(),
+                self.metaclass.result(),
+                self.bases.result(),
+                cname,
+                self.mkw.result(),
+                py_mod_name,
+                doc_code,
+                code.error_goto_if_null(self.result(), self.pos)))
+        code.put_gotref(self.py_result())
+
 class BoundMethodNode(ExprNode):
     #  Helper class used in the implementation of Python
     #  class definitions. Constructs an bound method
@@ -7319,74 +7461,20 @@ static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type) {
 create_class_utility_code = UtilityCode(
 proto = """
 static PyObject *__Pyx_CreateClass(PyObject *bases, PyObject *dict, PyObject *name,
-                                   PyObject *modname, PyObject *kwargs); /*proto*/
-static int __Pyx_PrepareClass(PyObject *metaclass, PyObject *bases, PyObject *name,
-                              PyObject *mkw, PyObject *dict); /*proto*/
+                                   PyObject *modname); /*proto*/
 """,
 impl = """
-static int __Pyx_PrepareClass(PyObject *metaclass, PyObject *bases, PyObject *name,
-                              PyObject *mkw, PyObject *dict) {
-    PyObject *prep;
-    PyObject *pargs;
-    PyObject *ns;
-
-    prep = PyObject_GetAttrString(metaclass, "__prepare__");
-    if (prep == NULL) {
-        if (!PyErr_ExceptionMatches(PyExc_AttributeError))
-            return -1;
-        PyErr_Clear();
-        return 0;
-    }
-    pargs = PyTuple_New(2);
-    if (!pargs) {
-        Py_DECREF(prep);
-        return -1;
-    }
-    Py_INCREF(name);
-    Py_INCREF(bases);
-    PyTuple_SET_ITEM(pargs, 0, name);
-    PyTuple_SET_ITEM(pargs, 1, bases);
-    ns = PyEval_CallObjectWithKeywords(prep, pargs, mkw);
-    Py_DECREF(pargs);
-    Py_DECREF(prep);
-    if (ns == NULL)
-        return -1;
-    /* XXX: This is hack, merge namespace back to dict,
-       __prepare__ should be ran before dict initialization */
-    if (PyDict_Merge(dict, ns, 0)) {
-        Py_DECREF(ns);
-        return -1;
-    }
-    Py_DECREF(ns);
-    return 0;
-}
-
 static PyObject *__Pyx_CreateClass(PyObject *bases, PyObject *dict, PyObject *name,
-                                   PyObject *modname, PyObject *kwargs) {
+                                   PyObject *modname) {
     PyObject *result = NULL;
     PyObject *metaclass = NULL;
-    PyObject *mkw = NULL;
 
     if (PyDict_SetItemString(dict, "__module__", modname) < 0)
         return NULL;
 
-    /* Python3 metaclasses */
-    if (kwargs) {
-        mkw = PyDict_Copy(kwargs); /* Don't modify kwargs passed in! */
-        if (!mkw)
-            return NULL;
-        metaclass = PyDict_GetItemString(mkw, "metaclass");
-        if (metaclass) {
-            Py_INCREF(metaclass);
-            if (PyDict_DelItemString(mkw, "metaclass") < 0)
-                goto bad;
-            if (__Pyx_PrepareClass(metaclass, bases, name, mkw, dict))
-                goto bad;
-        }
-    }
-    if (!metaclass) {
     /* Python2 __metaclass__ */
     metaclass = PyDict_GetItemString(dict, "__metaclass__");
+
     if (!metaclass) {
         /* Default metaclass */
 #if PY_MAJOR_VERSION < 3
@@ -7408,25 +7496,132 @@ static PyObject *__Pyx_CreateClass(PyObject *bases, PyObject *dict, PyObject *na
 #endif
     }
     Py_INCREF(metaclass);
+
+    result = PyObject_CallFunctionObjArgs(metaclass, name, bases, dict, NULL);
+    Py_DECREF(metaclass);
+    return result;
+}
+""")
+
+#------------------------------------------------------------------------------------
+
+create_py3class_utility_code = UtilityCode(
+proto = """
+static PyObject *__Pyx_Py3MetaclassGet(PyObject *bases, PyObject *mkw);
+static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, PyObject *mkw, PyObject *modname, PyObject *doc);
+static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObject *bases, PyObject *dict, PyObject *mkw);
+""",
+impl = """
+PyObject *__Pyx_Py3MetaclassGet(PyObject *bases, PyObject *mkw)
+{
+    PyObject *metaclass = NULL;
+
+    metaclass = PyDict_GetItemString(mkw, "metaclass");
+    if (metaclass) {
+        Py_INCREF(metaclass);
+        if (PyDict_DelItemString(mkw, "metaclass") < 0) {
+            Py_DECREF(metaclass);
+            return NULL;
         }
-    if (mkw && PyDict_Size(mkw) > 0) {
-        PyObject *margs = PyTuple_New(3);
-        if (!margs)
-            goto bad;
+        return metaclass;
+    }
+    /* Default metaclass */
+#if PY_MAJOR_VERSION < 3
+    if (PyTuple_Check(bases) && PyTuple_GET_SIZE(bases) > 0) {
+        PyObject *base = PyTuple_GET_ITEM(bases, 0);
+        metaclass = PyObject_GetAttrString(base, "__class__");
+        if (metaclass == NULL) {
+            PyErr_Clear();
+            metaclass = (PyObject *)base->ob_type;
+        }
+    } else {
+        metaclass = (PyObject *) &PyClass_Type;
+    }
+#else
+    if (PyTuple_Check(bases) && PyTuple_GET_SIZE(bases) > 0) {
+        PyObject *base = PyTuple_GET_ITEM(bases, 0);
+        metaclass = (PyObject *)base->ob_type;
+    } else {
+        metaclass = (PyObject *) &PyType_Type;
+    }
+#endif
+    Py_INCREF(metaclass);
+    return metaclass;
+}
+
+PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, PyObject *mkw, PyObject *modname, PyObject *doc)
+{
+    PyObject *prep;
+    PyObject *pargs;
+    PyObject *ns;
+    PyObject *str;
+
+    prep = PyObject_GetAttrString(metaclass, "__prepare__");
+    if (prep == NULL) {
+        if (!PyErr_ExceptionMatches(PyExc_AttributeError))
+            return NULL;
+        PyErr_Clear();
+        return PyDict_New();
+    }
+    pargs = PyTuple_New(2);
+    if (!pargs) {
+        Py_DECREF(prep);
+        return NULL;
+    }
+
     Py_INCREF(name);
     Py_INCREF(bases);
-        Py_INCREF(dict);
-        PyTuple_SET_ITEM(margs, 0, name);
-        PyTuple_SET_ITEM(margs, 1, bases);
-        PyTuple_SET_ITEM(margs, 2, dict);
+    PyTuple_SET_ITEM(pargs, 0, name);
+    PyTuple_SET_ITEM(pargs, 1, bases);
+
+    ns = PyEval_CallObjectWithKeywords(prep, pargs, mkw);
+
+    Py_DECREF(prep);
+    Py_DECREF(pargs);
+
+    if (ns == NULL)
+        return NULL;
+
+    /* Required here to emulate assignment order */
+    /* XXX: use consts here */
+    str = PyString_FromString("__module__");
+    if (!str) {
+        Py_DECREF(ns);
+        return NULL;
+    }
+
+    if (PyObject_SetItem(ns, str, modname) < 0) {
+        Py_DECREF(ns);
+        Py_DECREF(str);
+        return NULL;
+    }
+    Py_DECREF(str);
+    if (doc) {
+        str = PyString_FromString("__doc__");
+        if (!str) {
+            Py_DECREF(ns);
+            return NULL;
+        }
+        if (PyObject_SetItem(ns, str, doc) < 0) {
+            Py_DECREF(ns);
+            Py_DECREF(str);
+            return NULL;
+        }
+        Py_DECREF(str);
+    }
+    return ns;
+}
+
+PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObject *bases, PyObject *dict, PyObject *mkw)
+{
+    PyObject *result;
+    PyObject *margs;
+
+    margs = PyTuple_Pack(3, name, bases, dict, NULL);
+    if (!margs)
+        return NULL;
     result = PyEval_CallObjectWithKeywords(metaclass, margs, mkw);
     Py_DECREF(margs);
-    } else {
-        result = PyObject_CallFunctionObjArgs(metaclass, name, bases, dict, NULL);
-    }
-bad:
-    Py_DECREF(metaclass);
-    Py_XDECREF(mkw);
     return result;
 }
 """)

Cython/Compiler/Nodes.py

@@ -2934,12 +2934,13 @@ class PyClassDefNode(ClassDefNode):
     #
     #  The following subnodes are constructed internally:
     #
-    #  dict     DictNode   Class dictionary
+    #  dict     DictNode   Class dictionary or Py3 namespace
     #  classobj ClassNode  Class object
     #  target   NameNode   Variable to assign class object to
 
-    child_attrs = ["body", "dict", "classobj", "target"]
+    child_attrs = ["body", "dict", "metaclass", "mkw", "bases", "classobj", "target"]
     decorators = None
+    py3_style_class = False # Python3 style class
     
     def __init__(self, pos, name, bases, doc, body, decorators = None,
                  keyword_args = None, starstar_arg = None):
@@ -2949,22 +2950,40 @@ class PyClassDefNode(ClassDefNode):
         self.body = body
         self.decorators = decorators
         import ExprNodes
-        self.dict = ExprNodes.DictNode(pos, key_value_pairs = [])
         if self.doc and Options.docstrings:
             doc = embed_position(self.pos, self.doc)
             # FIXME: correct string node?
             doc_node = ExprNodes.StringNode(pos, value = doc)
         else:
             doc_node = None
-        self.classobj = ExprNodes.ClassNode(pos, name = name,
-            bases = bases, dict = self.dict, doc = doc_node,
+        if keyword_args or starstar_arg:
+            self.py3_style_class = True
+            self.bases = ExprNodes.PyClassBasesNode(pos, bases = bases)
+            self.mkw = ExprNodes.KeywordArgsNode(pos,
                     keyword_args = keyword_args, starstar_arg = starstar_arg)
+            self.metaclass = ExprNodes.PyClassMetaclassNode(pos, mkw = self.mkw, bases = self.bases)
+            self.dict = ExprNodes.PyClassNamespaceNode(pos, name = name,
+                        doc = doc_node, metaclass = self.metaclass, bases = self.bases,
+                        mkw = self.mkw, )
+            self.classobj = ExprNodes.Py3ClassNode(pos, name = name,
+                    bases = self.bases, dict = self.dict, doc = doc_node,
+                    metaclass = self.metaclass, mkw = self.mkw)
+        else:
+            self.dict = ExprNodes.DictNode(pos, key_value_pairs = [])
+            self.metaclass = None
+            self.mkw = None
+            self.bases = None
+            self.classobj = ExprNodes.ClassNode(pos, name = name,
+                    bases = bases, dict = self.dict, doc = doc_node)
         self.target = ExprNodes.NameNode(pos, name = name)
         
     def as_cclass(self):
         """
         Return this node as if it were declared as an extension class
         """
+        if self.py3_style_class:
+            error(self.classobj.pos, "Python3 style class could not be represented as C class")
+            return
         bases = self.classobj.bases.args
         if len(bases) == 0:
             base_class_name = None
@@ -3015,6 +3034,10 @@ class PyClassDefNode(ClassDefNode):
         self.body.analyse_declarations(cenv)
     
     def analyse_expressions(self, env):
+        if self.py3_style_class:
+            self.bases.analyse_expressions(env)
+            self.metaclass.analyse_expressions(env)
+            self.mkw.analyse_expressions(env)
         self.dict.analyse_expressions(env)
         self.classobj.analyse_expressions(env)
         genv = env.global_scope()
@@ -3028,6 +3051,10 @@ class PyClassDefNode(ClassDefNode):
     def generate_execution_code(self, code):
         code.pyclass_stack.append(self)
         cenv = self.scope
+        if self.py3_style_class:
+            self.bases.generate_evaluation_code(code)
+            self.mkw.generate_evaluation_code(code)
+            self.metaclass.generate_evaluation_code(code)
         self.dict.generate_evaluation_code(code)
         cenv.namespace_cname = cenv.class_obj_cname = self.dict.result()
         self.body.generate_execution_code(code)
@@ -3036,9 +3063,15 @@ class PyClassDefNode(ClassDefNode):
         self.target.generate_assignment_code(self.classobj, code)
         self.dict.generate_disposal_code(code)
         self.dict.free_temps(code)
+        if self.py3_style_class:
+            self.mkw.generate_disposal_code(code)
+            self.mkw.free_temps(code)
+            self.metaclass.generate_disposal_code(code)
+            self.metaclass.free_temps(code)
+            self.bases.generate_disposal_code(code)
+            self.bases.free_temps(code)
         code.pyclass_stack.pop()
 
-
 class CClassDefNode(ClassDefNode):
     #  An extension type definition.
     #

tests/run/metaclass.pyx

@@ -12,17 +12,28 @@ class Foo(object):
     """
     __metaclass__ = Base
 
+class ODict(dict):
+    def __init__(self):
+        dict.__init__(self)
+        self._order = []
+        dict.__setitem__(self, '_order', self._order)
+
+    def __setitem__(self, key, value):
+        dict.__setitem__(self, key, value)
+        self._order.append(key)
 
 class Py3Base(type):
-    def __new__(cls, name, bases, attrs, foo=None):
-        attrs['foo'] = foo
+    def __new__(cls, name, bases, attrs, **kwargs):
+        for key, value in kwargs.items():
+            attrs[key] = value
         return type.__new__(cls, name, bases, attrs)
 
-    def __init__(self, cls, attrs, obj, foo=None):
+    def __init__(self, cls, attrs, obj, **kwargs):
         pass
+
     @staticmethod
-    def __prepare__(name, bases, **kwargs):
-        return {'bar': 666, 'dirty': True}
+    def __prepare__(*args, **kwargs):
+        return ODict()
 
 class Py3Foo(object, metaclass=Py3Base, foo=123):
     """
@@ -30,8 +41,48 @@ class Py3Foo(object, metaclass=Py3Base, foo=123):
     >>> obj.foo
     123
     >>> obj.bar
-    666
-    >>> obj.dirty
-    False
+    321
+    >>> obj._order
+    ['__module__', '__doc__', 'bar', 'foo']
+    """
+    bar = 321
+
+kwargs = {'foo': 123, 'bar': 456}
+
+class Py3Mixed(metaclass=Py3Base, **kwargs):
+    """
+    >>> Py3Mixed.foo
+    123
+    >>> Py3Mixed.bar
+    456
+    """
+
+kwargs['metaclass'] = Py3Base
+
+class Py3Kwargs(**kwargs):
+    """
+    >>> Py3Kwargs.foo
+    123
+    >>> Py3Kwargs.bar
+    456
+    """
+
+class Base3(type):
+    def __new__(cls, name, bases, attrs, **kwargs):
+        kwargs['b'] = 2
+        return type.__new__(cls, name, bases, attrs)
+
+    def __init__(self, *args, **kwargs):
+        self.kwargs = kwargs
+
+    @staticmethod
+    def __prepare__(*args, **kwargs):
+        kwargs['a'] = 1
+        return {}
+
+kwargs = {'c': 0}
+class Foo3(metaclass=Base3, a=0, b=0, **kwargs):
+    """
+    >>> Foo3.kwargs
+    {'a': 0, 'c': 0, 'b': 0}
     """
-    dirty = False