Merge branch 'typeready'

Makefile

@@ -928,7 +928,7 @@ test_cpp_ll:
 	less test.ll
 	rm test.ll
 
-TEST_EXT_MODULE_NAMES := basic_test descr_test
+TEST_EXT_MODULE_NAMES := basic_test descr_test slots_test
 
 .PHONY: ext_pyston
 ext_pyston: $(TEST_EXT_MODULE_NAMES:%=$(TEST_DIR)/test_extension/%.pyston.so)

src/capi/descrobject.cpp

+// Copyright (c) 2014 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "capi/types.h"
+#include "runtime/objmodel.h"
+
+namespace pyston {
+
+Box* BoxedMethodDescriptor::__call__(BoxedMethodDescriptor* self, Box* obj, BoxedTuple* varargs, Box** _args) {
+    BoxedDict* kwargs = static_cast<BoxedDict*>(_args[0]);
+
+    assert(self->cls == method_cls);
+    assert(varargs->cls == tuple_cls);
+    assert(kwargs->cls == dict_cls);
+
+    if (!isSubclass(obj->cls, self->type))
+        raiseExcHelper(TypeError, "descriptor '%s' requires a '%s' object but received a '%s'", self->method->ml_name,
+                       getFullNameOfClass(self->type).c_str(), getFullTypeName(obj).c_str());
+
+    threading::GLPromoteRegion _gil_lock;
+
+    int ml_flags = self->method->ml_flags;
+    Box* rtn;
+    if (ml_flags == METH_NOARGS) {
+        assert(varargs->elts.size() == 0);
+        assert(kwargs->d.size() == 0);
+        rtn = (Box*)self->method->ml_meth(obj, NULL);
+    } else if (ml_flags == METH_VARARGS) {
+        assert(kwargs->d.size() == 0);
+        rtn = (Box*)self->method->ml_meth(obj, varargs);
+    } else if (ml_flags == (METH_VARARGS | METH_KEYWORDS)) {
+        rtn = (Box*)((PyCFunctionWithKeywords)self->method->ml_meth)(obj, varargs, kwargs);
+    } else if (ml_flags == METH_O) {
+        assert(kwargs->d.size() == 0);
+        assert(varargs->elts.size() == 1);
+        rtn = (Box*)self->method->ml_meth(obj, varargs->elts[0]);
+    } else {
+        RELEASE_ASSERT(0, "0x%x", ml_flags);
+    }
+    assert(rtn);
+    return rtn;
+}
+}

src/capi/typeobject.cpp

+// Copyright (c) 2014 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+#include "capi/typeobject.h"
+
+#include "capi/types.h"
+#include "runtime/objmodel.h"
+
+namespace pyston {
+
+// FIXME duplicated with objmodel.cpp
+static const std::string _new_str("__new__");
+
+extern "C" void conservativeGCHandler(GCVisitor* v, Box* b) {
+    v->visitPotentialRange((void* const*)b, (void* const*)((char*)b + b->cls->tp_basicsize));
+}
+
+static int check_num_args(PyObject* ob, int n) {
+    if (!PyTuple_CheckExact(ob)) {
+        PyErr_SetString(PyExc_SystemError, "PyArg_UnpackTuple() argument list is not a tuple");
+        return 0;
+    }
+    if (n == PyTuple_GET_SIZE(ob))
+        return 1;
+    PyErr_Format(PyExc_TypeError, "expected %d arguments, got %zd", n, PyTuple_GET_SIZE(ob));
+    return 0;
+}
+
+static PyObject* wrap_call(PyObject* self, PyObject* args, void* wrapped, PyObject* kwds) {
+    ternaryfunc func = (ternaryfunc)wrapped;
+
+    return (*func)(self, args, kwds);
+}
+
+static PyObject* wrap_unaryfunc(PyObject* self, PyObject* args, void* wrapped) {
+    unaryfunc func = (unaryfunc)wrapped;
+
+    if (!check_num_args(args, 0))
+        return NULL;
+    return (*func)(self);
+}
+
+PyObject* Py_CallPythonNew(PyTypeObject* self, PyObject* args, PyObject* kwds) {
+    try {
+        // TODO: runtime ICs?
+        Box* new_attr = typeLookup(self, _new_str, NULL);
+        assert(new_attr);
+        new_attr = processDescriptor(new_attr, None, self);
+
+        return runtimeCall(new_attr, ArgPassSpec(1, 0, true, true), self, args, kwds, NULL, NULL);
+    } catch (Box* e) {
+        abort();
+    }
+}
+
+PyObject* Py_CallPythonCall(PyObject* self, PyObject* args, PyObject* kwds) {
+    try {
+        Py_FatalError("this function is untested");
+
+        // TODO: runtime ICs?
+        return runtimeCall(self, ArgPassSpec(0, 0, true, true), args, kwds, NULL, NULL, NULL);
+    } catch (Box* e) {
+        abort();
+    }
+}
+
+PyObject* Py_CallPythonRepr(PyObject* self) {
+    try {
+        return repr(self);
+    } catch (Box* e) {
+        abort();
+    }
+}
+
+typedef wrapper_def slotdef;
+
+static void** slotptr(BoxedClass* self, int offset) {
+    // TODO handle indices into the indirected portions (tp_as_sequence, etc)
+    char* ptr = reinterpret_cast<char*>(self);
+    return reinterpret_cast<void**>(ptr + offset);
+}
+
+static void update_one_slot(BoxedClass* self, const slotdef& p) {
+    // TODO: CPython version is significantly more sophisticated
+    void** ptr = slotptr(self, p.offset);
+    assert(ptr);
+
+    if (typeLookup(self, p.name, NULL))
+        *ptr = p.function;
+    else
+        *ptr = NULL;
+}
+
+static slotdef slotdefs[] = {
+    { "__repr__", offsetof(PyTypeObject, tp_repr), (void*)&Py_CallPythonRepr, wrap_unaryfunc, 0 },
+    { "__call__", offsetof(PyTypeObject, tp_call), (void*)&Py_CallPythonCall, (wrapperfunc)wrap_call,
+      PyWrapperFlag_KEYWORDS },
+    { "__new__", offsetof(PyTypeObject, tp_new), (void*)&Py_CallPythonNew, NULL, 0 },
+};
+
+static void init_slotdefs() {
+    static bool initialized = false;
+    if (initialized)
+        return;
+
+    for (int i = 0; i < sizeof(slotdefs) / sizeof(slotdefs[0]); i++) {
+        if (i > 0) {
+            ASSERT(slotdefs[i].offset >= slotdefs[i - 1].offset, "%d %s", i, slotdefs[i - 1].name);
+            // CPython interns the name here
+        }
+    }
+
+    initialized = true;
+}
+
+bool update_slot(BoxedClass* self, const std::string& attr) {
+    bool updated = false;
+    for (const slotdef& p : slotdefs) {
+        if (p.name == attr) {
+            // TODO update subclasses;
+            update_one_slot(self, p);
+            updated = true;
+        }
+    }
+    return updated;
+}
+
+void fixup_slot_dispatchers(BoxedClass* self) {
+    init_slotdefs();
+
+    for (const slotdef& p : slotdefs) {
+        update_one_slot(self, p);
+    }
+
+    // TODO: CPython handles this by having the __name__ attribute wrap (via a getset object)
+    // the tp_name field, whereas we're (needlessly?) doing the opposite.
+    if (!self->tp_name) {
+        Box* b = self->getattr("__name__");
+        assert(b);
+        assert(b->cls == str_cls);
+        self->tp_name = static_cast<BoxedString*>(b)->s.c_str();
+    }
+}
+
+static PyObject* tp_new_wrapper(PyTypeObject* self, BoxedTuple* args, Box* kwds) {
+    RELEASE_ASSERT(isSubclass(self->cls, type_cls), "");
+
+    // ASSERT(self->tp_new != Py_CallPythonNew, "going to get in an infinite loop");
+
+    RELEASE_ASSERT(args->cls == tuple_cls, "");
+    RELEASE_ASSERT(kwds->cls == dict_cls, "");
+    RELEASE_ASSERT(args->elts.size() >= 1, "");
+
+    BoxedClass* subtype = static_cast<BoxedClass*>(args->elts[0]);
+    RELEASE_ASSERT(isSubclass(subtype->cls, type_cls), "");
+    RELEASE_ASSERT(isSubclass(subtype, self), "");
+
+    BoxedTuple* new_args = new BoxedTuple(BoxedTuple::GCVector(args->elts.begin() + 1, args->elts.end()));
+
+    return self->tp_new(subtype, new_args, kwds);
+}
+
+static void add_tp_new_wrapper(BoxedClass* type) {
+    if (type->getattr("__new__"))
+        return;
+
+    type->giveAttr("__new__",
+                   new BoxedCApiFunction(METH_VARARGS | METH_KEYWORDS, type, "__new__", (PyCFunction)tp_new_wrapper));
+}
+
+static void add_operators(BoxedClass* cls) {
+    init_slotdefs();
+
+    for (const slotdef& p : slotdefs) {
+        if (!p.wrapper)
+            continue;
+
+        void** ptr = slotptr(cls, p.offset);
+
+        if (!ptr || !*ptr)
+            continue;
+        if (cls->getattr(p.name))
+            continue;
+        // TODO PyObject_HashNotImplemented
+
+        cls->giveAttr(p.name, new BoxedWrapperDescriptor(&p, cls));
+    }
+
+    if (cls->tp_new)
+        add_tp_new_wrapper(cls);
+}
+
+extern "C" int PyType_IsSubtype(PyTypeObject* a, PyTypeObject* b) {
+    return isSubclass(a, b);
+}
+
+extern "C" int PyType_Ready(PyTypeObject* cls) {
+    gc::registerNonheapRootObject(cls);
+
+    // unhandled fields:
+    RELEASE_ASSERT(cls->tp_print == NULL, "");
+    RELEASE_ASSERT(cls->tp_getattr == NULL, "");
+    RELEASE_ASSERT(cls->tp_setattr == NULL, "");
+    RELEASE_ASSERT(cls->tp_compare == NULL, "");
+    RELEASE_ASSERT(cls->tp_as_number == NULL, "");
+    RELEASE_ASSERT(cls->tp_as_sequence == NULL, "");
+    RELEASE_ASSERT(cls->tp_as_mapping == NULL, "");
+    RELEASE_ASSERT(cls->tp_hash == NULL, "");
+    RELEASE_ASSERT(cls->tp_str == NULL, "");
+    RELEASE_ASSERT(cls->tp_getattro == NULL || cls->tp_getattro == PyObject_GenericGetAttr, "");
+    RELEASE_ASSERT(cls->tp_setattro == NULL, "");
+    RELEASE_ASSERT(cls->tp_as_buffer == NULL, "");
+
+    int ALLOWABLE_FLAGS = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC;
+    RELEASE_ASSERT((cls->tp_flags & ~ALLOWABLE_FLAGS) == 0, "");
+
+    RELEASE_ASSERT(cls->tp_richcompare == NULL, "");
+    RELEASE_ASSERT(cls->tp_iter == NULL, "");
+    RELEASE_ASSERT(cls->tp_iternext == NULL, "");
+    RELEASE_ASSERT(cls->tp_base == NULL, "");
+    RELEASE_ASSERT(cls->tp_descr_get == NULL, "");
+    RELEASE_ASSERT(cls->tp_descr_set == NULL, "");
+    RELEASE_ASSERT(cls->tp_init == NULL, "");
+    RELEASE_ASSERT(cls->tp_alloc == NULL, "");
+    RELEASE_ASSERT(cls->tp_free == NULL || cls->tp_free == PyObject_Del, "");
+    RELEASE_ASSERT(cls->tp_is_gc == NULL, "");
+    RELEASE_ASSERT(cls->tp_base == NULL, "");
+    RELEASE_ASSERT(cls->tp_mro == NULL, "");
+    RELEASE_ASSERT(cls->tp_cache == NULL, "");
+    RELEASE_ASSERT(cls->tp_subclasses == NULL, "");
+    RELEASE_ASSERT(cls->tp_weaklist == NULL, "");
+    RELEASE_ASSERT(cls->tp_del == NULL, "");
+    RELEASE_ASSERT(cls->tp_version_tag == 0, "");
+
+// I think it is safe to ignore these for for now:
+// RELEASE_ASSERT(cls->tp_weaklistoffset == 0, "");
+// RELEASE_ASSERT(cls->tp_traverse == NULL, "");
+// RELEASE_ASSERT(cls->tp_clear == NULL, "");
+
+#define INITIALIZE(a) new (&(a)) decltype(a)
+    INITIALIZE(cls->attrs);
+    INITIALIZE(cls->dependent_icgetattrs);
+#undef INITIALIZE
+
+    BoxedClass* base = cls->base = object_cls;
+    if (!cls->cls)
+        cls->cls = cls->base->cls;
+
+    assert(cls->tp_dict == NULL);
+    cls->tp_dict = makeAttrWrapper(cls);
+
+    assert(cls->tp_name);
+    cls->giveAttr("__name__", boxStrConstant(cls->tp_name));
+    // tp_name
+    // tp_basicsize, tp_itemsize
+    // tp_doc
+
+    if (!cls->tp_new && base != object_cls)
+        cls->tp_new = base->tp_new;
+
+    if (!cls->tp_alloc) {
+        cls->tp_alloc = reinterpret_cast<decltype(cls->tp_alloc)>(PyType_GenericAlloc);
+    }
+
+    try {
+        add_operators(cls);
+    } catch (Box* b) {
+        abort();
+    }
+
+    for (PyMethodDef* method = cls->tp_methods; method && method->ml_name; ++method) {
+        cls->giveAttr(method->ml_name, new BoxedMethodDescriptor(method, cls));
+    }
+
+    for (PyMemberDef* member = cls->tp_members; member && member->name; ++member) {
+        cls->giveAttr(member->name, new BoxedMemberDescriptor(member));
+    }
+
+    if (cls->tp_getset) {
+        if (VERBOSITY())
+            printf("warning: ignoring tp_getset for now\n");
+    }
+
+    cls->gc_visit = &conservativeGCHandler;
+
+    // TODO not sure how we can handle extension types that manually
+    // specify a dict...
+    RELEASE_ASSERT(cls->tp_dictoffset == 0, "");
+    // this should get automatically initialized to 0 on this path:
+    assert(cls->attrs_offset == 0);
+
+    return 0;
+}
+
+} // namespace pyston

src/capi/typeobject.h

+// Copyright (c) 2014 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef PYSTON_CAPI_TYPEOBJECT_H
+#define PYSTON_CAPI_TYPEOBJECT_H
+
+#include "runtime/types.h"
+
+namespace pyston {
+
+// Returns if a slot was updated
+bool update_slot(BoxedClass* self, const std::string& attr);
+
+void fixup_slot_dispatchers(BoxedClass* self);
+}
+
+#endif

src/capi/types.h

@@ -19,7 +19,21 @@
 
 namespace pyston {
 
-extern BoxedClass* capifunc_cls;
+typedef PyObject* (*wrapperfunc)(PyObject* self, PyObject* args, void* wrapped);
+typedef PyObject* (*wrapperfunc_kwds)(PyObject* self, PyObject* args, void* wrapped, PyObject* kwds);
+
+struct wrapper_def {
+    const char* name;
+    int offset;
+    void* function;      // "generic" handler that gets put in the tp_* slot which proxies to the python version
+    wrapperfunc wrapper; // "wrapper" that ends up getting called by the Python-visible WrapperDescr
+    // exists in CPython: const char* doc
+    int flags;
+    // exists in CPython: PyObject *name_strobj
+};
+
+extern BoxedClass* capifunc_cls, *wrapperdescr_cls, *wrapperobject_cls;
+
 class BoxedCApiFunction : public Box {
 private:
     int ml_flags;
@@ -64,6 +78,64 @@ public:
         return rtn;
     }
 };
-}
+
+class BoxedWrapperDescriptor : public Box {
+public:
+    const wrapper_def* wrapper;
+    BoxedClass* type;
+    BoxedWrapperDescriptor(const wrapper_def* wrapper, BoxedClass* type)
+        : Box(wrapperdescr_cls), wrapper(wrapper), type(type) {}
+
+    static Box* __get__(BoxedWrapperDescriptor* self, Box* inst, Box* owner);
+};
+
+class BoxedWrapperObject : public Box {
+public:
+    BoxedWrapperDescriptor* descr;
+    Box* obj;
+
+    BoxedWrapperObject(BoxedWrapperDescriptor* descr, Box* obj) : Box(wrapperobject_cls), descr(descr), obj(obj) {}
+
+    static Box* __call__(BoxedWrapperObject* self, Box* args, Box* kwds) {
+        assert(self->cls == wrapperobject_cls);
+        assert(args->cls == tuple_cls);
+        assert(kwds->cls == dict_cls);
+
+        int flags = self->descr->wrapper->flags;
+        wrapperfunc wrapper = self->descr->wrapper->wrapper;
+        assert(self->descr->wrapper->offset > 0);
+        char* ptr = (char*)self->descr->type + self->descr->wrapper->offset;
+        void* wrapped = *reinterpret_cast<void**>(ptr);
+
+        if (flags & PyWrapperFlag_KEYWORDS) {
+            wrapperfunc_kwds wk = (wrapperfunc_kwds)wrapper;
+            return (*wk)(self->obj, args, wrapped, kwds);
+        } else {
+            return (*wrapper)(self->obj, args, wrapped);
+        }
+        abort();
+    }
+};
+
+class BoxedMethodDescriptor : public Box {
+public:
+    PyMethodDef* method;
+    BoxedClass* type;
+
+    BoxedMethodDescriptor(PyMethodDef* method, BoxedClass* type) : Box(method_cls), method(method), type(type) {}
+
+    static Box* __get__(BoxedMethodDescriptor* self, Box* inst, Box* owner) {
+        RELEASE_ASSERT(self->cls == method_cls, "");
+
+        if (inst == None)
+            return self;
+        // CPython apparently returns a "builtin_function_or_method" object
+        return boxInstanceMethod(inst, self);
+    }
+
+    static Box* __call__(BoxedMethodDescriptor* self, Box* obj, BoxedTuple* varargs, Box** _args);
+};
+
+} // namespace pyston
 
 #endif

src/runtime/builtin_modules/builtins.cpp

@@ -18,6 +18,7 @@
 
 #include "llvm/Support/FileSystem.h"
 
+#include "capi/typeobject.h"
 #include "codegen/ast_interpreter.h"
 #include "codegen/irgen/hooks.h"
 #include "codegen/parser.h"
@@ -528,6 +529,8 @@ extern "C" PyObject* PyErr_NewException(char* name, PyObject* _base, PyObject* d
 
         cls->giveAttr("__module__", boxStrConstantSize(name, dot_pos - name));
         cls->giveAttr("__name__", boxStrConstantSize(dot_pos + 1, n - (dot_pos - name) - 1));
+        // TODO Not sure if this should be called here
+        fixup_slot_dispatchers(cls);
         return cls;
     } catch (Box* e) {
         abort();

src/runtime/capi.cpp

@@ -28,57 +28,6 @@
 namespace pyston {
 
 BoxedClass* method_cls;
-class BoxedMethodDescriptor : public Box {
-public:
-    PyMethodDef* method;
-    BoxedClass* type;
-
-    BoxedMethodDescriptor(PyMethodDef* method, BoxedClass* type) : Box(method_cls), method(method), type(type) {}
-
-    static Box* __get__(BoxedMethodDescriptor* self, Box* inst, Box* owner) {
-        RELEASE_ASSERT(self->cls == method_cls, "");
-
-        if (inst == None)
-            return self;
-        // CPython apparently returns a "builtin_function_or_method" object
-        return boxInstanceMethod(inst, self);
-    }
-
-    static Box* __call__(BoxedMethodDescriptor* self, Box* obj, BoxedTuple* varargs, Box** _args) {
-        BoxedDict* kwargs = static_cast<BoxedDict*>(_args[0]);
-
-        assert(self->cls == method_cls);
-        assert(varargs->cls == tuple_cls);
-        assert(kwargs->cls == dict_cls);
-
-        if (!isSubclass(obj->cls, self->type))
-            raiseExcHelper(TypeError, "descriptor '%s' requires a '%s' object but received a '%s'",
-                           self->method->ml_name, getFullNameOfClass(self->type).c_str(), getFullTypeName(obj).c_str());
-
-        threading::GLPromoteRegion _gil_lock;
-
-        int ml_flags = self->method->ml_flags;
-        Box* rtn;
-        if (ml_flags == METH_NOARGS) {
-            assert(varargs->elts.size() == 0);
-            assert(kwargs->d.size() == 0);
-            rtn = (Box*)self->method->ml_meth(obj, NULL);
-        } else if (ml_flags == METH_VARARGS) {
-            assert(kwargs->d.size() == 0);
-            rtn = (Box*)self->method->ml_meth(obj, varargs);
-        } else if (ml_flags == (METH_VARARGS | METH_KEYWORDS)) {
-            rtn = (Box*)((PyCFunctionWithKeywords)self->method->ml_meth)(obj, varargs, kwargs);
-        } else if (ml_flags == METH_O) {
-            assert(kwargs->d.size() == 0);
-            assert(varargs->elts.size() == 1);
-            rtn = (Box*)self->method->ml_meth(obj, varargs->elts[0]);
-        } else {
-            RELEASE_ASSERT(0, "0x%x", ml_flags);
-        }
-        assert(rtn);
-        return rtn;
-    }
-};
 
 #define MAKE_CHECK(NAME, cls_name)                                                                                     \
     extern "C" bool Py##NAME##_Check(PyObject* op) { return isSubclass(op->cls, cls_name); }
@@ -102,10 +51,6 @@ int Py_Py3kWarningFlag;
 
 BoxedClass* capifunc_cls;
 
-extern "C" void conservativeGCHandler(GCVisitor* v, Box* b) {
-    v->visitPotentialRange((void* const*)b, (void* const*)((char*)b + b->cls->tp_basicsize));
-}
-
 extern "C" PyObject* PyType_GenericAlloc(PyTypeObject* cls, Py_ssize_t nitems) {
     RELEASE_ASSERT(nitems == 0, "unimplemented");
     RELEASE_ASSERT(cls->tp_itemsize == 0, "unimplemented");
@@ -118,48 +63,6 @@ extern "C" PyObject* PyType_GenericAlloc(PyTypeObject* cls, Py_ssize_t nitems) {
 }
 
 BoxedClass* wrapperdescr_cls, *wrapperobject_cls;
-struct wrapper_def {
-    const char* name;
-    int offset;
-    int flags;
-};
-
-wrapper_def call_wrapper = { "__call__", offsetof(PyTypeObject, tp_call), PyWrapperFlag_KEYWORDS };
-
-class BoxedWrapperDescriptor : public Box {
-public:
-    const wrapper_def* wrapper;
-    BoxedClass* type;
-    BoxedWrapperDescriptor(const wrapper_def* wrapper, BoxedClass* type)
-        : Box(wrapperdescr_cls), wrapper(wrapper), type(type) {}
-
-    static Box* __get__(BoxedWrapperDescriptor* self, Box* inst, Box* owner);
-};
-
-class BoxedWrapperObject : public Box {
-public:
-    BoxedWrapperDescriptor* descr;
-    Box* obj;
-
-    BoxedWrapperObject(BoxedWrapperDescriptor* descr, Box* obj) : Box(wrapperobject_cls), descr(descr), obj(obj) {}
-
-    static Box* __call__(BoxedWrapperObject* self, Box* args, Box* kwds) {
-        assert(self->cls == wrapperobject_cls);
-        assert(args->cls == tuple_cls);
-        assert(kwds->cls == dict_cls);
-
-        int flags = self->descr->wrapper->flags;
-        char* ptr = (char*)self->descr->type + self->descr->wrapper->offset;
-
-        if (flags & PyWrapperFlag_KEYWORDS) {
-            PyCFunctionWithKeywords f = *(PyCFunctionWithKeywords*)ptr;
-            return f(self->obj, args, kwds);
-        } else {
-            abort();
-        }
-        abort();
-    }
-};
 
 Box* BoxedWrapperDescriptor::__get__(BoxedWrapperDescriptor* self, Box* inst, Box* owner) {
     RELEASE_ASSERT(self->cls == wrapperdescr_cls, "");
@@ -174,128 +77,6 @@ Box* BoxedWrapperDescriptor::__get__(BoxedWrapperDescriptor* self, Box* inst, Bo
     return new BoxedWrapperObject(self, inst);
 }
 
-PyObject* tp_new_wrapper(PyTypeObject* self, BoxedTuple* args, Box* kwds) {
-    RELEASE_ASSERT(isSubclass(self->cls, type_cls), "");
-
-    // ASSERT(self->tp_new != Py_CallPythonNew, "going to get in an infinite loop");
-
-    RELEASE_ASSERT(args->cls == tuple_cls, "");
-    RELEASE_ASSERT(kwds->cls == dict_cls, "");
-    RELEASE_ASSERT(args->elts.size() >= 1, "");
-
-    BoxedClass* subtype = static_cast<BoxedClass*>(args->elts[0]);
-    RELEASE_ASSERT(isSubclass(subtype->cls, type_cls), "");
-    RELEASE_ASSERT(isSubclass(subtype, self), "");
-
-    BoxedTuple* new_args = new BoxedTuple(BoxedTuple::GCVector(args->elts.begin() + 1, args->elts.end()));
-
-    return self->tp_new(subtype, new_args, kwds);
-}
-
-extern "C" int PyType_Ready(PyTypeObject* cls) {
-    gc::registerNonheapRootObject(cls);
-
-    // unhandled fields:
-    RELEASE_ASSERT(cls->tp_print == NULL, "");
-    RELEASE_ASSERT(cls->tp_getattr == NULL, "");
-    RELEASE_ASSERT(cls->tp_setattr == NULL, "");
-    RELEASE_ASSERT(cls->tp_compare == NULL, "");
-    RELEASE_ASSERT(cls->tp_repr == NULL, "");
-    RELEASE_ASSERT(cls->tp_as_number == NULL, "");
-    RELEASE_ASSERT(cls->tp_as_sequence == NULL, "");
-    RELEASE_ASSERT(cls->tp_as_mapping == NULL, "");
-    RELEASE_ASSERT(cls->tp_hash == NULL, "");
-    RELEASE_ASSERT(cls->tp_str == NULL, "");
-    RELEASE_ASSERT(cls->tp_getattro == NULL || cls->tp_getattro == PyObject_GenericGetAttr, "");
-    RELEASE_ASSERT(cls->tp_setattro == NULL, "");
-    RELEASE_ASSERT(cls->tp_as_buffer == NULL, "");
-
-    int ALLOWABLE_FLAGS = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC;
-    RELEASE_ASSERT((cls->tp_flags & ~ALLOWABLE_FLAGS) == 0, "");
-    // RELEASE_ASSERT(cls->tp_traverse == NULL, "");
-    // RELEASE_ASSERT(cls->tp_clear == NULL, "");
-
-    RELEASE_ASSERT(cls->tp_richcompare == NULL, "");
-    RELEASE_ASSERT(cls->tp_iter == NULL, "");
-    RELEASE_ASSERT(cls->tp_iternext == NULL, "");
-    RELEASE_ASSERT(cls->tp_base == NULL, "");
-    RELEASE_ASSERT(cls->tp_dict == NULL, "");
-    RELEASE_ASSERT(cls->tp_descr_get == NULL, "");
-    RELEASE_ASSERT(cls->tp_descr_set == NULL, "");
-    RELEASE_ASSERT(cls->tp_init == NULL, "");
-    RELEASE_ASSERT(cls->tp_alloc == NULL, "");
-    RELEASE_ASSERT(cls->tp_free == NULL || cls->tp_free == PyObject_Del, "");
-    RELEASE_ASSERT(cls->tp_is_gc == NULL, "");
-    RELEASE_ASSERT(cls->tp_base == NULL, "");
-    RELEASE_ASSERT(cls->tp_mro == NULL, "");
-    RELEASE_ASSERT(cls->tp_cache == NULL, "");
-    RELEASE_ASSERT(cls->tp_subclasses == NULL, "");
-    RELEASE_ASSERT(cls->tp_weaklist == NULL, "");
-    RELEASE_ASSERT(cls->tp_del == NULL, "");
-    RELEASE_ASSERT(cls->tp_version_tag == 0, "");
-
-// I think it is safe to ignore tp_weaklistoffset for now:
-// RELEASE_ASSERT(cls->tp_weaklistoffset == 0, "");
-
-#define INITIALIZE(a) new (&(a)) decltype(a)
-    INITIALIZE(cls->attrs);
-    INITIALIZE(cls->dependent_icgetattrs);
-#undef INITIALIZE
-
-    BoxedClass* base = cls->base = object_cls;
-    if (!cls->cls)
-        cls->cls = cls->base->cls;
-
-    assert(cls->tp_name);
-    cls->giveAttr("__name__", boxStrConstant(cls->tp_name));
-    // tp_name
-    // tp_basicsize, tp_itemsize
-    // tp_doc
-
-    if (!cls->tp_new && base != object_cls)
-        cls->tp_new = base->tp_new;
-
-    if (cls->tp_new) {
-        cls->giveAttr("__new__",
-                      new BoxedCApiFunction(METH_VARARGS | METH_KEYWORDS, cls, "__new__", (PyCFunction)tp_new_wrapper));
-    }
-
-    if (cls->tp_call) {
-        cls->giveAttr("__call__", new BoxedWrapperDescriptor(&call_wrapper, cls));
-    }
-
-    if (!cls->tp_alloc) {
-        cls->tp_alloc = reinterpret_cast<decltype(cls->tp_alloc)>(PyType_GenericAlloc);
-    }
-
-    for (PyMethodDef* method = cls->tp_methods; method && method->ml_name; ++method) {
-        cls->giveAttr(method->ml_name, new BoxedMethodDescriptor(method, cls));
-    }
-
-    for (PyMemberDef* member = cls->tp_members; member && member->name; ++member) {
-        cls->giveAttr(member->name, new BoxedMemberDescriptor(member));
-    }
-
-    if (cls->tp_getset) {
-        if (VERBOSITY())
-            printf("warning: ignoring tp_getset for now\n");
-    }
-
-    cls->gc_visit = &conservativeGCHandler;
-
-    // TODO not sure how we can handle extension types that manually
-    // specify a dict...
-    RELEASE_ASSERT(cls->tp_dictoffset == 0, "");
-    // this should get automatically initialized to 0 on this path:
-    assert(cls->attrs_offset == 0);
-
-    return 0;
-}
-
-extern "C" int PyType_IsSubtype(PyTypeObject*, PyTypeObject*) {
-    Py_FatalError("unimplemented");
-}
-
 // copied from CPython's getargs.c:
 extern "C" int PyBuffer_FillInfo(Py_buffer* view, PyObject* obj, void* buf, Py_ssize_t len, int readonly, int flags) {
     if (view == NULL)
@@ -473,8 +254,12 @@ extern "C" PyObject* PyObject_GetIter(PyObject*) {
     Py_FatalError("unimplemented");
 }
 
-extern "C" PyObject* PyObject_Repr(PyObject*) {
-    Py_FatalError("unimplemented");
+extern "C" PyObject* PyObject_Repr(PyObject* obj) {
+    try {
+        return repr(obj);
+    } catch (Box* b) {
+        Py_FatalError("unimplemented");
+    }
 }
 
 extern "C" PyObject* PyObject_GetAttr(PyObject* o, PyObject* attr_name) {
@@ -551,6 +336,10 @@ extern "C" int PyObject_GetBuffer(PyObject* exporter, Py_buffer* view, int flags
     Py_FatalError("unimplemented");
 }
 
+extern "C" int PyObject_Print(PyObject* obj, FILE* fp, int flags) {
+    Py_FatalError("unimplemented");
+};
+
 extern "C" int PySequence_Check(PyObject*) {
     Py_FatalError("unimplemented");
 }

src/runtime/import.cpp

@@ -143,12 +143,12 @@ static Box* importSub(const std::string& name, const std::string& full_name, Box
         }
     }
 
-    if (name == "basic_test") {
+    if (name == "basic_test")
         return importTestExtension("basic_test");
-    }
-    if (name == "descr_test") {
+    if (name == "descr_test")
         return importTestExtension("descr_test");
-    }
+    if (name == "slots_test")
+        return importTestExtension("slots_test");
 
     return NULL;
 }

src/runtime/objmodel.cpp

@@ -23,6 +23,7 @@
 
 #include "asm_writing/icinfo.h"
 #include "asm_writing/rewriter.h"
+#include "capi/typeobject.h"
 #include "codegen/ast_interpreter.h"
 #include "codegen/codegen.h"
 #include "codegen/compvars.h"
@@ -332,45 +333,11 @@ extern "C" Box** unpackIntoArray(Box* obj, int64_t expected_size) {
     return &elts[0];
 }
 
-PyObject* Py_CallPythonNew(PyTypeObject* self, PyObject* args, PyObject* kwds) {
-    try {
-        Py_FatalError("this function is untested");
-
-        Box* new_attr = typeLookup(self, _new_str, NULL);
-        assert(new_attr);
-        new_attr = processDescriptor(new_attr, None, self);
-        return runtimeCallInternal(new_attr, NULL, ArgPassSpec(1, 0, true, true), self, args, kwds, NULL, NULL);
-    } catch (Box* e) {
-        abort();
-    }
-}
-
-PyObject* Py_CallPythonCall(PyObject* self, PyObject* args, PyObject* kwds) {
-    try {
-        Py_FatalError("this function is untested");
-
-        return runtimeCallInternal(self, NULL, ArgPassSpec(0, 0, true, true), args, kwds, NULL, NULL, NULL);
-    } catch (Box* e) {
-        abort();
-    }
-}
-
 void BoxedClass::freeze() {
     assert(!is_constant);
     assert(getattr("__name__")); // otherwise debugging will be very hard
 
-    // This will probably share a lot in common with Py_TypeReady:
-    if (!tp_new) {
-        this->tp_new = &Py_CallPythonNew;
-    } else if (tp_new != Py_CallPythonNew) {
-        ASSERT(0, "need to set __new__?");
-    }
-
-    if (!tp_call) {
-        this->tp_call = &Py_CallPythonCall;
-    } else if (tp_call != Py_CallPythonCall) {
-        ASSERT(0, "need to set __call__?");
-    }
+    fixup_slot_dispatchers(this);
 
     is_constant = true;
 }
@@ -1485,19 +1452,9 @@ void setattrInternal(Box* obj, const std::string& attr, Box* val, SetattrRewrite
         if (attr == "__base__" && self->getattr("__base__"))
             raiseExcHelper(TypeError, "readonly attribute");
 
-        if (attr == "__new__") {
-            self->tp_new = &Py_CallPythonNew;
-            // TODO update subclasses
-
+        bool touched_slot = update_slot(self, attr);
+        if (touched_slot)
             rewrite_args = NULL;
-        }
-
-        if (attr == "__call__") {
-            self->tp_call = &Py_CallPythonCall;
-            // TODO update subclasses
-
-            rewrite_args = NULL;
-        }
     }
 
     Box* _set_ = NULL;
@@ -3332,10 +3289,10 @@ Box* typeNew(Box* _cls, Box* arg1, Box* arg2, Box** _args) {
     // Note: make sure to do this after assigning the attrs, since it will overwrite any defined __name__
     made->setattr("__name__", name, NULL);
 
-    // TODO this function (typeNew) should probably call PyType_Ready
-
+    // TODO should this function (typeNew) call PyType_Ready?
     made->tp_new = base->tp_new;
     made->tp_alloc = reinterpret_cast<decltype(cls->tp_alloc)>(PyType_GenericAlloc);
+    fixup_slot_dispatchers(made);
 
     return made;
 }

test/test_extension/setup.py

@@ -6,5 +6,6 @@ setup(name="test",
         ext_modules=[
             Extension("basic_test", sources = ["basic_test.c"]),
             Extension("descr_test", sources = ["descr_test.c"]),
+            Extension("slots_test", sources = ["slots_test.c"]),
         ],
     )

test/test_extension/slots_test.c

+#include <Python.h>
+
+typedef struct {
+    PyObject_HEAD
+
+    int n;
+} slots_tester_object;
+
+static PyObject *
+slots_tester_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+    slots_tester_object* obj;
+
+    if (!_PyArg_NoKeywords("attrgetter()", kwds))
+        return NULL;
+
+    int n;
+    if (!PyArg_ParseTuple(args, "n", &n))
+        return NULL;
+
+    /* create attrgetterobject structure */
+    obj = PyObject_New(slots_tester_object, type);
+    if (obj == NULL)
+        return NULL;
+
+    obj->n = n;
+
+    return (PyObject *)obj;
+}
+
+static PyObject *
+slots_tester_repr(slots_tester_object *obj)
+{
+    char buf[80];
+    snprintf(buf, sizeof(buf), "<my custom repr: %d>", obj->n);
+    return PyString_FromString(buf);
+}
+
+static PyObject *
+slots_tester_call(slots_tester_object *obj, PyObject *args, PyObject *kw)
+{
+    if (!PyArg_ParseTuple(args, ""))
+        return NULL;
+
+    return PyInt_FromLong(obj->n);
+}
+
+PyDoc_STRVAR(slots_tester_doc, "slots_tester doc");
+
+static PyTypeObject slots_tester = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "slots_test.slots_tester",            /* tp_name */
+    sizeof(slots_tester_object),          /* tp_basicsize */
+    0,                                  /* tp_itemsize */
+    /* methods */
+    0,                                  /* tp_dealloc */
+    0,                                  /* tp_print */
+    0,                                  /* tp_getattr */
+    0,                                  /* tp_setattr */
+    0,                                  /* tp_compare */
+    (reprfunc)slots_tester_repr,        /* tp_repr */
+    0,                                  /* tp_as_number */
+    0,                                  /* tp_as_sequence */
+    0,                                  /* tp_as_mapping */
+    0,                                  /* tp_hash */
+    (ternaryfunc)slots_tester_call,     /* tp_call */
+    0,                                  /* tp_str */
+    0,                                  /* tp_getattro */
+    0,                                  /* tp_setattro */
+    0,                                  /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,                 /* tp_flags */
+    slots_tester_doc,                   /* tp_doc */
+    0,                                  /* tp_traverse */
+    0,                                  /* tp_clear */
+    0,                                  /* tp_richcompare */
+    0,                                  /* tp_weaklistoffset */
+    0,                                  /* tp_iter */
+    0,                                  /* tp_iternext */
+    0,                                  /* tp_methods */
+    0,                                  /* tp_members */
+    0,                                  /* tp_getset */
+    0,                                  /* tp_base */
+    0,                                  /* tp_dict */
+    0,                                  /* tp_descr_get */
+    0,                                  /* tp_descr_set */
+    0,                                  /* tp_dictoffset */
+    0,                                  /* tp_init */
+    0,                                  /* tp_alloc */
+    slots_tester_new,                   /* tp_new */
+    0,                                  /* tp_free */
+};
+
+// Tests the correctness of the CAPI slots when the attributes get set in Python code:
+static PyObject *
+call_funcs(PyObject* _module, PyObject* args) {
+    PyObject* obj;
+    if (!PyArg_ParseTuple(args, "O", &obj))
+        return NULL;
+
+    printf("\n");
+
+    PyTypeObject* cls = Py_TYPE(obj);
+    printf("Received a %s object\n", cls->tp_name);
+
+    if (cls->tp_repr) {
+        PyObject* rtn = cls->tp_repr(obj);
+        printf("tp_repr exists and returned: '%s'\n", PyString_AsString(rtn));
+        Py_DECREF(rtn);
+    }
+
+    if (cls->tp_new) {
+        PyObject* rtn = cls->tp_new(cls, PyTuple_New(0), PyDict_New());
+        printf("tp_new exists and returned an object of type: '%s'\n", Py_TYPE(rtn)->tp_name);
+        Py_DECREF(rtn);
+    }
+
+    if (cls->tp_call) {
+        printf("tp_call exists\n");
+    } else {
+        printf("tp_call doesnt exist\n");
+    }
+
+    Py_DECREF(obj);
+
+    Py_RETURN_NONE;
+}
+
+static PyMethodDef SlotsMethods[] = {
+    {"call_funcs", call_funcs, METH_VARARGS, "Call slotted functions."},
+    {NULL, NULL, 0, NULL}        /* Sentinel */
+};
+
+PyMODINIT_FUNC
+initslots_test(void)
+{
+    PyObject *m;
+
+    m = Py_InitModule("slots_test", SlotsMethods);
+    if (m == NULL)
+        return;
+
+    int res = PyType_Ready(&slots_tester);
+    if (res < 0)
+        return;
+
+    // Not sure if the result of PyInt_FromLong needs to be decref'd
+    PyDict_SetItemString(slots_tester.tp_dict, "set_through_tpdict", PyInt_FromLong(123));
+
+    PyModule_AddObject(m, "SlotsTester", (PyObject *)&slots_tester);
+}

test/tests/capi_slots.py

+import slots_test
+
+for i in xrange(3):
+    t = slots_test.SlotsTester(i + 5)
+    print t, repr(t), t()
+
+print slots_test.SlotsTester.set_through_tpdict, slots_test.SlotsTester(5).set_through_tpdict
+
+class C(object):
+    def __repr__(self):
+        print "__repr__()"
+        return "<C object>"
+
+slots_test.call_funcs(C())
+
+# Test to make sure that updating an existing class also updates the tp_* slots:
+def repr2(self):
+    return "repr2()"
+C.repr2 = repr2
+slots_test.call_funcs(C())

tools/tester.py

@@ -52,6 +52,21 @@ def set_ulimits():
     MAX_MEM_MB = 100
     resource.setrlimit(resource.RLIMIT_RSS, (MAX_MEM_MB * 1024 * 1024, MAX_MEM_MB * 1024 * 1024))
 
+EXTMODULE_DIR = os.path.dirname(os.path.realpath(__file__)) + "/../test/test_extension/build/lib.linux-x86_64-2.7/"
+_extmodule_mtime = None
+def get_extmodule_mtime():
+    global _extmodule_mtime
+    if _extmodule_mtime is not None:
+        return _extmodule_mtime
+
+    rtn = 0
+    for fn in os.listdir(EXTMODULE_DIR):
+        if not fn.endswith(".so"):
+            continue
+        rtn = max(rtn, os.stat(os.path.join(EXTMODULE_DIR, fn)).st_mtime)
+    _extmodule_mtime = rtn
+    return rtn
+
 def get_expected_output(fn):
     sys.stdout.flush()
     assert fn.endswith(".py")
@@ -61,7 +76,8 @@ def get_expected_output(fn):
 
     cache_fn = fn[:-3] + ".expected_cache"
     if os.path.exists(cache_fn):
-        if os.stat(cache_fn).st_mtime > os.stat(fn).st_mtime:
+        cache_mtime = os.stat(cache_fn).st_mtime
+        if cache_mtime > os.stat(fn).st_mtime and cache_mtime > get_extmodule_mtime():
             try:
                 return cPickle.load(open(cache_fn))
             except EOFError:
@@ -69,7 +85,7 @@ def get_expected_output(fn):
 
     # TODO don't suppress warnings globally:
     env = dict(os.environ)
-    env["PYTHONPATH"] = os.path.dirname(os.path.realpath(__file__)) + "/../test/test_extension/build/lib.linux-x86_64-2.7/"
+    env["PYTHONPATH"] = EXTMODULE_DIR
     p = subprocess.Popen(["python", "-Wignore", fn], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=open("/dev/null"), preexec_fn=set_ulimits, env=env)
     out, err = p.communicate()
     code = p.wait()
@@ -208,7 +224,7 @@ def run_test(fn, check_stats, run_memcheck):
             out_fd, out_fn = tempfile.mkstemp(prefix="received_")
             os.fdopen(exp_fd, 'w').write(expected_out)
             os.fdopen(out_fd, 'w').write(out)
-            p = subprocess.Popen(["diff", "-C2", "-a", exp_fn, out_fn], stdout=subprocess.PIPE, preexec_fn=set_ulimits)
+            p = subprocess.Popen(["diff", "-u", "-a", exp_fn, out_fn], stdout=subprocess.PIPE, preexec_fn=set_ulimits)
             diff = p.stdout.read()
             assert p.wait() in (0, 1)
             os.unlink(exp_fn)