Add tp_subclasses so we can update slots on subclasses

ie if we set __getattribute__ on a superclass, we have to update
tp_getattro on all subclasses.  We've always been broken on this, but
this didn't really matter as much until now since we would internally
check __getattribute__ and not tp_getattro.

tp_subclasses is a list of weakrefs, which means that we need weakref
support before we can start finalizing classes.  which means we have to
add a whole slew of things into the bootstrapping section.

We should probably just do this in a more systematic way, but I'm
not sure what that would be...

src/capi/typeobject.cpp

@@ -25,6 +25,7 @@ namespace pyston {
 static const std::string _new_str("__new__");
 static const std::string _getattr_str("__getattr__");
 static const std::string _getattribute_str("__getattribute__");
+typedef int (*update_callback)(PyTypeObject*, void*);
 
 extern "C" void conservativeGCHandler(GCVisitor* v, Box* b) noexcept {
     v->visitPotentialRange((void* const*)b, (void* const*)((char*)b + b->cls->tp_basicsize));
@@ -1496,18 +1497,55 @@ static const slotdef* update_one_slot(BoxedClass* type, const slotdef* p) noexce
     return p;
 }
 
-bool update_slot(BoxedClass* self, const std::string& attr) noexcept {
-    bool updated = false;
-    for (const slotdef& p : slotdefs) {
-        if (!p.name)
-            continue;
-        if (p.name == attr) {
-            // TODO update subclasses;
-            update_one_slot(self, &p);
-            updated = true;
-        }
+/* In the type, update the slots whose slotdefs are gathered in the pp array.
+   This is a callback for update_subclasses(). */
+static int update_slots_callback(PyTypeObject* type, void* data) noexcept {
+    slotdef** pp = (slotdef**)data;
+
+    for (; *pp; pp++)
+        update_one_slot(type, *pp);
+    return 0;
+}
+
+static int update_subclasses(PyTypeObject* type, PyObject* name, update_callback callback, void* data) noexcept;
+static int recurse_down_subclasses(PyTypeObject* type, PyObject* name, update_callback callback, void* data) noexcept;
+
+bool update_slot(BoxedClass* type, const std::string& attr) noexcept {
+    slotdef* ptrs[MAX_EQUIV];
+    slotdef* p;
+    slotdef** pp;
+    int offset;
+
+    /* Clear the VALID_VERSION flag of 'type' and all its
+       subclasses.  This could possibly be unified with the
+       update_subclasses() recursion below, but carefully:
+       they each have their own conditions on which to stop
+       recursing into subclasses. */
+    PyType_Modified(type);
+
+    init_slotdefs();
+    pp = ptrs;
+    for (p = slotdefs; p->name; p++) {
+        /* XXX assume name is interned! */
+        if (p->name == attr)
+            *pp++ = p;
     }
-    return updated;
+    *pp = NULL;
+    for (pp = ptrs; *pp; pp++) {
+        p = *pp;
+        offset = p->offset;
+        while (p > slotdefs && (p - 1)->offset == offset)
+            --p;
+        *pp = p;
+    }
+    if (ptrs[0] == NULL)
+        return false; /* Not an attribute that affects any slots */
+    int r = update_subclasses(type, new BoxedString(attr), update_slots_callback, (void*)ptrs);
+
+    // TODO this is supposed to be a CAPI function!
+    if (r)
+        throwCAPIException();
+    return true;
 }
 
 void fixup_slot_dispatchers(BoxedClass* self) noexcept {
@@ -1518,6 +1556,40 @@ void fixup_slot_dispatchers(BoxedClass* self) noexcept {
         p = update_one_slot(self, p);
 }
 
+static int update_subclasses(PyTypeObject* type, PyObject* name, update_callback callback, void* data) noexcept {
+    if (callback(type, data) < 0)
+        return -1;
+    return recurse_down_subclasses(type, name, callback, data);
+}
+
+static int recurse_down_subclasses(PyTypeObject* type, PyObject* name, update_callback callback, void* data) noexcept {
+    PyTypeObject* subclass;
+    PyObject* ref, *subclasses, *dict;
+    Py_ssize_t i, n;
+
+    subclasses = type->tp_subclasses;
+    if (subclasses == NULL)
+        return 0;
+    assert(PyList_Check(subclasses));
+    n = PyList_GET_SIZE(subclasses);
+    for (i = 0; i < n; i++) {
+        ref = PyList_GET_ITEM(subclasses, i);
+        assert(PyWeakref_CheckRef(ref));
+        subclass = (PyTypeObject*)PyWeakref_GET_OBJECT(ref);
+        assert(subclass != NULL);
+        if ((PyObject*)subclass == Py_None)
+            continue;
+        assert(PyType_Check(subclass));
+        /* Avoid recursing down into unaffected classes */
+        dict = subclass->tp_dict;
+        if (dict != NULL && PyDict_Check(dict) && PyDict_GetItem(dict, name) != NULL)
+            continue;
+        if (update_subclasses(subclass, name, callback, data) < 0)
+            return -1;
+    }
+    return 0;
+}
+
 static PyObject* tp_new_wrapper(PyTypeObject* self, BoxedTuple* args, Box* kwds) noexcept {
     RELEASE_ASSERT(isSubclass(self->cls, type_cls), "");
 
@@ -2370,6 +2442,52 @@ static void inherit_slots(PyTypeObject* type, PyTypeObject* base) noexcept {
     }
 }
 
+static int add_subclass(PyTypeObject* base, PyTypeObject* type) noexcept {
+    Py_ssize_t i;
+    int result;
+    PyObject* list, *ref, *newobj;
+
+    list = base->tp_subclasses;
+    if (list == NULL) {
+        base->tp_subclasses = list = PyList_New(0);
+        if (list == NULL)
+            return -1;
+    }
+    assert(PyList_Check(list));
+    newobj = PyWeakref_NewRef((PyObject*)type, NULL);
+    i = PyList_GET_SIZE(list);
+    while (--i >= 0) {
+        ref = PyList_GET_ITEM(list, i);
+        assert(PyWeakref_CheckRef(ref));
+        if (PyWeakref_GET_OBJECT(ref) == Py_None)
+            return PyList_SetItem(list, i, newobj);
+    }
+    result = PyList_Append(list, newobj);
+    Py_DECREF(newobj);
+    return result;
+}
+
+static void remove_subclass(PyTypeObject* base, PyTypeObject* type) noexcept {
+    Py_ssize_t i;
+    PyObject* list, *ref;
+
+    list = base->tp_subclasses;
+    if (list == NULL) {
+        return;
+    }
+    assert(PyList_Check(list));
+    i = PyList_GET_SIZE(list);
+    while (--i >= 0) {
+        ref = PyList_GET_ITEM(list, i);
+        assert(PyWeakref_CheckRef(ref));
+        if (PyWeakref_GET_OBJECT(ref) == (PyObject*)type) {
+            /* this can't fail, right? */
+            PySequence_DelItem(list, i);
+            return;
+        }
+    }
+}
+
 // commonClassSetup is for the common code between PyType_Ready (which is just for extension classes)
 // and our internal type-creation endpoints (BoxedClass::BoxedClass()).
 // TODO: Move more of the duplicated logic into here.
@@ -2381,6 +2499,12 @@ void commonClassSetup(BoxedClass* cls) {
             cls->tp_bases = new BoxedTuple({});
     }
 
+    /* Link into each base class's list of subclasses */
+    for (PyObject* b : static_cast<BoxedTuple*>(cls->tp_bases)->elts) {
+        if (PyType_Check(b) && add_subclass((PyTypeObject*)b, cls) < 0)
+            throwCAPIException();
+    }
+
     /* Calculate method resolution order */
     if (mro_internal(cls) < 0)
         throwCAPIException();

src/runtime/bool.cpp

@@ -93,12 +93,6 @@ void setupBool() {
     bool_cls->giveAttr("__xor__", new BoxedFunction(boxRTFunction((void*)boolXor, BOXED_BOOL, 2)));
 
     bool_cls->freeze();
-
-    True = new BoxedBool(true);
-    False = new BoxedBool(false);
-
-    gc::registerPermanentRoot(True);
-    gc::registerPermanentRoot(False);
 }
 
 void teardownBool() {

src/runtime/builtin_modules/sys.cpp

@@ -194,6 +194,7 @@ public:
 
     BoxedSysFlags() {
         auto zero = boxInt(0);
+        assert(zero);
         division_warning = zero;
         bytes_warning = zero;
         no_user_site = zero;
@@ -297,8 +298,8 @@ void setupSys() {
 
     sys_module->giveAttr("maxint", boxInt(PYSTON_INT_MAX));
 
-    sys_flags_cls = BoxedHeapClass::create(type_cls, object_cls, BoxedSysFlags::gcHandler, 0, 0, sizeof(BoxedSysFlags),
-                                           false, "flags");
+    sys_flags_cls = new BoxedHeapClass(object_cls, BoxedSysFlags::gcHandler, 0, 0, sizeof(BoxedSysFlags), false,
+                                       new BoxedString("flags"));
     sys_flags_cls->giveAttr("__new__",
                             new BoxedFunction(boxRTFunction((void*)BoxedSysFlags::__new__, UNKNOWN, 1, 0, true, true)));
 #define ADD(name)                                                                                                      \
@@ -309,6 +310,7 @@ void setupSys() {
     ADD(no_user_site);
 #undef ADD
 
+    sys_flags_cls->tp_mro = new BoxedTuple({ sys_flags_cls, object_cls });
     sys_flags_cls->freeze();
 
     sys_module->giveAttr("flags", new BoxedSysFlags());
@@ -324,5 +326,6 @@ void setupSysEnd() {
                                                               PyLt());
 
     sys_module->giveAttr("builtin_module_names", new BoxedTuple(std::move(builtin_module_names)));
+    sys_flags_cls->finishInitialization();
 }
 }

src/runtime/capi.cpp

@@ -1291,9 +1291,6 @@ static Box* methodGetDoc(Box* b, void*) {
 }
 
 void setupCAPI() {
-    capifunc_cls
-        = BoxedHeapClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(BoxedCApiFunction), false, "capifunc");
-
     capifunc_cls->giveAttr("__repr__",
                            new BoxedFunction(boxRTFunction((void*)BoxedCApiFunction::__repr__, UNKNOWN, 1)));
 
@@ -1303,8 +1300,6 @@ void setupCAPI() {
 
     capifunc_cls->freeze();
 
-    method_cls
-        = BoxedHeapClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(BoxedMethodDescriptor), false, "method");
     method_cls->giveAttr("__get__",
                          new BoxedFunction(boxRTFunction((void*)BoxedMethodDescriptor::__get__, UNKNOWN, 3)));
     method_cls->giveAttr("__call__", new BoxedFunction(boxRTFunction((void*)BoxedMethodDescriptor::__call__, UNKNOWN, 2,
@@ -1318,8 +1313,6 @@ void setupCAPI() {
                                                                            UNKNOWN, 2, 0, true, true)));
     wrapperdescr_cls->freeze();
 
-    wrapperobject_cls
-        = BoxedHeapClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(BoxedWrapperObject), false, "method-wrapper");
     wrapperobject_cls->giveAttr(
         "__call__", new BoxedFunction(boxRTFunction((void*)BoxedWrapperObject::__call__, UNKNOWN, 1, 0, true, true)));
     wrapperobject_cls->freeze();

src/runtime/stacktrace.cpp

@@ -144,7 +144,11 @@ extern "C" void abort() {
         // that object, _printStackTrace will hang waiting for the first construction
         // to finish.)
         alarm(1);
+        try {
             _printStacktrace();
+        } catch (ExcInfo) {
+            fprintf(stderr, "error printing stack trace during abort()");
+        }
 
         // Cancel the alarm.
         // This is helpful for when running in a debugger, since the debugger will catch the

src/runtime/types.h

@@ -204,6 +204,7 @@ protected:
                bool is_user_defined);
 
     friend void setupRuntime();
+    friend void setupSysEnd();
 };
 
 class BoxedHeapClass : public BoxedClass {
@@ -232,6 +233,7 @@ private:
                    bool is_user_defined, BoxedString* name);
 
     friend void setupRuntime();
+    friend void setupSys();
 
     DEFAULT_CLASS(type_cls);
 };