Fix a bunch of destructor stuff

First, our implementation of slot_tp_del (what calls Python-level __del__ methods)
was pretty bad -- it didn't re-incref the object to keep it alive.

Then we had some issues around running destructors at shutdown.  I made this section
look a bit more like CPython's, but there are still differences.

from_cpython/Include/Python.h

@@ -136,6 +136,9 @@ PyAPI_FUNC(void) PyType_SetDict(PyTypeObject*, PyObject*) PYSTON_NOEXCEPT;
 // PyType_Ready calls this automatically.
 PyAPI_FUNC(PyObject*) PyGC_RegisterStaticConstant(PyObject*) PYSTON_NOEXCEPT;
 
+// Gets gc.garbage
+PyAPI_FUNC(PyObject*) _PyGC_GetGarbage() PYSTON_NOEXCEPT;
+
 // Pyston addition:
 PyAPI_FUNC(void) PyGC_Enable() PYSTON_NOEXCEPT;
 PyAPI_FUNC(void) PyGC_Disable() PYSTON_NOEXCEPT;

from_cpython/Modules/gcmodule.c

@@ -777,6 +777,15 @@ debug_cycle(char *msg, PyObject *op)
     }
 }
 
+PyObject* _PyGC_GetGarbage() {
+    if (garbage == NULL) {
+        garbage = PyList_New(0);
+        if (garbage == NULL)
+            Py_FatalError("gc couldn't create gc.garbage list");
+    }
+    return garbage;
+}
+
 /* Handle uncollectable garbage (cycles with finalizers, and stuff reachable
  * only from such cycles).
  * If DEBUG_SAVEALL, all objects in finalizers are appended to the module
@@ -791,12 +800,9 @@ handle_finalizers(PyGC_Head *finalizers, PyGC_Head *old)
 {
     PyGC_Head *gc = finalizers->gc.gc_next;
 
-    if (garbage == NULL) {
-        garbage = PyList_New(0);
-        if (garbage == NULL)
-            Py_FatalError("gc couldn't create gc.garbage list");
-        PyGC_RegisterStaticConstant(garbage);
-    }
+    if (garbage == NULL)
+        _PyGC_GetGarbage();
+
     for (; gc != finalizers; gc = gc->gc.gc_next) {
         PyObject *op = FROM_GC(gc);
 

from_cpython/Python/import.c

@@ -441,15 +441,16 @@ static char* sys_files[] = {
 
 /* Un-initialize things, as good as we can */
 // Pyston change: we don't support calling cleanup currently
-#if 0
 void
 PyImport_Cleanup(void)
 {
     Py_ssize_t pos, ndone;
     char *name;
     PyObject *key, *value, *dict;
-    PyInterpreterState *interp = PyThreadState_GET()->interp;
-    PyObject *modules = interp->modules;
+    // Pyston change:
+    //PyInterpreterState *interp = PyThreadState_GET()->interp;
+    //PyObject *modules = interp->modules;
+    PyObject *modules = PySys_GetModulesDict();
 
     if (modules == NULL)
         return; /* Already done */
@@ -568,12 +569,14 @@ PyImport_Cleanup(void)
 
     /* Finally, clear and delete the modules directory */
     PyDict_Clear(modules);
-    interp->modules = NULL;
-    Py_DECREF(modules);
-    Py_CLEAR(interp->modules_reloading);
+    // Pyston change:
+    //interp->modules = NULL;
+    //Py_DECREF(modules);
+    //Py_CLEAR(interp->modules_reloading);
 }
 
 
+#if 0
 /* Helper for pythonrun.c -- return magic number */
 
 long

src/capi/typeobject.cpp

@@ -1117,24 +1117,61 @@ template Box* slotTpGetattrHookInternal<CXX, NOT_REWRITABLE>(Box* self, BoxedStr
     }
 }
 
-static PyObject* slot_tp_del(PyObject* self) noexcept {
-    static BoxedString* del_str = getStaticString("__del__");
-    try {
-        // TODO: runtime ICs?
-        Box* del_attr = typeLookup(self->cls, del_str);
-        assert(del_attr);
-
-        CallattrFlags flags{.cls_only = false,
-                            .null_on_nonexistent = true,
-                            .argspec = ArgPassSpec(0, 0, false, false) };
-        return callattr(self, del_str, flags, NULL, NULL, NULL, NULL, NULL);
-    } catch (ExcInfo e) {
-        // Python does not support exceptions thrown inside finalizers. Instead, it just
-        // prints a warning that an exception was throw to stderr but ignores it.
-        setCAPIException(e);
-        PyErr_WriteUnraisable(self);
-        return NULL;
+static void slot_tp_del(PyObject* self) noexcept {
+    static PyObject* del_str = NULL;
+    PyObject* del, *res;
+    PyObject* error_type, *error_value, *error_traceback;
+
+    /* Temporarily resurrect the object. */
+    assert(self->ob_refcnt == 0);
+    self->ob_refcnt = 1;
+
+    /* Save the current exception, if any. */
+    PyErr_Fetch(&error_type, &error_value, &error_traceback);
+
+    /* Execute __del__ method, if any. */
+    del = lookup_maybe(self, "__del__", &del_str);
+    if (del != NULL) {
+        res = PyEval_CallObject(del, NULL);
+        if (res == NULL)
+            PyErr_WriteUnraisable(del);
+        else
+            Py_DECREF(res);
+        Py_DECREF(del);
     }
+
+    /* Restore the saved exception. */
+    PyErr_Restore(error_type, error_value, error_traceback);
+
+    /* Undo the temporary resurrection; can't use DECREF here, it would
+     * cause a recursive call.
+     */
+    assert(self->ob_refcnt > 0);
+    if (--self->ob_refcnt == 0)
+        return; /* this is the normal path out */
+
+    /* __del__ resurrected it!  Make it look like the original Py_DECREF
+     * never happened.
+     */
+    {
+        Py_ssize_t refcnt = self->ob_refcnt;
+        _Py_NewReference(self);
+        self->ob_refcnt = refcnt;
+    }
+    assert(!PyType_IS_GC(Py_TYPE(self)) || _Py_AS_GC(self)->gc.gc_refs != _PyGC_REFS_UNTRACKED);
+    /* If Py_REF_DEBUG, _Py_NewReference bumped _Py_RefTotal, so
+     * we need to undo that. */
+    _Py_DEC_REFTOTAL;
+/* If Py_TRACE_REFS, _Py_NewReference re-added self to the object
+ * chain, so no more to do there.
+ * If COUNT_ALLOCS, the original decref bumped tp_frees, and
+ * _Py_NewReference bumped tp_allocs:  both of those need to be
+ * undone.
+ */
+#ifdef COUNT_ALLOCS
+    --Py_TYPE(self)->tp_frees;
+    --Py_TYPE(self)->tp_allocs;
+#endif
 }
 
 /* Pyston change: static */ int slot_tp_init(PyObject* self, PyObject* args, PyObject* kwds) noexcept {

src/core/types.h

@@ -650,6 +650,7 @@ public:
 
     int traverse(visitproc visit, void* arg) noexcept;
     void clear() noexcept;
+    void moduleClear() noexcept; // slightly different order of clearing attributes, meant for modules
 };
 static_assert(sizeof(HCAttrs) == sizeof(struct _hcattrs), "");
 

src/jit.cpp

@@ -189,7 +189,8 @@ int handleArg(char code) {
     else if (code == 'q')
         GLOBAL_VERBOSITY = 0;
     else if (code == 'v') {
-        Py_VerboseFlag++;
+        if (GLOBAL_VERBOSITY)
+            Py_VerboseFlag++;
         GLOBAL_VERBOSITY++;
     } else if (code == 'd')
         SHOW_DISASM = true;

src/runtime/builtin_modules/sys.cpp

@@ -675,7 +675,7 @@ void setupSys() {
 
     // sys_module is what holds on to all of the other modules:
     Py_INCREF(sys_module);
-    constants.push_back(sys_module);
+    late_constants.push_back(sys_module);
 
     sys_module->giveAttrBorrowed("modules", sys_modules_dict);
 

src/runtime/dict.cpp

@@ -310,18 +310,10 @@ extern "C" int PyDict_SetItem(PyObject* mp, PyObject* _key, PyObject* _item) noe
         return 0;
     }
 
-    ASSERT(PyDict_Check(mp) || mp->cls == attrwrapper_cls, "%s", getTypeName(mp));
-
-    assert(mp);
-    Box* b = static_cast<Box*>(mp);
-    Box* key = static_cast<Box*>(_key);
-    Box* item = static_cast<Box*>(_item);
-
-    assert(key);
-    assert(item);
+    ASSERT(mp->cls == attrwrapper_cls, "%s", getTypeName(mp));
 
     try {
-        setitem(b, key, item);
+        attrwrapperSet(mp, _key, _item);
     } catch (ExcInfo e) {
         setCAPIException(e);
         return -1;

src/runtime/objmodel.cpp

@@ -1241,12 +1241,13 @@ void HCAttrs::clear() noexcept {
 
     if (unlikely(hcls->type == HiddenClass::DICT_BACKED)) {
         Box* d = this->attr_list->attrs[0];
-        Py_DECREF(d);
 
         // Skips the attrlist freelist
         PyObject_FREE(this->attr_list);
         this->attr_list = NULL;
 
+        Py_DECREF(d);
+
         return;
     }
 
@@ -1264,6 +1265,37 @@ void HCAttrs::clear() noexcept {
     }
 }
 
+void HCAttrs::moduleClear() noexcept {
+    auto hcls = this->hcls;
+    if (!hcls)
+        return;
+
+    RELEASE_ASSERT(hcls->type == HiddenClass::NORMAL || hcls->type == HiddenClass::SINGLETON, "");
+
+    auto attr_list = this->attr_list;
+    auto attr_list_size = hcls->attributeArraySize();
+
+    for (auto&& p : hcls->getStrAttrOffsets()) {
+        const char* s = p.first->c_str();
+        if (s[0] == '_' && s[1] != '_') {
+            int idx = p.second;
+            Box* b = attr_list->attrs[idx];
+            attr_list->attrs[idx] = incref(None);
+            Py_DECREF(b);
+        }
+    }
+
+    for (auto&& p : hcls->getStrAttrOffsets()) {
+        const char* s = p.first->c_str();
+        if (s[0] != '_' || strcmp(s, "__builtins__") != 0) {
+            int idx = p.second;
+            Box* b = attr_list->attrs[idx];
+            attr_list->attrs[idx] = incref(None);
+            Py_DECREF(b);
+        }
+    }
+}
+
 void Box::appendNewHCAttr(BORROWED(Box*) new_attr, SetattrRewriteArgs* rewrite_args) {
     assert(cls->instancesHaveHCAttrs());
     HCAttrs* attrs = getHCAttrsPtr();

src/runtime/types.cpp

@@ -2815,6 +2815,12 @@ BORROWED(Box*) unwrapAttrWrapper(Box* b) {
     return static_cast<AttrWrapper*>(b)->getUnderlying();
 }
 
+void attrwrapperSet(Box* b, Box* k, Box* v) {
+    assert(b->cls == attrwrapper_cls);
+
+    autoDecref(AttrWrapper::setitem(b, k, v));
+}
+
 
 void Box::setDictBacked(STOLEN(Box*) val) {
     // this checks for: v.__dict__ = v.__dict__
@@ -3815,9 +3821,11 @@ template <typename CM> void clearContiguousMap(CM& cm) {
     }
 }
 
-int BoxedModule::clear(Box* b) noexcept {
+extern "C" void _PyModule_Clear(PyObject* b) noexcept {
     BoxedModule* self = static_cast<BoxedModule*>(b);
-    self->clearAttrs();
+
+    HCAttrs* attrs = self->getHCAttrsPtr();
+    attrs->moduleClear();
 
     clearContiguousMap(self->str_constants);
     clearContiguousMap(self->unicode_constants);
@@ -3825,7 +3833,13 @@ int BoxedModule::clear(Box* b) noexcept {
     clearContiguousMap(self->float_constants);
     clearContiguousMap(self->imaginary_constants);
     clearContiguousMap(self->long_constants);
+
     assert(!self->keep_alive.size());
+}
+
+int BoxedModule::clear(Box* b) noexcept {
+    _PyModule_Clear(b);
+    b->clearAttrs();
 
     return 0;
 }
@@ -4765,7 +4779,9 @@ extern "C" void Py_Finalize() noexcept {
         g.func_addr_registry.dumpPerfMap();
 
     call_sys_exitfunc();
-// initialized = 0;
+    // initialized = 0;
+
+    PyImport_Cleanup();
 
 #ifdef Py_REF_DEBUG
     IN_SHUTDOWN = true;
@@ -4810,6 +4826,16 @@ extern "C" void Py_Finalize() noexcept {
     while (PyGC_Collect())
         ;
     assert(!constants.size());
+
+    BoxedList* garbage = static_cast<BoxedList*>(_PyGC_GetGarbage());
+    int num_garbage_objects = garbage->size;
+
+    // Free the garbage list, but let all the elements in it stay alive:
+    for (int i = 0; i < num_garbage_objects; i++) {
+        Py_INCREF(garbage->elts->elts[i]);
+    }
+    Py_DECREF(garbage);
+
 #endif
 
 // PyGC_Collect());
@@ -4868,15 +4894,20 @@ extern "C" void Py_Finalize() noexcept {
 
     teardownCodegen();
 
+#ifdef Py_REF_DEBUG
     if (VERBOSITY())
         PRINT_TOTAL_REFS();
-#ifdef Py_REF_DEBUG
+
+    if (num_garbage_objects == 0) {
 #ifdef Py_TRACE_REFS
-    if (_Py_RefTotal != 0)
-        _Py_PrintReferenceAddressesCapped(stderr, 10);
+        if (_Py_RefTotal != 0)
+            _Py_PrintReferenceAddressesCapped(stderr, 10);
 #endif
 
-    RELEASE_ASSERT(_Py_RefTotal == 0, "%ld refs remaining!", _Py_RefTotal);
+        RELEASE_ASSERT(_Py_RefTotal == 0, "%ld refs remaining!", _Py_RefTotal);
+    } else if (VERBOSITY()) {
+        fprintf(stderr, "[%d garbage objects]\n", num_garbage_objects);
+    }
 #endif
 }
 }

src/runtime/types.h

@@ -1040,6 +1040,7 @@ public:
     DEFAULT_CLASS(builtin_function_or_method_cls);
 };
 
+extern "C" void _PyModule_Clear(PyObject*) noexcept;
 class BoxedModule : public Box {
 public:
     HCAttrs attrs;
@@ -1076,6 +1077,8 @@ private:
 
 public:
     DEFAULT_CLASS(module_cls);
+
+    friend void _PyModule_Clear(PyObject*) noexcept;
 };
 
 class BoxedSlice : public Box {
@@ -1334,6 +1337,7 @@ Box* attrwrapperKeys(Box* b);
 void attrwrapperDel(Box* b, llvm::StringRef attr);
 void attrwrapperClear(Box* b);
 BoxedDict* attrwrapperToDict(Box* b);
+void attrwrapperSet(Box* b, Box* k, Box* v);
 
 Box* boxAst(AST* ast);
 AST* unboxAst(Box* b);

test/tests/class_finalization.py

-# expected: reffail
 from testing_helpers import test_gc
 
 unordered_finalize = {}

test/tests/finalizer_exception.py

-# expected: reffail
 # Exceptions from finalizers should get caught:
 import sys
 from testing_helpers import test_gc

test/tests/finalizer_gc_call.py

-# expected: reffail
 import gc
 from testing_helpers import test_gc
 

test/tests/finalizers_inheritance.py

-# expected: reffail
 from testing_helpers import test_gc
 
 class C(object):

test/tests/int_ids2.py

-# expected: reffail
 # while I think nothing requires that this works I actually found this in a library...
 import subprocess
 def f():

test/tests/nonself_resurrection.py

-# expected: reffail
 # Objects are allowed to resurrect other objects too, I guess
 from testing_helpers import test_gc
 

test/tests/setattr_patching_under.py

-# expected: fail
-# - finalization not implemented yet
-# This test might also be broken in the presence of GC
-
 class C(object):
     pass
 

test/tests/weakref_finalizers.py

-# expected: reffail
 # test to ensure that weakref callbacks and finalizers get called in the
 # right order