Merge pull request #1120 from undingen/threading_fixes

Threading and multiprocessing related fixes

from_cpython/Include/abstract.h

@@ -1359,7 +1359,7 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
        */
 #define PyMapping_Items(O) PyObject_CallMethod(O,"items",NULL)
 
-     PyAPI_FUNC(PyObject *) PyMapping_GetItemString(PyObject *o, char *key) PYSTON_NOEXCEPT;
+     PyAPI_FUNC(PyObject *) PyMapping_GetItemString(PyObject *o, const char *key) PYSTON_NOEXCEPT;
 
        /*
      Return element of o corresponding to the object, key, or NULL
@@ -1367,7 +1367,7 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
      o[key].
        */
 
-     PyAPI_FUNC(int) PyMapping_SetItemString(PyObject *o, char *key,
+     PyAPI_FUNC(int) PyMapping_SetItemString(PyObject *o, const char *key,
                                             PyObject *value) PYSTON_NOEXCEPT;
 
        /*

from_cpython/Include/pyerrors.h

@@ -249,7 +249,7 @@ PyAPI_FUNC(void) _PyErr_BadInternalCall(const char *filename, int lineno) PYSTON
 
 /* Function to create a new exception */
 PyAPI_FUNC(PyObject *) PyErr_NewException(
-    char *name, PyObject *base, PyObject *dict) PYSTON_NOEXCEPT;
+    const char *name, PyObject *base, PyObject *dict) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) PyErr_NewExceptionWithDoc(
     char *name, char *doc, PyObject *base, PyObject *dict) PYSTON_NOEXCEPT;
 PyAPI_FUNC(void) PyErr_WriteUnraisable(PyObject *) PYSTON_NOEXCEPT;

from_cpython/Python/errors.c

@@ -551,7 +551,7 @@ PyErr_Format(PyObject *exception, const char *format, ...)
 
 
 PyObject *
-PyErr_NewException(char *name, PyObject *base, PyObject *dict)
+PyErr_NewException(const char *name, PyObject *base, PyObject *dict)
 {
     char *dot;
     PyObject *modulename = NULL;

src/capi/abstract.cpp

@@ -1804,7 +1804,7 @@ extern "C" int PyMapping_HasKey(PyObject* o, PyObject* key) noexcept {
     return 0;
 }
 
-extern "C" PyObject* PyMapping_GetItemString(PyObject* o, char* key) noexcept {
+extern "C" PyObject* PyMapping_GetItemString(PyObject* o, const char* key) noexcept {
     PyObject* okey, *r;
 
     if (key == NULL)
@@ -1818,7 +1818,7 @@ extern "C" PyObject* PyMapping_GetItemString(PyObject* o, char* key) noexcept {
     return r;
 }
 
-extern "C" int PyMapping_SetItemString(PyObject* o, char* key, PyObject* value) noexcept {
+extern "C" int PyMapping_SetItemString(PyObject* o, const char* key, PyObject* value) noexcept {
     PyObject* okey;
     int r;
 

src/capi/types.h

@@ -41,7 +41,10 @@ public:
 
     static BoxedString* __repr__(BoxedCApiFunction* self) {
         assert(self->cls == capifunc_cls);
-        return boxString(self->method_def->ml_name);
+        if (self->passthrough == NULL)
+            return (BoxedString*)PyString_FromFormat("<built-in function %s>", self->method_def->ml_name);
+        return (BoxedString*)PyString_FromFormat("<built-in method %s of %s object at %p>", self->method_def->ml_name,
+                                                 self->passthrough->cls->tp_name, self->passthrough);
     }
 
     static Box* __call__(BoxedCApiFunction* self, BoxedTuple* varargs, BoxedDict* kwargs);

src/core/threading.cpp

@@ -522,11 +522,34 @@ void registerMainThread() {
     assert(!PyErr_Occurred());
 }
 
+/* Wait until threading._shutdown completes, provided
+   the threading module was imported in the first place.
+   The shutdown routine will wait until all non-daemon
+   "threading" threads have completed. */
+static void wait_for_thread_shutdown(void) noexcept {
+#ifdef WITH_THREAD
+    PyObject* result;
+    PyThreadState* tstate = PyThreadState_GET();
+    PyObject* threading = PyMapping_GetItemString(getSysModulesDict(), "threading");
+    if (threading == NULL) {
+        /* threading not imported */
+        PyErr_Clear();
+        return;
+    }
+    result = PyObject_CallMethod(threading, "_shutdown", "");
+    if (result == NULL)
+        PyErr_WriteUnraisable(threading);
+    else
+        Py_DECREF(result);
+    Py_DECREF(threading);
+#endif
+}
+
 void finishMainThread() {
     assert(current_internal_thread_state);
     current_internal_thread_state->assertNoGenerators();
 
-    // TODO maybe this is the place to wait for non-daemon threads?
+    wait_for_thread_shutdown();
 }
 
 bool isMainThread() {
@@ -601,6 +624,22 @@ extern "C" void PyEval_ReInitThreads() noexcept {
     threads_waiting_on_gil = 0;
 
     PerThreadSetBase::runAllForkHandlers();
+
+    /* Update the threading module with the new state.
+     */
+    Box* threading = PyMapping_GetItemString(getSysModulesDict(), "threading");
+    if (threading == NULL) {
+        /* threading not imported */
+        PyErr_Clear();
+        return;
+    }
+    Box* result = PyObject_CallMethod(threading, "_after_fork", NULL);
+    if (result == NULL)
+        PyErr_WriteUnraisable(threading);
+    else
+        Py_DECREF(result);
+    Py_DECREF(threading);
+
 }
 
 void acquireGLWrite() {

src/runtime/builtin_modules/builtins.cpp

@@ -2540,7 +2540,7 @@ void setupBuiltins() {
         { "reload", builtin_reload, METH_O, reload_doc },
     };
     for (auto& md : builtin_methods) {
-        builtins_module->giveAttr(md.ml_name, new BoxedCApiFunction(&md, builtins_module, boxString("__builtin__")));
+        builtins_module->giveAttr(md.ml_name, new BoxedCApiFunction(&md, NULL, boxString("__builtin__")));
     }
 }
 }

src/runtime/builtin_modules/sys.cpp

@@ -67,9 +67,6 @@ Box* sysExcClear() {
 static Box* sysExit(Box* arg) {
     assert(arg);
     Box* exc = runtimeCall(SystemExit, ArgPassSpec(1), arg, NULL, NULL, NULL, NULL);
-    // TODO this should be handled by the SystemExit constructor
-    exc->giveAttr("code", arg);
-
     raiseExc(exc);
 }
 
@@ -728,7 +725,7 @@ void setupSys() {
     sys_flags_cls->freeze();
 
     for (auto& md : sys_methods) {
-        sys_module->giveAttr(md.ml_name, new BoxedCApiFunction(&md, sys_module, boxString("sys")));
+        sys_module->giveAttr(md.ml_name, new BoxedCApiFunction(&md, NULL, boxString("sys")));
     }
 
     sys_module->giveAttr("__displayhook__", sys_module->getattr(internStringMortal("displayhook")));

src/runtime/builtin_modules/thread.cpp

@@ -90,7 +90,7 @@ static void* thread_start(Box* target, Box* varargs, Box* kwargs) {
 // TODO this should take kwargs, which defaults to empty
 Box* startNewThread(Box* target, Box* args, Box* kw) {
     intptr_t thread_id = start_thread(&thread_start, target, args, kw);
-    return boxInt(thread_id ^ 0x12345678901L);
+    return boxInt(thread_id);
 }
 
 #define CHECK_STATUS(name)                                                                                             \
@@ -244,11 +244,7 @@ void setupThread() {
     thread_lock_cls->giveAttr("locked_lock", thread_lock_cls->getattr(internStringMortal("locked")));
     thread_lock_cls->freeze();
 
-    ThreadError = BoxedClass::create(type_cls, Exception, NULL, Exception->attrs_offset, Exception->tp_weaklistoffset,
-                                     Exception->tp_basicsize, false, "error");
-    ThreadError->giveAttr("__module__", boxString("thread"));
-    ThreadError->freeze();
-
+    ThreadError = (BoxedClass*)PyErr_NewException("thread.error", NULL, NULL);
     thread_module->giveAttr("error", ThreadError);
 }
 }

src/runtime/types.cpp

@@ -1427,20 +1427,6 @@ static Box* typeSubDict(Box* obj, void* context) {
     abort();
 }
 
-void Box::setDictBacked(Box* val) {
-    assert(this->cls->instancesHaveHCAttrs());
-
-    RELEASE_ASSERT(val->cls == dict_cls || val->cls == attrwrapper_cls, "");
-
-    auto new_attr_list = (HCAttrs::AttrList*)gc_alloc(sizeof(HCAttrs::AttrList) + sizeof(Box*), gc::GCKind::PRECISE);
-    new_attr_list->attrs[0] = val;
-
-    HCAttrs* hcattrs = this->getHCAttrsPtr();
-
-    hcattrs->hcls = HiddenClass::dict_backed;
-    hcattrs->attr_list = new_attr_list;
-}
-
 static void typeSubSetDict(Box* obj, Box* val, void* context) {
     if (obj->cls->instancesHaveDictAttrs()) {
         RELEASE_ASSERT(val->cls == dict_cls, "");
@@ -1449,16 +1435,7 @@ static void typeSubSetDict(Box* obj, Box* val, void* context) {
     }
 
     if (obj->cls->instancesHaveHCAttrs()) {
-        RELEASE_ASSERT(PyDict_Check(val) || val->cls == attrwrapper_cls, "%s", val->cls->tp_name);
-
-        auto new_attr_list
-            = (HCAttrs::AttrList*)gc_alloc(sizeof(HCAttrs::AttrList) + sizeof(Box*), gc::GCKind::PRECISE);
-        new_attr_list->attrs[0] = val;
-
-        HCAttrs* hcattrs = obj->getHCAttrsPtr();
-
-        hcattrs->hcls = HiddenClass::dict_backed;
-        hcattrs->attr_list = new_attr_list;
+        obj->setDictBacked(val);
         return;
     }
 
@@ -2294,32 +2271,45 @@ public:
 // or PyModule_GetDict to return real dicts.
 class AttrWrapper : public Box {
 private:
-    Box* b;
+    // The fields are not not allowed to be set at the same time
+    Box* b;                  // set when this attribute wrapper wraps a Box (default mode)
+    BoxedDict* private_dict; // set when the attribute wrapper does not wrap a Box anymore because the Box.__dict__
+                             // changed to another dict.
 
+    // This is useful when we assign non str_cls keys, because the attribute array only supports strs.
     void convertToDictBacked() {
-        HCAttrs* attrs = this->b->getHCAttrsPtr();
-        if (attrs->hcls->type == HiddenClass::DICT_BACKED)
+        if (isDictBacked())
             return;
 
-        BoxedDict* d = new BoxedDict();
+        HCAttrs* attrs = this->b->getHCAttrsPtr();
+        assert(attrs->hcls->type != HiddenClass::DICT_BACKED);
+        BoxedDict* d = (BoxedDict*)AttrWrapper::copy(this);
 
-        RELEASE_ASSERT(attrs->hcls->type == HiddenClass::NORMAL || attrs->hcls->type == HiddenClass::SINGLETON, "");
-        for (const auto& p : attrs->hcls->getStrAttrOffsets()) {
-            d->d[p.first] = attrs->attr_list->attrs[p.second];
-        }
+        HCAttrs* hcattrs = b->getHCAttrsPtr();
+        auto new_attr_list
+            = (HCAttrs::AttrList*)gc_alloc(sizeof(HCAttrs::AttrList) + sizeof(Box*), gc::GCKind::PRECISE);
+        new_attr_list->attrs[0] = d;
 
-        b->setDictBacked(d);
+        hcattrs->hcls = HiddenClass::dict_backed;
+        hcattrs->attr_list = new_attr_list;
     }
 
-    bool isDictBacked() { return b->getHCAttrsPtr()->hcls->type == HiddenClass::DICT_BACKED; }
+    bool isDictBacked() {
+        if (private_dict)
+            return private_dict;
+        return b->getHCAttrsPtr()->hcls->type == HiddenClass::DICT_BACKED;
+    }
 
     Box* getDictBacking() {
         assert(isDictBacked());
+        if (private_dict)
+            return private_dict;
         return b->getHCAttrsPtr()->attr_list->attrs[0];
     }
 
+
 public:
-    AttrWrapper(Box* b) : b(b) {
+    AttrWrapper(Box* b) : b(b), private_dict(NULL) {
         assert(b->cls->instancesHaveHCAttrs());
 
         // We currently don't support creating an attrwrapper around a dict-backed object,
@@ -2331,15 +2321,28 @@ public:
                || b->getHCAttrsPtr()->hcls->type == HiddenClass::SINGLETON);
     }
 
+    void convertToPivateDict() {
+        RELEASE_ASSERT(!private_dict, "");
+        RELEASE_ASSERT(b, "");
+        private_dict = (BoxedDict*)AttrWrapper::copy(this);
+        assert(PyDict_CheckExact(private_dict));
+        b = NULL;
+    }
+
     DEFAULT_CLASS(attrwrapper_cls);
 
-    Box* getUnderlying() { return b; }
+    Box* getUnderlying() {
+        if (private_dict)
+            return private_dict;
+        return b;
+    }
 
     static void gcHandler(GCVisitor* v, Box* b) {
         Box::gcHandler(v, b);
 
         AttrWrapper* aw = (AttrWrapper*)b;
         v->visit(&aw->b);
+        v->visit(&aw->private_dict);
     }
 
     static Box* setitem(Box* _self, Box* _key, Box* value) {
@@ -2408,7 +2411,14 @@ public:
         RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
         AttrWrapper* self = static_cast<AttrWrapper*>(_self);
 
-        _key = coerceUnicodeToStr<CXX>(_key);
+        if (_key->cls != str_cls)
+            self->convertToDictBacked();
+
+        if (self->isDictBacked()) {
+            static BoxedString* get_str = internStringImmortal("get");
+            return callattrInternal<CXX, NOT_REWRITABLE>(self->getDictBacking(), get_str, LookupScope::CLASS_ONLY, NULL,
+                                                         ArgPassSpec(2), _key, def, NULL, NULL, NULL);
+        }
 
         RELEASE_ASSERT(_key->cls == str_cls, "");
         BoxedString* key = static_cast<BoxedString*>(_key);
@@ -2424,9 +2434,14 @@ public:
         RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
         AttrWrapper* self = static_cast<AttrWrapper*>(_self);
 
-        _key = coerceUnicodeToStr<S>(_key);
-        if (S == CAPI && !_key)
-            return NULL;
+        if (_key->cls != str_cls)
+            self->convertToDictBacked();
+
+        if (self->isDictBacked()) {
+            static BoxedString* getitem_str = internStringImmortal("__getitem__");
+            return callattrInternal<S, NOT_REWRITABLE>(self->getDictBacking(), getitem_str, LookupScope::CLASS_ONLY,
+                                                       NULL, ArgPassSpec(1), _key, NULL, NULL, NULL, NULL);
+        }
 
         RELEASE_ASSERT(_key->cls == str_cls, "");
         BoxedString* key = static_cast<BoxedString*>(_key);
@@ -2446,7 +2461,14 @@ public:
         RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
         AttrWrapper* self = static_cast<AttrWrapper*>(_self);
 
-        _key = coerceUnicodeToStr<CXX>(_key);
+        if (_key->cls != str_cls)
+            self->convertToDictBacked();
+
+        if (self->isDictBacked()) {
+            static BoxedString* pop_str = internStringImmortal("pop");
+            return callattrInternal<CXX, NOT_REWRITABLE>(self->getDictBacking(), pop_str, LookupScope::CLASS_ONLY, NULL,
+                                                         ArgPassSpec(2), _key, default_, NULL, NULL, NULL);
+        }
 
         RELEASE_ASSERT(_key->cls == str_cls, "");
         BoxedString* key = static_cast<BoxedString*>(_key);
@@ -2467,7 +2489,14 @@ public:
         RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
         AttrWrapper* self = static_cast<AttrWrapper*>(_self);
 
-        _key = coerceUnicodeToStr<CXX>(_key);
+        if (_key->cls != str_cls)
+            self->convertToDictBacked();
+
+        if (self->isDictBacked()) {
+            static BoxedString* delitem_str = internStringImmortal("__delitem__");
+            return callattrInternal<CXX, NOT_REWRITABLE>(self->getDictBacking(), delitem_str, LookupScope::CLASS_ONLY,
+                                                         NULL, ArgPassSpec(1), _key, NULL, NULL, NULL, NULL);
+        }
 
         RELEASE_ASSERT(_key->cls == str_cls, "");
         BoxedString* key = static_cast<BoxedString*>(_key);
@@ -2487,20 +2516,31 @@ public:
         std::string O("");
         llvm::raw_string_ostream os(O);
 
-        os << "attrwrapper({";
-
-        HCAttrs* attrs = self->b->getHCAttrsPtr();
-        RELEASE_ASSERT(attrs->hcls->type == HiddenClass::NORMAL || attrs->hcls->type == HiddenClass::SINGLETON, "");
-        bool first = true;
-        for (const auto& p : attrs->hcls->getStrAttrOffsets()) {
-            if (!first)
-                os << ", ";
-            first = false;
-
-            BoxedString* v = attrs->attr_list->attrs[p.second]->reprICAsString();
-            os << p.first->s() << ": " << v->s();
+        os << "attrwrapper(";
+        if (self->isDictBacked()) {
+            static BoxedString* repr_str = internStringImmortal("__repr__");
+            Box* dict_repr
+                = callattrInternal<CXX, NOT_REWRITABLE>(self->getDictBacking(), repr_str, LookupScope::CLASS_ONLY, NULL,
+                                                        ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
+            RELEASE_ASSERT(PyString_CheckExact(dict_repr), "");
+            os << ((BoxedString*)dict_repr)->s();
+        } else {
+            HCAttrs* attrs = self->b->getHCAttrsPtr();
+            os << "{";
+            RELEASE_ASSERT(attrs->hcls->type == HiddenClass::NORMAL || attrs->hcls->type == HiddenClass::SINGLETON, "");
+            bool first = true;
+            for (const auto& p : attrs->hcls->getStrAttrOffsets()) {
+                if (!first)
+                    os << ", ";
+                first = false;
+
+                BoxedString* v = attrs->attr_list->attrs[p.second]->reprICAsString();
+                os << p.first->s() << ": " << v->s();
+            }
+            os << "}";
         }
-        os << "})";
+        os << ")";
+
         return boxString(os.str());
     }
 
@@ -2508,9 +2548,14 @@ public:
         RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
         AttrWrapper* self = static_cast<AttrWrapper*>(_self);
 
-        _key = coerceUnicodeToStr<S>(_key);
-        if (S == CAPI && !_key)
-            return NULL;
+        if (_key->cls != str_cls)
+            self->convertToDictBacked();
+
+        if (self->isDictBacked()) {
+            static BoxedString* contains_str = internStringImmortal("__contains__");
+            return callattrInternal<S, NOT_REWRITABLE>(self->getDictBacking(), contains_str, LookupScope::CLASS_ONLY,
+                                                       NULL, ArgPassSpec(1), _key, NULL, NULL, NULL, NULL);
+        }
 
         RELEASE_ASSERT(_key->cls == str_cls, "");
         BoxedString* key = static_cast<BoxedString*>(_key);
@@ -2537,6 +2582,13 @@ public:
         RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
         AttrWrapper* self = static_cast<AttrWrapper*>(_self);
 
+        if (self->isDictBacked()) {
+            static BoxedString* keys_str = internStringImmortal("keys");
+            return callattrInternal<CXX, NOT_REWRITABLE>(self->getDictBacking(), keys_str, LookupScope::CLASS_ONLY,
+                                                         NULL, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
+        }
+
+
         BoxedList* rtn = new BoxedList();
 
         HCAttrs* attrs = self->b->getHCAttrsPtr();
@@ -2551,6 +2603,12 @@ public:
         RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
         AttrWrapper* self = static_cast<AttrWrapper*>(_self);
 
+        if (self->isDictBacked()) {
+            static BoxedString* values_str = internStringImmortal("values");
+            return callattrInternal<CXX, NOT_REWRITABLE>(self->getDictBacking(), values_str, LookupScope::CLASS_ONLY,
+                                                         NULL, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
+        }
+
         BoxedList* rtn = new BoxedList();
 
         HCAttrs* attrs = self->b->getHCAttrsPtr();
@@ -2565,6 +2623,12 @@ public:
         RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
         AttrWrapper* self = static_cast<AttrWrapper*>(_self);
 
+        if (self->isDictBacked()) {
+            static BoxedString* items_str = internStringImmortal("items");
+            return callattrInternal<CXX, NOT_REWRITABLE>(self->getDictBacking(), items_str, LookupScope::CLASS_ONLY,
+                                                         NULL, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
+        }
+
         BoxedList* rtn = new BoxedList();
 
         HCAttrs* attrs = self->b->getHCAttrsPtr();
@@ -2595,6 +2659,12 @@ public:
         RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
         AttrWrapper* self = static_cast<AttrWrapper*>(_self);
 
+        if (self->isDictBacked()) {
+            static BoxedString* copy_str = internStringImmortal("copy");
+            return callattrInternal<CXX, NOT_REWRITABLE>(self->getDictBacking(), copy_str, LookupScope::CLASS_ONLY,
+                                                         NULL, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
+        }
+
         BoxedDict* rtn = new BoxedDict();
 
         HCAttrs* attrs = self->b->getHCAttrsPtr();
@@ -2609,6 +2679,12 @@ public:
         RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
         AttrWrapper* self = static_cast<AttrWrapper*>(_self);
 
+        if (self->isDictBacked()) {
+            static BoxedString* clear_str = internStringImmortal("clear");
+            return callattrInternal<CXX, NOT_REWRITABLE>(self->getDictBacking(), clear_str, LookupScope::CLASS_ONLY,
+                                                         NULL, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
+        }
+
         HCAttrs* attrs = self->b->getHCAttrsPtr();
         RELEASE_ASSERT(attrs->hcls->type == HiddenClass::NORMAL || attrs->hcls->type == HiddenClass::SINGLETON, "");
 
@@ -2625,6 +2701,12 @@ public:
         RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
         AttrWrapper* self = static_cast<AttrWrapper*>(_self);
 
+        if (self->isDictBacked()) {
+            static BoxedString* len_str = internStringImmortal("__len__");
+            return callattrInternal<CXX, NOT_REWRITABLE>(self->getDictBacking(), len_str, LookupScope::CLASS_ONLY, NULL,
+                                                         ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
+        }
+
         HCAttrs* attrs = self->b->getHCAttrsPtr();
         RELEASE_ASSERT(attrs->hcls->type == HiddenClass::NORMAL || attrs->hcls->type == HiddenClass::SINGLETON, "");
         return boxInt(attrs->hcls->getStrAttrOffsets().size());
@@ -2639,9 +2721,17 @@ public:
 
         RELEASE_ASSERT(args->size() <= 1, ""); // should throw a TypeError
 
+        if (self->isDictBacked()) {
+            static BoxedString* update_str = internStringImmortal("update");
+            return callattrInternal<CXX, NOT_REWRITABLE>(self->getDictBacking(), update_str, LookupScope::CLASS_ONLY,
+                                                         NULL, ArgPassSpec(0, 0, true, true), args, kwargs, NULL, NULL,
+                                                         NULL);
+        }
+
         auto handle = [&](Box* _container) {
             if (_container->cls == attrwrapper_cls) {
                 AttrWrapper* container = static_cast<AttrWrapper*>(_container);
+                RELEASE_ASSERT(!container->isDictBacked(), "not implemented");
                 HCAttrs* attrs = container->b->getHCAttrsPtr();
 
                 RELEASE_ASSERT(attrs->hcls->type == HiddenClass::NORMAL || attrs->hcls->type == HiddenClass::SINGLETON,
@@ -2785,6 +2875,36 @@ Box* unwrapAttrWrapper(Box* b) {
     return static_cast<AttrWrapper*>(b)->getUnderlying();
 }
 
+
+void Box::setDictBacked(Box* val) {
+    // this checks for: v.__dict__ = v.__dict__
+    if (val->cls == attrwrapper_cls && unwrapAttrWrapper(val) == this)
+        return;
+
+    assert(this->cls->instancesHaveHCAttrs());
+    HCAttrs* hcattrs = this->getHCAttrsPtr();
+    RELEASE_ASSERT(PyDict_Check(val) || val->cls == attrwrapper_cls, "");
+
+    // If there is an old attrwrapper it is not allowed to wrap the instance anymore instead it has to switch to a
+    // private dictonary.
+    // e.g.:
+    //     a = v.__dict__
+    //     v.__dict__ = {} # 'a' must switch now from wrapping 'v' to a the private dict.
+    int offset = hcattrs->hcls->type != HiddenClass::DICT_BACKED ? hcattrs->hcls->getAttrwrapperOffset() : -1;
+    if (offset != -1) {
+        AttrWrapper* wrapper = (AttrWrapper*)hcattrs->attr_list->attrs[offset];
+        RELEASE_ASSERT(wrapper->cls == attrwrapper_cls, "");
+        wrapper->convertToPivateDict();
+    }
+
+    // assign the dict to the attribute list and switch to the dict backed strategy
+    auto new_attr_list = (HCAttrs::AttrList*)gc_alloc(sizeof(HCAttrs::AttrList) + sizeof(Box*), gc::GCKind::PRECISE);
+    new_attr_list->attrs[0] = val;
+
+    hcattrs->hcls = HiddenClass::dict_backed;
+    hcattrs->attr_list = new_attr_list;
+}
+
 Box* attrwrapperKeys(Box* b) {
     return AttrWrapper::keys(b);
 }

test/tests/cPickle_test.py

@@ -46,3 +46,6 @@ for obj in [(1, 2), "hello world", u"hola world", 1.0, 1j, 1L, 2, {1:2}, set([3]
 import cStringIO
 StringIO = cPickle.loads(cPickle.dumps(cStringIO.StringIO))
 print type(StringIO())
+
+import thread
+print cPickle.loads(cPickle.dumps(thread.error))

test/tests/compile_eval.py

+print compile
 c = compile("a", "test.py", "eval")
 print type(c), c.co_filename, c.co_name
 

test/tests/dict_setting.py

@@ -51,6 +51,19 @@ print hasattr(c, "attr")
 print hasattr(c, "foo")
 print c.__dict__ is d1
 
+c = C()
+a = c.__dict__
+c.__dict__ = c.__dict__
+c.__dict__[u"uni"] = "u"
+c.__dict__[u"\u20ac"] = "u"
+c.__dict__[(1, 2, 3)] = "t"
+print sorted(c.__dict__.keys()), sorted(a.keys())
+print "all non str attributes:", sorted([item for item in dir(c) if not isinstance(item, str)])
+print "can we retrieve unicode attributes by there ascii value?", c.uni, c.__dict__["uni"]
+print "attrwrapper:", a["uni"], a[(1, 2, 3)]
+setattr(c, "uni", 1)
+print "test setattr()", c.uni, c.__dict__["uni"], a["uni"], len(c.__dict__), len(a)
+
 dictproxy = C.__dict__
 print type(dictproxy)
 print "foo" in dictproxy

test/tests/dict_setting_old_dict.py

-# expected: fail
-# - haven't bothered to implement this yet
-
 # Funny case: if we get the attrwrapper of an object, then change it to be dict-backed,
 # the original attrwrapper should remain valid but no longer connected to that object.
 
@@ -13,3 +10,10 @@ c1.a = 1
 print aw.items()
 c1.__dict__ = d = {}
 print aw.items()
+
+c2 = C()
+olddict = c2.__dict__
+c2.__dict__ = { "should not be there" : 1 }
+print olddict.has_key("should not be there")
+
+

test/tests/sys_test.py

 import sys
 import os.path
 
+print sys.excepthook, sys.displayhook
 print sys.version[:3]
 print os.path.exists(sys.executable)
 print sys.copyright[-200:]