Support extension classes that specify tp_dictoffset

I think there are a couple ways we can go about this.  It looks like
extensions just specify tp_dictoffset if they want their instances
to have instance attributes, but don't usually actually look at the
dict or really make use of the fact that it's a real dict.  So possibly,
we have the ability to stuff a non-dict into the slot they reserved for
us.

Seems risky though.  Instead, the approach in this commit is to create
an actual dict and put it in like they expect.  All the attribute-accessing
paths have been updated to look at both the fast Pyston HCAttrs method, and
the CAPI dict method.

Makefile

@@ -869,6 +869,10 @@ runpy_%: %.py ext_python
 	PYTHONPATH=test/test_extension/build/lib.linux-x86_64-2.7 python $<
 $(call make_search,runpy_%)
 
+check_%: %.py ext_python ext_pyston
+	$(MAKE) check_dbg ARGS="$(patsubst %.py,%,$(notdir $<)) -K"
+$(call make_search,check_%)
+
 dbgpy_%: %.py ext_pythondbg
 	export PYTHON_VERSION=$$(python2.7-dbg -V 2>&1 | awk '{print $$2}'); PYTHONPATH=test/test_extension/build/lib.linux-x86_64-2.7-pydebug $(GDB) --ex "dir $(DEPS_DIR)/python-src/python2.7-$$PYTHON_VERSION/debian" $(GDB_CMDS) --args python2.7-dbg $<
 $(call make_search,dbgpy_%)

src/capi/typeobject.cpp

@@ -1762,9 +1762,6 @@ extern "C" int PyType_Ready(PyTypeObject* cls) noexcept {
     cls->gc_visit = &conservativeGCHandler;
     cls->is_user_defined = true;
 
-    // TODO not sure how we can handle extension types that manually
-    // specify a dict...
-    RELEASE_ASSERT(cls->tp_dictoffset == 0, "");
     // this should get automatically initialized to 0 on this path:
     assert(cls->attrs_offset == 0);
 

src/codegen/compvars.cpp

@@ -1274,8 +1274,13 @@ public:
         return c == cls;
     }
 
+    bool canStaticallyResolveGetattrs() {
+        return (cls->is_constant && !cls->instancesHaveHCAttrs() && !cls->instancesHaveDictAttrs()
+                && cls->hasGenericGetattr());
+    }
+
     CompilerType* getattrType(const std::string* attr, bool cls_only) override {
-        if (cls->is_constant && !cls->instancesHaveAttrs() && cls->hasGenericGetattr()) {
+        if (canStaticallyResolveGetattrs()) {
             Box* rtattr = cls->getattr(*attr);
             if (rtattr == NULL)
                 return UNDEF;
@@ -1300,7 +1305,7 @@ public:
     CompilerVariable* getattr(IREmitter& emitter, const OpInfo& info, ConcreteCompilerVariable* var,
                               const std::string* attr, bool cls_only) override {
         // printf("%s.getattr %s\n", debugName().c_str(), attr->c_str());
-        if (cls->is_constant && !cls->instancesHaveAttrs() && cls->hasGenericGetattr()) {
+        if (canStaticallyResolveGetattrs()) {
             Box* rtattr = cls->getattr(*attr);
             if (rtattr == NULL) {
                 llvm::CallSite call = emitter.createCall2(info.unw_info, g.funcs.raiseAttributeErrorStr,
@@ -1346,7 +1351,7 @@ public:
                                                   const std::vector<CompilerVariable*>& args,
                                                   const std::vector<const std::string*>* keyword_names,
                                                   bool raise_on_missing = true) {
-        if (!cls->is_constant || cls->instancesHaveAttrs() || !cls->hasGenericGetattr())
+        if (!canStaticallyResolveGetattrs())
             return NULL;
 
         Box* rtattr = cls->getattr(*attr);

src/core/types.h

@@ -381,6 +381,7 @@ public:
     HCAttrs() : hcls(root_hcls), attr_list(nullptr) {}
 };
 
+class BoxedDict;
 class BoxedString;
 
 class Box {
@@ -396,7 +397,8 @@ public:
 
     llvm::iterator_range<BoxIterator> pyElements();
 
-    HCAttrs* getAttrsPtr();
+    HCAttrs* getHCAttrsPtr();
+    BoxedDict* getDict();
 
     void setattr(const std::string& attr, Box* val, SetattrRewriteArgs* rewrite_args);
     void giveAttr(const std::string& attr, Box* val) {

src/runtime/builtin_modules/builtins.cpp

@@ -91,12 +91,15 @@ extern "C" Box* dir(Box* obj) {
     for (auto const& kv : obj->cls->attrs.hcls->attr_offsets) {
         listAppend(result, boxString(kv.first));
     }
-    if (obj->cls->instancesHaveAttrs()) {
-        HCAttrs* attrs = obj->getAttrsPtr();
+    if (obj->cls->instancesHaveHCAttrs()) {
+        HCAttrs* attrs = obj->getHCAttrsPtr();
         for (auto const& kv : attrs->hcls->attr_offsets) {
             listAppend(result, boxString(kv.first));
         }
     }
+    if (obj->cls->instancesHaveDictAttrs()) {
+        Py_FatalError("unimplemented");
+    }
     return result;
 }
 

src/runtime/types.cpp

@@ -349,8 +349,8 @@ extern "C" void boxGCHandler(GCVisitor* v, Box* b) {
     if (b->cls) {
         v->visit(b->cls);
 
-        if (b->cls->instancesHaveAttrs()) {
-            HCAttrs* attrs = b->getAttrsPtr();
+        if (b->cls->instancesHaveHCAttrs()) {
+            HCAttrs* attrs = b->getHCAttrsPtr();
 
             v->visit(attrs->hcls);
             int nattrs = attrs->hcls->attr_offsets.size();
@@ -361,6 +361,10 @@ extern "C" void boxGCHandler(GCVisitor* v, Box* b) {
                 v->visitRange((void**)&attr_list->attrs[0], (void**)&attr_list->attrs[nattrs]);
             }
         }
+
+        if (b->cls->instancesHaveDictAttrs()) {
+            RELEASE_ASSERT(0, "Shouldn't all of these objects be conservatively scanned?");
+        }
     } else {
         assert(type_cls == NULL || b == type_cls);
     }
@@ -776,7 +780,7 @@ private:
     Box* b;
 
 public:
-    AttrWrapper(Box* b) : b(b) {}
+    AttrWrapper(Box* b) : b(b) { assert(b->cls->instancesHaveHCAttrs()); }
 
     DEFAULT_CLASS(attrwrapper_cls);
 
@@ -829,7 +833,7 @@ public:
         std::ostringstream os("");
         os << "attrwrapper({";
 
-        HCAttrs* attrs = self->b->getAttrsPtr();
+        HCAttrs* attrs = self->b->getHCAttrsPtr();
         bool first = true;
         for (const auto& p : attrs->hcls->attr_offsets) {
             if (!first)
@@ -859,7 +863,7 @@ public:
 
         BoxedList* rtn = new BoxedList();
 
-        HCAttrs* attrs = self->b->getAttrsPtr();
+        HCAttrs* attrs = self->b->getHCAttrsPtr();
         for (const auto& p : attrs->hcls->attr_offsets) {
             BoxedTuple* t = new BoxedTuple({ boxString(p.first), attrs->attr_list->attrs[p.second] });
             listAppend(rtn, t);
@@ -869,7 +873,7 @@ public:
 };
 
 Box* makeAttrWrapper(Box* b) {
-    assert(b->cls->instancesHaveAttrs());
+    assert(b->cls->instancesHaveHCAttrs());
     return new AttrWrapper(b);
 }
 
@@ -908,8 +912,8 @@ Box* objectStr(Box* obj) {
 // Added as parameter because it should typically be available
 inline void initUserAttrs(Box* obj, BoxedClass* cls) {
     assert(obj->cls == cls);
-    if (cls->attrs_offset) {
-        HCAttrs* attrs = obj->getAttrsPtr();
+    if (cls->instancesHaveHCAttrs()) {
+        HCAttrs* attrs = obj->getHCAttrsPtr();
         attrs = new ((void*)attrs) HCAttrs();
     }
 }

src/runtime/types.h

@@ -199,7 +199,8 @@ public:
     // Analogous to tp_dictoffset
     const int attrs_offset;
 
-    bool instancesHaveAttrs() { return attrs_offset != 0; }
+    bool instancesHaveHCAttrs() { return attrs_offset != 0; }
+    bool instancesHaveDictAttrs() { return tp_dictoffset != 0; }
 
     // Whether this class object is constant or not, ie whether or not class-level
     // attributes can be changed or added.

test/tests/capi_slots.py

@@ -102,7 +102,7 @@ except AttributeError, e:
 c = C3(5)
 c.foo = 1
 print c.foo
-print c.__dict__
+print c.__dict__.items()
 
 s = slots_test.SlotsTesterMap(6)
 s.bar = 2