Merge pull request #784 from rudi-c/cleanup

Refactors and cleanup in objmodel.cpp

src/CMakeLists.txt

@@ -94,6 +94,7 @@ add_library(PYSTON_OBJECTS OBJECT ${OPTIONAL_SRCS}
 		runtime/float.cpp
 		runtime/frame.cpp
 		runtime/generator.cpp
+		runtime/hiddenclass.cpp
 		runtime/ics.cpp
 		runtime/import.cpp
 		runtime/int.cpp

src/capi/typeobject.cpp

@@ -17,6 +17,7 @@
 
 #include "capi/types.h"
 #include "runtime/classobj.h"
+#include "runtime/hiddenclass.h"
 #include "runtime/objmodel.h"
 #include "runtime/rewrite_args.h"
 

src/codegen/ast_interpreter.cpp

@@ -44,6 +44,7 @@
 #include "runtime/objmodel.h"
 #include "runtime/set.h"
 #include "runtime/types.h"
+#include "runtime/util.h"
 
 #ifndef NDEBUG
 #define DEBUG 1

src/codegen/baseline_jit.cpp

@@ -27,6 +27,7 @@
 #include "runtime/objmodel.h"
 #include "runtime/set.h"
 #include "runtime/types.h"
+#include "runtime/util.h"
 
 namespace pyston {
 

src/codegen/compvars.cpp

@@ -31,6 +31,7 @@
 #include "runtime/int.h"
 #include "runtime/objmodel.h"
 #include "runtime/types.h"
+#include "runtime/util.h"
 
 namespace pyston {
 
@@ -1415,12 +1416,7 @@ private:
 
     static std::unordered_map<BoxedClass*, NormalObjectType*> made;
 
-    NormalObjectType(BoxedClass* cls) : cls(cls) {
-        // ASSERT(!isUserDefined(cls) && "instances of user-defined classes can change their __class__, plus even if
-        // they couldn't we couldn't statically resolve their attributes", "%s", getNameOfClass(cls)->c_str());
-
-        assert(cls);
-    }
+    NormalObjectType(BoxedClass* cls) : cls(cls) { assert(cls); }
 
 public:
     llvm::Type* llvmType() override { return g.llvm_value_type_ptr; }

src/codegen/runtime_hooks.cpp

@@ -46,6 +46,7 @@
 #include "runtime/objmodel.h"
 #include "runtime/set.h"
 #include "runtime/types.h"
+#include "runtime/util.h"
 
 extern "C" void* __cxa_begin_catch(void*);
 extern "C" void __cxa_end_catch();

src/gc/collector.cpp

@@ -25,6 +25,7 @@
 #include "core/types.h"
 #include "core/util.h"
 #include "gc/heap.h"
+#include "runtime/hiddenclass.h"
 #include "runtime/objmodel.h"
 #include "runtime/types.h"
 

src/gc/heap.cpp

@@ -21,6 +21,7 @@
 #include "core/common.h"
 #include "core/util.h"
 #include "gc/gc_alloc.h"
+#include "runtime/hiddenclass.h"
 #include "runtime/objmodel.h"
 #include "runtime/types.h"
 

src/runtime/hiddenclass.cpp

+// Copyright (c) 2014-2015 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// limitations under the License.
+
+#include "runtime/hiddenclass.h"
+
+#include <cassert>
+
+#include "asm_writing/icinfo.h"
+#include "asm_writing/rewriter.h"
+#include "core/stats.h"
+#include "core/types.h"
+#include "runtime/objmodel.h"
+#include "runtime/types.h"
+#include "runtime/util.h"
+
+namespace pyston {
+
+void HiddenClass::appendAttribute(BoxedString* attr) {
+    assert(attr->interned_state != SSTATE_NOT_INTERNED);
+    assert(type == SINGLETON);
+    dependent_getattrs.invalidateAll();
+    assert(attr_offsets.count(attr) == 0);
+    int n = this->attributeArraySize();
+    attr_offsets[attr] = n;
+}
+
+void HiddenClass::appendAttrwrapper() {
+    assert(type == SINGLETON);
+    dependent_getattrs.invalidateAll();
+    assert(attrwrapper_offset == -1);
+    attrwrapper_offset = this->attributeArraySize();
+}
+
+void HiddenClass::delAttribute(BoxedString* attr) {
+    assert(attr->interned_state != SSTATE_NOT_INTERNED);
+    assert(type == SINGLETON);
+    dependent_getattrs.invalidateAll();
+    assert(attr_offsets.count(attr));
+
+    int prev_idx = attr_offsets[attr];
+    attr_offsets.erase(attr);
+
+    for (auto it = attr_offsets.begin(), end = attr_offsets.end(); it != end; ++it) {
+        assert(it->second != prev_idx);
+        if (it->second > prev_idx)
+            it->second--;
+    }
+    if (attrwrapper_offset != -1 && attrwrapper_offset > prev_idx)
+        attrwrapper_offset--;
+}
+
+void HiddenClass::addDependence(Rewriter* rewriter) {
+    assert(type == SINGLETON);
+    rewriter->addDependenceOn(dependent_getattrs);
+}
+
+HiddenClass* HiddenClass::getOrMakeChild(BoxedString* attr) {
+    STAT_TIMER(t0, "us_timer_hiddenclass_getOrMakeChild", 0);
+
+    assert(attr->interned_state != SSTATE_NOT_INTERNED);
+    assert(type == NORMAL);
+
+    auto it = children.find(attr);
+    if (it != children.end())
+        return children.getMapped(it->second);
+
+    static StatCounter num_hclses("num_hidden_classes");
+    num_hclses.log();
+
+    HiddenClass* rtn = new HiddenClass(this);
+    rtn->attr_offsets[attr] = this->attributeArraySize();
+    this->children[attr] = rtn;
+    assert(rtn->attributeArraySize() == this->attributeArraySize() + 1);
+    return rtn;
+}
+
+HiddenClass* HiddenClass::getAttrwrapperChild() {
+    assert(type == NORMAL);
+    assert(attrwrapper_offset == -1);
+
+    if (!attrwrapper_child) {
+        HiddenClass* made = new HiddenClass(this);
+        made->attrwrapper_offset = this->attributeArraySize();
+        this->attrwrapper_child = made;
+        assert(made->attributeArraySize() == this->attributeArraySize() + 1);
+    }
+
+    return attrwrapper_child;
+}
+
+/**
+ * del attr from current HiddenClass, maintaining the order of the remaining attrs
+ */
+HiddenClass* HiddenClass::delAttrToMakeHC(BoxedString* attr) {
+    STAT_TIMER(t0, "us_timer_hiddenclass_delAttrToMakeHC", 0);
+
+    assert(attr->interned_state != SSTATE_NOT_INTERNED);
+    assert(type == NORMAL);
+    int idx = getOffset(attr);
+    assert(idx >= 0);
+
+    std::vector<BoxedString*> new_attrs(attributeArraySize() - 1);
+    for (auto it = attr_offsets.begin(); it != attr_offsets.end(); ++it) {
+        if (it->second < idx)
+            new_attrs[it->second] = it->first;
+        else if (it->second > idx) {
+            new_attrs[it->second - 1] = it->first;
+        }
+    }
+
+    int new_attrwrapper_offset = attrwrapper_offset;
+    if (new_attrwrapper_offset > idx)
+        new_attrwrapper_offset--;
+
+    // TODO we can first locate the parent HiddenClass of the deleted
+    // attribute and hence avoid creation of its ancestors.
+    HiddenClass* cur = root_hcls;
+    int curidx = 0;
+    for (const auto& attr : new_attrs) {
+        if (curidx == new_attrwrapper_offset)
+            cur = cur->getAttrwrapperChild();
+        else
+            cur = cur->getOrMakeChild(attr);
+        curidx++;
+    }
+    return cur;
+}
+
+} // namespace pyston

src/runtime/hiddenclass.h

+// Copyright (c) 2014-2015 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef PYSTON_RUNTIME_HIDDENCLASS_H
+#define PYSTON_RUNTIME_HIDDENCLASS_H
+
+#include <llvm/ADT/StringMap.h>
+
+#include "Python.h"
+
+#include "core/contiguous_map.h"
+#include "core/types.h"
+#include "gc/gc_alloc.h"
+
+namespace pyston {
+
+class HiddenClass : public GCAllocated<gc::GCKind::HIDDEN_CLASS> {
+public:
+    // We have a couple different storage strategies for attributes, which
+    // are distinguished by having a different hidden class type.
+    enum HCType {
+        NORMAL,      // attributes stored in attributes array, name->offset map stored in hidden class
+        DICT_BACKED, // first attribute in array is a dict-like object which stores the attributes
+        SINGLETON,   // name->offset map stored in hidden class, but hcls is mutable
+    } const type;
+
+    static HiddenClass* dict_backed;
+
+private:
+    HiddenClass(HCType type) : type(type) {}
+    HiddenClass(HiddenClass* parent)
+        : type(NORMAL), attr_offsets(parent->attr_offsets), attrwrapper_offset(parent->attrwrapper_offset) {
+        assert(parent->type == NORMAL);
+    }
+
+    // These fields only make sense for NORMAL or SINGLETON hidden classes:
+    llvm::DenseMap<BoxedString*, int> attr_offsets;
+    // If >= 0, is the offset where we stored an attrwrapper object
+    int attrwrapper_offset = -1;
+
+    // These are only for NORMAL hidden classes:
+    ContiguousMap<BoxedString*, HiddenClass*, llvm::DenseMap<BoxedString*, int>> children;
+    HiddenClass* attrwrapper_child = NULL;
+
+    // Only for SINGLETON hidden classes:
+    ICInvalidator dependent_getattrs;
+
+public:
+    static HiddenClass* makeSingleton() { return new HiddenClass(SINGLETON); }
+
+    static HiddenClass* makeRoot() {
+#ifndef NDEBUG
+        static bool made = false;
+        assert(!made);
+        made = true;
+#endif
+        return new HiddenClass(NORMAL);
+    }
+    static HiddenClass* makeDictBacked() {
+#ifndef NDEBUG
+        static bool made = false;
+        assert(!made);
+        made = true;
+#endif
+        return new HiddenClass(DICT_BACKED);
+    }
+
+    void gc_visit(GCVisitor* visitor) {
+        // Visit children even for the dict-backed case, since children will just be empty
+        visitor->visitRange((void* const*)&children.vector()[0], (void* const*)&children.vector()[children.size()]);
+        visitor->visit(attrwrapper_child);
+
+        // We don't need to visit the keys of the 'children' map, since the children should have those as entries
+        // in the attr_offssets map.
+        // Also, if we have any children, we can skip scanning our attr_offsets map, since it will be a subset
+        // of our child's map.
+        if (children.empty())
+            for (auto p : attr_offsets)
+                visitor->visit(p.first);
+    }
+
+    // The total size of the attribute array.  The slots in the attribute array may not correspond 1:1 to Python
+    // attributes.
+    int attributeArraySize() {
+        if (type == DICT_BACKED)
+            return 1;
+
+        ASSERT(type == NORMAL || type == SINGLETON, "%d", type);
+        int r = attr_offsets.size();
+        if (attrwrapper_offset != -1)
+            r += 1;
+        return r;
+    }
+
+    // The mapping from string attribute names to attribute offsets.  There may be other objects in the attributes
+    // array.
+    // Only valid for NORMAL or SINGLETON hidden classes
+    const llvm::DenseMap<BoxedString*, int>& getStrAttrOffsets() {
+        assert(type == NORMAL || type == SINGLETON);
+        return attr_offsets;
+    }
+
+    // Only valid for NORMAL hidden classes:
+    HiddenClass* getOrMakeChild(BoxedString* attr);
+
+    // Only valid for NORMAL or SINGLETON hidden classes:
+    int getOffset(BoxedString* attr) {
+        assert(type == NORMAL || type == SINGLETON);
+        auto it = attr_offsets.find(attr);
+        if (it == attr_offsets.end())
+            return -1;
+        return it->second;
+    }
+
+    int getAttrwrapperOffset() {
+        assert(type == NORMAL || type == SINGLETON);
+        return attrwrapper_offset;
+    }
+
+    // Only valid for SINGLETON hidden classes:
+    void appendAttribute(BoxedString* attr);
+    void appendAttrwrapper();
+    void delAttribute(BoxedString* attr);
+    void addDependence(Rewriter* rewriter);
+
+    // Only valid for NORMAL hidden classes:
+    HiddenClass* getAttrwrapperChild();
+
+    // Only valid for NORMAL hidden classes:
+    HiddenClass* delAttrToMakeHC(BoxedString* attr);
+};
+}
+#endif

src/runtime/import.cpp

@@ -26,6 +26,7 @@
 #include "core/ast.h"
 #include "runtime/inline/list.h"
 #include "runtime/objmodel.h"
+#include "runtime/util.h"
 
 namespace pyston {
 

src/runtime/inline/link_forcer.cpp

@@ -32,6 +32,7 @@
 #include "runtime/objmodel.h"
 #include "runtime/set.h"
 #include "runtime/types.h"
+#include "runtime/util.h"
 
 namespace pyston {
 

src/runtime/objmodel.cpp

@@ -44,6 +44,7 @@
 #include "runtime/file.h"
 #include "runtime/float.h"
 #include "runtime/generator.h"
+#include "runtime/hiddenclass.h"
 #include "runtime/ics.h"
 #include "runtime/iterobject.h"
 #include "runtime/long.h"
@@ -62,15 +63,10 @@ namespace pyston {
 using namespace pyston::ExceptionStyle;
 using pyston::ExceptionStyle::ExceptionStyle;
 
-static const std::string delitem_str("__delitem__");
-static const std::string getattribute_str("__getattribute__");
-static const std::string getattr_str("__getattr__");
-static const std::string init_str("__init__");
 static const std::string iter_str("__iter__");
 static const std::string new_str("__new__");
 static const std::string none_str("None");
 static const std::string repr_str("__repr__");
-static const std::string setitem_str("__setitem__");
 static const std::string str_str("__str__");
 
 #if 0
@@ -586,117 +582,6 @@ const char* getNameOfClass(BoxedClass* cls) {
     return cls->tp_name;
 }
 
-void HiddenClass::appendAttribute(BoxedString* attr) {
-    assert(attr->interned_state != SSTATE_NOT_INTERNED);
-    assert(type == SINGLETON);
-    dependent_getattrs.invalidateAll();
-    assert(attr_offsets.count(attr) == 0);
-    int n = this->attributeArraySize();
-    attr_offsets[attr] = n;
-}
-
-void HiddenClass::appendAttrwrapper() {
-    assert(type == SINGLETON);
-    dependent_getattrs.invalidateAll();
-    assert(attrwrapper_offset == -1);
-    attrwrapper_offset = this->attributeArraySize();
-}
-
-void HiddenClass::delAttribute(BoxedString* attr) {
-    assert(attr->interned_state != SSTATE_NOT_INTERNED);
-    assert(type == SINGLETON);
-    dependent_getattrs.invalidateAll();
-    assert(attr_offsets.count(attr));
-
-    int prev_idx = attr_offsets[attr];
-    attr_offsets.erase(attr);
-
-    for (auto it = attr_offsets.begin(), end = attr_offsets.end(); it != end; ++it) {
-        assert(it->second != prev_idx);
-        if (it->second > prev_idx)
-            it->second--;
-    }
-    if (attrwrapper_offset != -1 && attrwrapper_offset > prev_idx)
-        attrwrapper_offset--;
-}
-
-void HiddenClass::addDependence(Rewriter* rewriter) {
-    assert(type == SINGLETON);
-    rewriter->addDependenceOn(dependent_getattrs);
-}
-
-HiddenClass* HiddenClass::getOrMakeChild(BoxedString* attr) {
-    STAT_TIMER(t0, "us_timer_hiddenclass_getOrMakeChild", 0);
-
-    assert(attr->interned_state != SSTATE_NOT_INTERNED);
-    assert(type == NORMAL);
-
-    auto it = children.find(attr);
-    if (it != children.end())
-        return children.getMapped(it->second);
-
-    static StatCounter num_hclses("num_hidden_classes");
-    num_hclses.log();
-
-    HiddenClass* rtn = new HiddenClass(this);
-    rtn->attr_offsets[attr] = this->attributeArraySize();
-    this->children[attr] = rtn;
-    assert(rtn->attributeArraySize() == this->attributeArraySize() + 1);
-    return rtn;
-}
-
-HiddenClass* HiddenClass::getAttrwrapperChild() {
-    assert(type == NORMAL);
-    assert(attrwrapper_offset == -1);
-
-    if (!attrwrapper_child) {
-        HiddenClass* made = new HiddenClass(this);
-        made->attrwrapper_offset = this->attributeArraySize();
-        this->attrwrapper_child = made;
-        assert(made->attributeArraySize() == this->attributeArraySize() + 1);
-    }
-
-    return attrwrapper_child;
-}
-
-/**
- * del attr from current HiddenClass, maintaining the order of the remaining attrs
- */
-HiddenClass* HiddenClass::delAttrToMakeHC(BoxedString* attr) {
-    STAT_TIMER(t0, "us_timer_hiddenclass_delAttrToMakeHC", 0);
-
-    assert(attr->interned_state != SSTATE_NOT_INTERNED);
-    assert(type == NORMAL);
-    int idx = getOffset(attr);
-    assert(idx >= 0);
-
-    std::vector<BoxedString*> new_attrs(attributeArraySize() - 1);
-    for (auto it = attr_offsets.begin(); it != attr_offsets.end(); ++it) {
-        if (it->second < idx)
-            new_attrs[it->second] = it->first;
-        else if (it->second > idx) {
-            new_attrs[it->second - 1] = it->first;
-        }
-    }
-
-    int new_attrwrapper_offset = attrwrapper_offset;
-    if (new_attrwrapper_offset > idx)
-        new_attrwrapper_offset--;
-
-    // TODO we can first locate the parent HiddenClass of the deleted
-    // attribute and hence avoid creation of its ancestors.
-    HiddenClass* cur = root_hcls;
-    int curidx = 0;
-    for (const auto& attr : new_attrs) {
-        if (curidx == new_attrwrapper_offset)
-            cur = cur->getAttrwrapperChild();
-        else
-            cur = cur->getOrMakeChild(attr);
-        curidx++;
-    }
-    return cur;
-}
-
 size_t Box::getHCAttrsOffset() {
     assert(cls->instancesHaveHCAttrs());
 
@@ -1273,26 +1158,6 @@ Box* boxChar(char c) {
     return boxString(llvm::StringRef(d, 1));
 }
 
-static Box* noneIfNull(Box* b) {
-    if (b == NULL) {
-        return None;
-    } else {
-        return b;
-    }
-}
-
-static Box* boxStringOrNone(const char* s) {
-    if (s == NULL) {
-        return None;
-    } else {
-        return boxString(s);
-    }
-}
-
-static Box* boxStringFromCharPtr(const char* s) {
-    return boxString(s);
-}
-
 Box* dataDescriptorInstanceSpecialCases(GetattrRewriteArgs* rewrite_args, BoxedString* attr_name, Box* obj, Box* descr,
                                         RewriterVar* r_descr, bool for_call, Box** bind_obj_out,
                                         RewriterVar** r_bind_obj_out) {
@@ -2159,7 +2024,7 @@ void setattrGeneric(Box* obj, BoxedString* attr, Box* val, SetattrRewriteArgs* r
     // TODO this should be in type_setattro
     if (obj->cls == type_cls) {
         BoxedClass* cobj = static_cast<BoxedClass*>(obj);
-        if (!isUserDefined(cobj)) {
+        if (!cobj->is_user_defined) {
             raiseExcHelper(TypeError, "can't set attributes of built-in/extension type '%s'", getNameOfClass(cobj));
         }
     }
@@ -2340,11 +2205,6 @@ extern "C" void setattr(Box* obj, BoxedString* attr, Box* attr_val) {
     }
 }
 
-bool isUserDefined(BoxedClass* cls) {
-    return cls->is_user_defined;
-    // return cls->hasattrs && (cls != function_cls && cls != type_cls) && !cls->is_constant;
-}
-
 extern "C" bool nonzero(Box* obj) {
     STAT_TIMER(t0, "us_timer_slowpath_nonzero", 10);
 
@@ -2453,7 +2313,7 @@ extern "C" bool nonzero(Box* obj) {
         func = getclsattrInternal(obj, len_str, NULL);
 
     if (func == NULL) {
-        ASSERT(isUserDefined(obj->cls) || obj->cls == classobj_cls || obj->cls == type_cls
+        ASSERT(obj->cls->is_user_defined || obj->cls == classobj_cls || obj->cls == type_cls
                    || isSubclass(obj->cls, Exception) || obj->cls == file_cls || obj->cls == traceback_cls
                    || obj->cls == instancemethod_cls || obj->cls == module_cls || obj->cls == capifunc_cls
                    || obj->cls == builtin_function_or_method_cls || obj->cls == method_cls || obj->cls == frame_cls
@@ -2750,177 +2610,6 @@ extern "C" i64 unboxedLen(Box* obj) {
     return rtn;
 }
 
-extern "C" void dumpEx(void* p, int levels) {
-    printf("\n");
-    printf("Raw address: %p\n", p);
-
-    bool is_gc = gc::isValidGCMemory(p);
-    if (!is_gc) {
-        printf("non-gc memory\n");
-        return;
-    }
-
-    if (gc::isNonheapRoot(p)) {
-        printf("Non-heap GC object\n");
-
-        printf("Assuming it's a class object...\n");
-        PyTypeObject* type = (PyTypeObject*)(p);
-        printf("tp_name: %s\n", type->tp_name);
-        return;
-    }
-
-    gc::GCAllocation* al = gc::GCAllocation::fromUserData(p);
-    if (al->kind_id == gc::GCKind::UNTRACKED) {
-        printf("gc-untracked object\n");
-        return;
-    }
-
-    if (al->kind_id == gc::GCKind::PRECISE) {
-        printf("precise gc array\n");
-        return;
-    }
-
-    if (al->kind_id == gc::GCKind::CONSERVATIVE) {
-        printf("conservatively-scanned object object\n");
-        return;
-    }
-
-    if (al->kind_id == gc::GCKind::PYTHON || al->kind_id == gc::GCKind::CONSERVATIVE_PYTHON) {
-        if (al->kind_id == gc::GCKind::PYTHON)
-            printf("Python object (precisely scanned)\n");
-        else
-            printf("Python object (conservatively scanned)\n");
-        Box* b = (Box*)p;
-
-        printf("Class: %s", getFullTypeName(b).c_str());
-        if (b->cls->cls != type_cls) {
-            printf(" (metaclass: %s)\n", getFullTypeName(b->cls).c_str());
-        } else {
-            printf("\n");
-        }
-
-        if (b->cls == bool_cls) {
-            printf("The %s object\n", b == True ? "True" : "False");
-        }
-
-        if (isSubclass(b->cls, type_cls)) {
-            auto cls = static_cast<BoxedClass*>(b);
-            printf("Type name: %s\n", getFullNameOfClass(cls).c_str());
-
-            printf("MRO:");
-
-            if (cls->tp_mro && cls->tp_mro->cls == tuple_cls) {
-                bool first = true;
-                for (auto b : *static_cast<BoxedTuple*>(cls->tp_mro)) {
-                    if (!first)
-                        printf(" ->");
-                    first = false;
-                    printf(" %s", getFullNameOfClass(static_cast<BoxedClass*>(b)).c_str());
-                }
-            }
-            printf("\n");
-        }
-
-        if (isSubclass(b->cls, str_cls)) {
-            printf("String value: %s\n", static_cast<BoxedString*>(b)->data());
-        }
-
-        if (isSubclass(b->cls, tuple_cls)) {
-            BoxedTuple* t = static_cast<BoxedTuple*>(b);
-            printf("%ld elements\n", t->size());
-
-            if (levels > 0) {
-                int i = 0;
-                for (auto e : *t) {
-                    printf("\nElement %d:", i);
-                    i++;
-                    dumpEx(e, levels - 1);
-                }
-            }
-        }
-
-        if (isSubclass(b->cls, dict_cls)) {
-            BoxedDict* d = static_cast<BoxedDict*>(b);
-            printf("%ld elements\n", d->d.size());
-
-            if (levels > 0) {
-                int i = 0;
-                for (auto t : d->d) {
-                    printf("\nKey:");
-                    dumpEx(t.first, levels - 1);
-                    printf("Value:");
-                    dumpEx(t.second, levels - 1);
-                }
-            }
-        }
-
-        if (isSubclass(b->cls, int_cls)) {
-            printf("Int value: %ld\n", static_cast<BoxedInt*>(b)->n);
-        }
-
-        if (isSubclass(b->cls, list_cls)) {
-            auto l = static_cast<BoxedList*>(b);
-            printf("%ld elements\n", l->size);
-
-            if (levels > 0) {
-                int i = 0;
-                for (int i = 0; i < l->size; i++) {
-                    printf("\nElement %d:", i);
-                    dumpEx(l->elts->elts[i], levels - 1);
-                }
-            }
-        }
-
-        if (isSubclass(b->cls, function_cls)) {
-            BoxedFunction* f = static_cast<BoxedFunction*>(b);
-
-            CLFunction* cl = f->f;
-            if (cl->source) {
-                printf("User-defined function '%s'\n", cl->source->getName().data());
-            } else {
-                printf("A builtin function\n");
-            }
-
-            printf("Has %ld function versions\n", cl->versions.size());
-            for (CompiledFunction* cf : cl->versions) {
-                bool got_name;
-                std::string name = g.func_addr_registry.getFuncNameAtAddress(cf->code, true, &got_name);
-                if (got_name)
-                    printf("%s\n", name.c_str());
-                else
-                    printf("%p\n", cf->code);
-            }
-        }
-
-        if (isSubclass(b->cls, module_cls)) {
-            printf("The '%s' module\n", static_cast<BoxedModule*>(b)->name().c_str());
-        }
-
-        /*
-        if (b->cls->instancesHaveHCAttrs()) {
-            HCAttrs* attrs = b->getHCAttrsPtr();
-            printf("Has %ld attrs\n", attrs->hcls->attr_offsets.size());
-            for (const auto& p : attrs->hcls->attr_offsets) {
-                printf("Index %d: %s: %p\n", p.second, p.first.c_str(), attrs->attr_list->attrs[p.second]);
-            }
-        }
-        */
-
-        return;
-    }
-
-    if (al->kind_id == gc::GCKind::HIDDEN_CLASS) {
-        printf("Hidden class object\n");
-        return;
-    }
-
-    RELEASE_ASSERT(0, "%d", (int)al->kind_id);
-}
-
-extern "C" void dump(void* p) {
-    dumpEx(p, 0);
-}
-
 // For rewriting purposes, this function assumes that nargs will be constant.
 // That's probably fine for some uses (ex binops), but otherwise it should be guarded on beforehand.
 extern "C" Box* callattrInternal(Box* obj, BoxedString* attr, LookupScope scope, CallRewriteArgs* rewrite_args,
@@ -4211,7 +3900,7 @@ extern "C" Box* binopInternal(Box* lhs, Box* rhs, int op_type, bool inplace, Bin
 
 extern "C" Box* binop(Box* lhs, Box* rhs, int op_type) {
     STAT_TIMER(t0, "us_timer_slowpath_binop", 10);
-    bool can_patchpoint = !isUserDefined(lhs->cls) && !isUserDefined(rhs->cls);
+    bool can_patchpoint = !lhs->cls->is_user_defined && !rhs->cls->is_user_defined;
 #if 0
     static uint64_t* st_id = Stats::getStatCounter("us_timer_slowpath_binop_patchable");
     static uint64_t* st_id_nopatch = Stats::getStatCounter("us_timer_slowpath_binop_nopatch");
@@ -4268,7 +3957,7 @@ extern "C" Box* augbinop(Box* lhs, Box* rhs, int op_type) {
     // resolving it one way right now (ex, using the value from lhs.__add__) means that later
     // we'll resolve it the same way, even for the same argument types.
     // TODO implement full resolving semantics inside the rewrite?
-    bool can_patchpoint = !isUserDefined(lhs->cls) && !isUserDefined(rhs->cls);
+    bool can_patchpoint = !lhs->cls->is_user_defined && !rhs->cls->is_user_defined;
     if (can_patchpoint)
         rewriter.reset(
             Rewriter::createRewriter(__builtin_extract_return_addr(__builtin_return_address(0)), 3, "binop"));
@@ -4427,7 +4116,7 @@ Box* compareInternal(Box* lhs, Box* rhs, int op_type, CompareRewriteArgs* rewrit
         return boxBool(b);
     }
 
-    bool any_user_defined = isUserDefined(lhs->cls) || isUserDefined(rhs->cls);
+    bool any_user_defined = lhs->cls->is_user_defined || rhs->cls->is_user_defined;
     if (any_user_defined) {
         rewrite_args = NULL;
         REWRITE_ABORTED("");
@@ -4476,7 +4165,7 @@ Box* compareInternal(Box* lhs, Box* rhs, int op_type, CompareRewriteArgs* rewrit
         // simplified by using the assumption that tp_richcompare exists and never returns NotImplemented
         // for builtin types when both arguments are the right type.
 
-        assert(!isUserDefined(lhs->cls));
+        assert(!lhs->cls->is_user_defined);
 
         Box* r = lhs->cls->tp_richcompare(lhs, rhs, cpython_op_type);
         RELEASE_ASSERT(r != NotImplemented, "%s returned notimplemented?", lhs->cls->tp_name);
@@ -5052,7 +4741,7 @@ extern "C" void delattr(Box* obj, BoxedString* attr) {
 
     if (obj->cls == type_cls) {
         BoxedClass* cobj = static_cast<BoxedClass*>(obj);
-        if (!isUserDefined(cobj)) {
+        if (!cobj->is_user_defined) {
             raiseExcHelper(TypeError, "can't set attributes of built-in/extension type '%s'\n", getNameOfClass(cobj));
         }
     }
@@ -5131,10 +4820,6 @@ Box* getiter(Box* o) {
     return getiterHelper(o);
 }
 
-extern "C" bool hasnext(Box* o) {
-    return o->cls->tpp_hasnext(o);
-}
-
 llvm::iterator_range<BoxIterator> Box::pyElements() {
     return BoxIterator::getRange(this);
 }
@@ -5660,19 +5345,6 @@ extern "C" Box* importStar(Box* _from_module, Box* to_globals) {
     return None;
 }
 
-Box* coerceUnicodeToStr(Box* unicode) {
-    if (!isSubclass(unicode->cls, unicode_cls))
-        return unicode;
-
-    Box* r = PyUnicode_AsASCIIString(unicode);
-    if (!r) {
-        PyErr_Clear();
-        raiseExcHelper(TypeError, "Cannot use non-ascii unicode strings as attribute names or keywords");
-    }
-
-    return r;
-}
-
 // TODO Make these fast, do inline caches and stuff
 
 extern "C" void boxedLocalsSet(Box* boxedLocals, BoxedString* attr, Box* val) {

src/runtime/objmodel.h

@@ -99,10 +99,6 @@ Box* getiter(Box* o);
 extern "C" Box* getPystonIter(Box* o);
 extern "C" Box* getiterHelper(Box* o);
 extern "C" Box* createBoxedIterWrapperIfNeeded(Box* o);
-extern "C" bool hasnext(Box* o);
-
-extern "C" void dump(void* p);
-extern "C" void dumpEx(void* p, int levels = 0);
 
 struct SetattrRewriteArgs;
 void setattrGeneric(Box* obj, BoxedString* attr, Box* val, SetattrRewriteArgs* rewrite_args);
@@ -165,7 +161,6 @@ extern "C" void raiseIndexErrorStr(const char* typeName) __attribute__((__noretu
 
 Box* typeCall(Box*, BoxedTuple*, BoxedDict*);
 Box* typeNew(Box* cls, Box* arg1, Box* arg2, Box** _args);
-bool isUserDefined(BoxedClass* cls);
 
 // These process a potential descriptor, differing in their behavior if the object was not a descriptor.
 // the OrNull variant returns NULL to signify it wasn't a descriptor, and the processDescriptor version
@@ -186,16 +181,6 @@ static const char* objectNewParameterTypeErrorMsg() {
     }
 }
 
-// This function will ascii-encode any unicode objects it gets passed, or return the argument
-// unmodified if it wasn't a unicode object.
-// This is intended for functions that deal with attribute or variable names, which we internally
-// assume will always be strings, but CPython lets be unicode.
-// If we used an encoding like utf8 instead of ascii, we would allow collisions between unicode
-// strings and a string that happens to be its encoding.  It seems safer to just encode as ascii,
-// which will throw an exception if you try to pass something that might run into this risk.
-// (We wrap the unicode error and throw a TypeError)
-Box* coerceUnicodeToStr(Box* unicode);
-
 inline std::tuple<Box*, Box*, Box*, Box**> getTupleFromArgsArray(Box** args, int num_args) {
     Box* arg1 = num_args >= 1 ? args[0] : nullptr;
     Box* arg2 = num_args >= 2 ? args[1] : nullptr;

src/runtime/types.cpp

@@ -38,6 +38,7 @@
 #include "runtime/complex.h"
 #include "runtime/dict.h"
 #include "runtime/file.h"
+#include "runtime/hiddenclass.h"
 #include "runtime/ics.h"
 #include "runtime/iterobject.h"
 #include "runtime/list.h"
@@ -1832,7 +1833,7 @@ Box* typeRepr(BoxedClass* self) {
     std::string O("");
     llvm::raw_string_ostream os(O);
 
-    if ((self->tp_flags & Py_TPFLAGS_HEAPTYPE) && isUserDefined(self))
+    if ((self->tp_flags & Py_TPFLAGS_HEAPTYPE) && self->is_user_defined)
         os << "<class '";
     else
         os << "<type '";

src/runtime/types.h

@@ -327,123 +327,6 @@ static_assert(offsetof(pyston::BoxedHeapClass, as_sequence) == offsetof(PyHeapTy
 static_assert(offsetof(pyston::BoxedHeapClass, as_buffer) == offsetof(PyHeapTypeObject, as_buffer), "");
 static_assert(sizeof(pyston::BoxedHeapClass) == sizeof(PyHeapTypeObject), "");
 
-
-class HiddenClass : public GCAllocated<gc::GCKind::HIDDEN_CLASS> {
-public:
-    // We have a couple different storage strategies for attributes, which
-    // are distinguished by having a different hidden class type.
-    enum HCType {
-        NORMAL,      // attributes stored in attributes array, name->offset map stored in hidden class
-        DICT_BACKED, // first attribute in array is a dict-like object which stores the attributes
-        SINGLETON,   // name->offset map stored in hidden class, but hcls is mutable
-    } const type;
-
-    static HiddenClass* dict_backed;
-
-private:
-    HiddenClass(HCType type) : type(type) {}
-    HiddenClass(HiddenClass* parent)
-        : type(NORMAL), attr_offsets(parent->attr_offsets), attrwrapper_offset(parent->attrwrapper_offset) {
-        assert(parent->type == NORMAL);
-    }
-
-    // These fields only make sense for NORMAL or SINGLETON hidden classes:
-    llvm::DenseMap<BoxedString*, int> attr_offsets;
-    // If >= 0, is the offset where we stored an attrwrapper object
-    int attrwrapper_offset = -1;
-
-    // These are only for NORMAL hidden classes:
-    ContiguousMap<BoxedString*, HiddenClass*, llvm::DenseMap<BoxedString*, int>> children;
-    HiddenClass* attrwrapper_child = NULL;
-
-    // Only for SINGLETON hidden classes:
-    ICInvalidator dependent_getattrs;
-
-public:
-    static HiddenClass* makeSingleton() { return new HiddenClass(SINGLETON); }
-
-    static HiddenClass* makeRoot() {
-#ifndef NDEBUG
-        static bool made = false;
-        assert(!made);
-        made = true;
-#endif
-        return new HiddenClass(NORMAL);
-    }
-    static HiddenClass* makeDictBacked() {
-#ifndef NDEBUG
-        static bool made = false;
-        assert(!made);
-        made = true;
-#endif
-        return new HiddenClass(DICT_BACKED);
-    }
-
-    void gc_visit(GCVisitor* visitor) {
-        // Visit children even for the dict-backed case, since children will just be empty
-        visitor->visitRange((void* const*)&children.vector()[0], (void* const*)&children.vector()[children.size()]);
-        visitor->visit(attrwrapper_child);
-
-        // We don't need to visit the keys of the 'children' map, since the children should have those as entries
-        // in the attr_offssets map.
-        // Also, if we have any children, we can skip scanning our attr_offsets map, since it will be a subset
-        // of our child's map.
-        if (children.empty())
-            for (auto p : attr_offsets)
-                visitor->visit(p.first);
-    }
-
-    // The total size of the attribute array.  The slots in the attribute array may not correspond 1:1 to Python
-    // attributes.
-    int attributeArraySize() {
-        if (type == DICT_BACKED)
-            return 1;
-
-        ASSERT(type == NORMAL || type == SINGLETON, "%d", type);
-        int r = attr_offsets.size();
-        if (attrwrapper_offset != -1)
-            r += 1;
-        return r;
-    }
-
-    // The mapping from string attribute names to attribute offsets.  There may be other objects in the attributes
-    // array.
-    // Only valid for NORMAL or SINGLETON hidden classes
-    const llvm::DenseMap<BoxedString*, int>& getStrAttrOffsets() {
-        assert(type == NORMAL || type == SINGLETON);
-        return attr_offsets;
-    }
-
-    // Only valid for NORMAL hidden classes:
-    HiddenClass* getOrMakeChild(BoxedString* attr);
-
-    // Only valid for NORMAL or SINGLETON hidden classes:
-    int getOffset(BoxedString* attr) {
-        assert(type == NORMAL || type == SINGLETON);
-        auto it = attr_offsets.find(attr);
-        if (it == attr_offsets.end())
-            return -1;
-        return it->second;
-    }
-
-    int getAttrwrapperOffset() {
-        assert(type == NORMAL || type == SINGLETON);
-        return attrwrapper_offset;
-    }
-
-    // Only valid for SINGLETON hidden classes:
-    void appendAttribute(BoxedString* attr);
-    void appendAttrwrapper();
-    void delAttribute(BoxedString* attr);
-    void addDependence(Rewriter* rewriter);
-
-    // Only valid for NORMAL hidden classes:
-    HiddenClass* getAttrwrapperChild();
-
-    // Only valid for NORMAL hidden classes:
-    HiddenClass* delAttrToMakeHC(BoxedString* attr);
-};
-
 class BoxedInt : public Box {
 public:
     int64_t n;

src/runtime/util.cpp

@@ -14,6 +14,7 @@
 
 #include "runtime/util.h"
 
+#include "codegen/codegen.h"
 #include "core/options.h"
 #include "core/types.h"
 #include "runtime/objmodel.h"
@@ -71,4 +72,212 @@ void boundSliceWithLength(i64* start_out, i64* stop_out, i64 start, i64 stop, i6
     *start_out = start;
     *stop_out = stop;
 }
+
+Box* boxStringOrNone(const char* s) {
+    if (s == NULL) {
+        return None;
+    } else {
+        return boxString(s);
+    }
+}
+
+Box* noneIfNull(Box* b) {
+    if (b == NULL) {
+        return None;
+    } else {
+        return b;
+    }
+}
+
+Box* coerceUnicodeToStr(Box* unicode) {
+    if (!isSubclass(unicode->cls, unicode_cls))
+        return unicode;
+
+    Box* r = PyUnicode_AsASCIIString(unicode);
+    if (!r) {
+        PyErr_Clear();
+        raiseExcHelper(TypeError, "Cannot use non-ascii unicode strings as attribute names or keywords");
+    }
+
+    return r;
+}
+
+Box* boxStringFromCharPtr(const char* s) {
+    return boxString(s);
+}
+
+extern "C" bool hasnext(Box* o) {
+    return o->cls->tpp_hasnext(o);
+}
+
+extern "C" void dump(void* p) {
+    dumpEx(p, 0);
+}
+
+extern "C" void dumpEx(void* p, int levels) {
+    printf("\n");
+    printf("Raw address: %p\n", p);
+
+    bool is_gc = gc::isValidGCMemory(p);
+    if (!is_gc) {
+        printf("non-gc memory\n");
+        return;
+    }
+
+    if (gc::isNonheapRoot(p)) {
+        printf("Non-heap GC object\n");
+
+        printf("Assuming it's a class object...\n");
+        PyTypeObject* type = (PyTypeObject*)(p);
+        printf("tp_name: %s\n", type->tp_name);
+        return;
+    }
+
+    gc::GCAllocation* al = gc::GCAllocation::fromUserData(p);
+    if (al->kind_id == gc::GCKind::UNTRACKED) {
+        printf("gc-untracked object\n");
+        return;
+    }
+
+    if (al->kind_id == gc::GCKind::PRECISE) {
+        printf("precise gc array\n");
+        return;
+    }
+
+    if (al->kind_id == gc::GCKind::CONSERVATIVE) {
+        printf("conservatively-scanned object object\n");
+        return;
+    }
+
+    if (al->kind_id == gc::GCKind::PYTHON || al->kind_id == gc::GCKind::CONSERVATIVE_PYTHON) {
+        if (al->kind_id == gc::GCKind::PYTHON)
+            printf("Python object (precisely scanned)\n");
+        else
+            printf("Python object (conservatively scanned)\n");
+        Box* b = (Box*)p;
+
+        printf("Class: %s", getFullTypeName(b).c_str());
+        if (b->cls->cls != type_cls) {
+            printf(" (metaclass: %s)\n", getFullTypeName(b->cls).c_str());
+        } else {
+            printf("\n");
+        }
+
+        if (b->cls == bool_cls) {
+            printf("The %s object\n", b == True ? "True" : "False");
+        }
+
+        if (isSubclass(b->cls, type_cls)) {
+            auto cls = static_cast<BoxedClass*>(b);
+            printf("Type name: %s\n", getFullNameOfClass(cls).c_str());
+
+            printf("MRO:");
+
+            if (cls->tp_mro && cls->tp_mro->cls == tuple_cls) {
+                bool first = true;
+                for (auto b : *static_cast<BoxedTuple*>(cls->tp_mro)) {
+                    if (!first)
+                        printf(" ->");
+                    first = false;
+                    printf(" %s", getFullNameOfClass(static_cast<BoxedClass*>(b)).c_str());
+                }
+            }
+            printf("\n");
+        }
+
+        if (isSubclass(b->cls, str_cls)) {
+            printf("String value: %s\n", static_cast<BoxedString*>(b)->data());
+        }
+
+        if (isSubclass(b->cls, tuple_cls)) {
+            BoxedTuple* t = static_cast<BoxedTuple*>(b);
+            printf("%ld elements\n", t->size());
+
+            if (levels > 0) {
+                int i = 0;
+                for (auto e : *t) {
+                    printf("\nElement %d:", i);
+                    i++;
+                    dumpEx(e, levels - 1);
+                }
+            }
+        }
+
+        if (isSubclass(b->cls, dict_cls)) {
+            BoxedDict* d = static_cast<BoxedDict*>(b);
+            printf("%ld elements\n", d->d.size());
+
+            if (levels > 0) {
+                int i = 0;
+                for (auto t : d->d) {
+                    printf("\nKey:");
+                    dumpEx(t.first, levels - 1);
+                    printf("Value:");
+                    dumpEx(t.second, levels - 1);
+                }
+            }
+        }
+
+        if (isSubclass(b->cls, int_cls)) {
+            printf("Int value: %ld\n", static_cast<BoxedInt*>(b)->n);
+        }
+
+        if (isSubclass(b->cls, list_cls)) {
+            auto l = static_cast<BoxedList*>(b);
+            printf("%ld elements\n", l->size);
+
+            if (levels > 0) {
+                int i = 0;
+                for (int i = 0; i < l->size; i++) {
+                    printf("\nElement %d:", i);
+                    dumpEx(l->elts->elts[i], levels - 1);
+                }
+            }
+        }
+
+        if (isSubclass(b->cls, function_cls)) {
+            BoxedFunction* f = static_cast<BoxedFunction*>(b);
+
+            CLFunction* cl = f->f;
+            if (cl->source) {
+                printf("User-defined function '%s'\n", cl->source->getName().data());
+            } else {
+                printf("A builtin function\n");
+            }
+
+            printf("Has %ld function versions\n", cl->versions.size());
+            for (CompiledFunction* cf : cl->versions) {
+                bool got_name;
+                std::string name = g.func_addr_registry.getFuncNameAtAddress(cf->code, true, &got_name);
+                if (got_name)
+                    printf("%s\n", name.c_str());
+                else
+                    printf("%p\n", cf->code);
+            }
+        }
+
+        if (isSubclass(b->cls, module_cls)) {
+            printf("The '%s' module\n", static_cast<BoxedModule*>(b)->name().c_str());
+        }
+
+        /*
+        if (b->cls->instancesHaveHCAttrs()) {
+            HCAttrs* attrs = b->getHCAttrsPtr();
+            printf("Has %ld attrs\n", attrs->hcls->attr_offsets.size());
+            for (const auto& p : attrs->hcls->attr_offsets) {
+                printf("Index %d: %s: %p\n", p.second, p.first.c_str(), attrs->attr_list->attrs[p.second]);
+            }
+        }
+        */
+
+        return;
+    }
+
+    if (al->kind_id == gc::GCKind::HIDDEN_CLASS) {
+        printf("Hidden class object\n");
+        return;
+    }
+
+    RELEASE_ASSERT(0, "%d", (int)al->kind_id);
+}
 }

src/runtime/util.h

@@ -44,6 +44,24 @@ void adjustNegativeIndicesOnObject(Box* obj, i64* start, i64* stop);
 // Ensure stop >= start and remain within bounds.
 void boundSliceWithLength(i64* start_out, i64* stop_out, i64 start, i64 stop, i64 size);
 
+Box* noneIfNull(Box* b);
+Box* boxStringOrNone(const char* s);
+Box* boxStringFromCharPtr(const char* s);
+
+// This function will ascii-encode any unicode objects it gets passed, or return the argument
+// unmodified if it wasn't a unicode object.
+// This is intended for functions that deal with attribute or variable names, which we internally
+// assume will always be strings, but CPython lets be unicode.
+// If we used an encoding like utf8 instead of ascii, we would allow collisions between unicode
+// strings and a string that happens to be its encoding.  It seems safer to just encode as ascii,
+// which will throw an exception if you try to pass something that might run into this risk.
+// (We wrap the unicode error and throw a TypeError)
+Box* coerceUnicodeToStr(Box* unicode);
+
+extern "C" bool hasnext(Box* o);
+extern "C" void dump(void* p);
+extern "C" void dumpEx(void* p, int levels = 0);
+
 template <typename T> void copySlice(T* __restrict__ dst, const T* __restrict__ src, i64 start, i64 step, i64 length) {
     assert(dst != src);
     if (step == 1) {