Merge pull request #919 from kmod/sqlalchemy_merge

Misc fixes for sqlalchemy

src/analysis/type_analysis.cpp

@@ -33,7 +33,7 @@
 #include "runtime/types.h"
 
 //#undef VERBOSITY
-//#define VERBOSITY(x) 2
+//#define VERBOSITY(x) 4
 
 namespace pyston {
 
@@ -116,9 +116,11 @@ private:
         if (speculated_cls != NULL && speculated_cls->is_constant) {
             ConcreteCompilerType* speculated_type = unboxedType(typeFromClass(speculated_cls));
             if (VERBOSITY() >= 2) {
-                printf("in propagator, speculating that %s would actually be %s, at:\n", old_type->debugName().c_str(),
+                printf("in propagator, speculating that %s would actually be %s, at ", old_type->debugName().c_str(),
                        speculated_type->debugName().c_str());
+                fflush(stdout);
                 print_ast(node);
+                llvm::outs().flush();
                 printf("\n");
             }
 
@@ -152,8 +154,9 @@ private:
         CompilerType* rtn = static_cast<CompilerType*>(raw_rtn);
 
         if (VERBOSITY() >= 3) {
+            printf("Type of ");
             print_ast(node);
-            printf(" %s\n", rtn->debugName().c_str());
+            printf(" is %s\n", rtn->debugName().c_str());
         }
 
         expr_types[node] = rtn;
@@ -428,10 +431,6 @@ private:
             return UNKNOWN;
         }
 
-        if (name_scope == ScopeInfo::VarScopeType::CLOSURE) {
-            return UNKNOWN;
-        }
-
         if (name_scope == ScopeInfo::VarScopeType::NAME) {
             return UNKNOWN;
         }
@@ -440,7 +439,7 @@ private:
             return UNKNOWN;
         }
 
-        if (name_scope == ScopeInfo::VarScopeType::FAST) {
+        if (name_scope == ScopeInfo::VarScopeType::FAST || name_scope == ScopeInfo::VarScopeType::CLOSURE) {
             CompilerType*& t = sym_table[node->id];
             if (t == NULL) {
                 // if (VERBOSITY() >= 2) {

src/capi/typeobject.cpp

@@ -977,6 +977,27 @@ Box* slotTpGetattrHookInternal(Box* self, BoxedString* name, GetattrRewriteArgs*
                 rewrite_args->out_rtn = grewrite_args.out_rtn;
                 rewrite_args->out_return_convention = grewrite_args.out_return_convention;
             }
+
+            // Guarding here is a bit tricky, since we need to make sure that we call getattr
+            // (or not) at the right times.
+            // Right now this section is a bit conservative.
+            if (rewrite_args) {
+                if (grewrite_args.out_return_convention == GetattrRewriteArgs::NO_RETURN) {
+                    // Do nothing
+                } else if (grewrite_args.out_return_convention == GetattrRewriteArgs::VALID_RETURN) {
+                    // TODO we should have a HAS_RETURN that takes out the NULL case
+                    assert(res);
+                    if (res)
+                        grewrite_args.out_rtn->addGuardNotEq(0);
+                    else
+                        grewrite_args.out_rtn->addGuard(0);
+                } else if (grewrite_args.out_return_convention == GetattrRewriteArgs::NOEXC_POSSIBLE) {
+                    // TODO maybe we could handle this
+                    rewrite_args = NULL;
+                } else {
+                    RELEASE_ASSERT(0, "%d", grewrite_args.out_return_convention);
+                }
+            }
         } else {
             try {
                 res = getattrInternalGeneric(self, name, NULL, false, false, NULL, NULL);

src/codegen/irgen/irgenerator.cpp

@@ -1170,7 +1170,7 @@ private:
             llvm::Value* r = emitter.createCall3(unw_info, g.funcs.boxedLocalsGet, boxedLocals, attr, module);
             return new ConcreteCompilerVariable(UNKNOWN, r, true);
         } else {
-            // vst is one of {FAST, CLOSURE, NAME}
+            // vst is one of {FAST, CLOSURE}
             if (symbol_table.find(node->id) == symbol_table.end()) {
                 // TODO should mark as DEAD here, though we won't end up setting all the names appropriately
                 // state = DEAD;
@@ -1541,14 +1541,24 @@ private:
             if (VERBOSITY("irgen") >= 2) {
                 printf("Speculating that %s is actually %s, at ", rtn->getConcreteType()->debugName().c_str(),
                        speculated_type->debugName().c_str());
-                PrintVisitor printer;
-                node->accept(&printer);
+                fflush(stdout);
+                print_ast(node);
+                llvm::outs().flush();
                 printf("\n");
             }
 
+#ifndef NDEBUG
             // That's not really a speculation.... could potentially handle this here, but
             // I think it's better to just not generate bad speculations:
-            assert(!rtn->canConvertTo(speculated_type));
+            if (rtn->canConvertTo(speculated_type)) {
+                auto source = irstate->getSourceInfo();
+                printf("On %s:%d, function %s:\n", source->getFn()->c_str(), source->body[0]->lineno,
+                       source->getName()->c_str());
+                irstate->getSourceInfo()->cfg->print();
+            }
+            RELEASE_ASSERT(!rtn->canConvertTo(speculated_type), "%s %s", rtn->getType()->debugName().c_str(),
+                           speculated_type->debugName().c_str());
+#endif
 
             ConcreteCompilerVariable* old_rtn = rtn->makeConverted(emitter, UNKNOWN);
             rtn->decvref(emitter);
@@ -2003,7 +2013,7 @@ private:
         ConcreteCompilerVariable* dest = NULL;
         if (node->dest) {
             auto d = evalExpr(node->dest, unw_info);
-            dest = d->makeConverted(emitter, d->getConcreteType());
+            dest = d->makeConverted(emitter, d->getBoxType());
             d->decvref(emitter);
         } else {
             llvm::Value* sys_stdout_val = emitter.createCall(unw_info, g.funcs.getSysStdout);
@@ -2012,63 +2022,22 @@ private:
         }
         assert(dest);
 
-        static BoxedString* write_str = internStringImmortal("write");
-        static BoxedString* newline_str = internStringImmortal("\n");
-        static BoxedString* space_str = internStringImmortal(" ");
+        assert(node->values.size() <= 1);
+        ConcreteCompilerVariable* converted;
 
-        // TODO: why are we inline-generating all this code instead of just emitting a call to some runtime function?
-        // (=printHelper())
-        int nvals = node->values.size();
-        for (int i = 0; i < nvals; i++) {
-            CompilerVariable* var = evalExpr(node->values[i], unw_info);
-            ConcreteCompilerVariable* converted = var->makeConverted(emitter, var->getBoxType());
+        if (node->values.size() == 1) {
+            CompilerVariable* var = evalExpr(node->values[0], unw_info);
+            converted = var->makeConverted(emitter, var->getBoxType());
             var->decvref(emitter);
-
-            // begin code for handling of softspace
-            bool new_softspace = (i < nvals - 1) || (!node->nl);
-            llvm::Value* dospace = emitter.createCall(unw_info, g.funcs.softspace,
-                                                      { dest->getValue(), getConstantInt(new_softspace, g.i1) });
-            assert(dospace->getType() == g.i1);
-
-            llvm::BasicBlock* ss_block = llvm::BasicBlock::Create(g.context, "softspace", irstate->getLLVMFunction());
-            llvm::BasicBlock* join_block = llvm::BasicBlock::Create(g.context, "print", irstate->getLLVMFunction());
-
-            emitter.getBuilder()->CreateCondBr(dospace, ss_block, join_block);
-
-            curblock = ss_block;
-            emitter.getBuilder()->SetInsertPoint(ss_block);
-            CallattrFlags flags = {.cls_only = false, .null_on_nonexistent = false, .argspec = ArgPassSpec(1) };
-            auto r = dest->callattr(emitter, getOpInfoForNode(node, unw_info), write_str, flags, { makeStr(space_str) },
-                                    NULL);
-            r->decvref(emitter);
-
-            emitter.getBuilder()->CreateBr(join_block);
-            curblock = join_block;
-            emitter.getBuilder()->SetInsertPoint(join_block);
-            // end code for handling of softspace
-
-
-            llvm::Value* v = emitter.createCall(unw_info, g.funcs.strOrUnicode, converted->getValue());
-            v = emitter.getBuilder()->CreateBitCast(v, g.llvm_value_type_ptr);
-            auto s = new ConcreteCompilerVariable(STR, v, true);
-            r = dest->callattr(emitter, getOpInfoForNode(node, unw_info), write_str, flags, { s }, NULL);
-            s->decvref(emitter);
-            r->decvref(emitter);
-            converted->decvref(emitter);
+        } else {
+            converted = new ConcreteCompilerVariable(UNKNOWN, getNullPtr(g.llvm_value_type_ptr), true);
         }
 
-        if (node->nl) {
-            CallattrFlags flags = {.cls_only = false, .null_on_nonexistent = false, .argspec = ArgPassSpec(1) };
-            auto r = dest->callattr(emitter, getOpInfoForNode(node, unw_info), write_str, flags,
-                                    { makeStr(newline_str) }, NULL);
-            r->decvref(emitter);
-
-            if (nvals == 0) {
-                emitter.createCall(unw_info, g.funcs.softspace, { dest->getValue(), getConstantInt(0, g.i1) });
-            }
-        }
+        emitter.createCall3(unw_info, g.funcs.printHelper, dest->getValue(), converted->getValue(),
+                            getConstantInt(node->nl, g.i1));
 
         dest->decvref(emitter);
+        converted->decvref(emitter);
     }
 
     void doReturn(AST_Return* node, const UnwindInfo& unw_info) {

src/codegen/irgen/util.cpp

@@ -243,7 +243,7 @@ public:
                         if (!lookup_success) {
                             llvm::Constant* int_val
                                 = llvm::ConstantInt::get(g.i64, reinterpret_cast<uintptr_t>(addr), false);
-                            llvm::Constant* ptr_val = llvm::ConstantExpr::getIntToPtr(int_val, g.i8);
+                            llvm::Constant* ptr_val = llvm::ConstantExpr::getIntToPtr(int_val, g.i8_ptr);
                             ii->setArgOperand(i, ptr_val);
                             continue;
                         } else {

src/codegen/runtime_hooks.cpp

@@ -240,6 +240,7 @@ void initGlobalFuncs(GlobalState& g) {
     GET(assertFailDerefNameDefined);
     GET(assertFail);
     GET(printExprHelper);
+    GET(printHelper);
 
     GET(printFloat);
     GET(listAppendInternal);

src/codegen/runtime_hooks.h

@@ -42,7 +42,7 @@ struct GlobalFuncs {
 
     llvm::Value* unpackIntoArray, *raiseAttributeError, *raiseAttributeErrorStr, *raiseAttributeErrorCapi,
         *raiseAttributeErrorStrCapi, *raiseNotIterableError, *raiseIndexErrorStr, *raiseIndexErrorStrCapi,
-        *assertNameDefined, *assertFail, *assertFailDerefNameDefined, *printExprHelper;
+        *assertNameDefined, *assertFail, *assertFailDerefNameDefined, *printExprHelper, *printHelper;
     llvm::Value* printFloat, *listAppendInternal, *getSysStdout;
     ExceptionSwitchable<llvm::Value*> runtimeCall0, runtimeCall1, runtimeCall2, runtimeCall3, runtimeCall, runtimeCallN;
     ExceptionSwitchable<llvm::Value*> callattr0, callattr1, callattr2, callattr3, callattr, callattrN;

src/core/ast.cpp

@@ -1056,6 +1056,7 @@ void* AST_MakeClass::accept_expr(ExprVisitor* v) {
 void print_ast(AST* ast) {
     PrintVisitor v;
     ast->accept(&v);
+    v.flush();
 }
 
 void PrintVisitor::printIndent() {

src/core/ast.h

@@ -1338,6 +1338,9 @@ private:
 public:
     PrintVisitor(int indent = 0, llvm::raw_ostream& stream = llvm::outs()) : stream(stream), indent(indent) {}
     virtual ~PrintVisitor() {}
+    void flush() {
+        stream.flush();
+    }
 
     virtual bool visit_alias(AST_alias* node);
     virtual bool visit_arguments(AST_arguments* node);

src/core/types.h

@@ -527,10 +527,13 @@ public:
 
     llvm::iterator_range<BoxIterator> pyElements();
 
+    // For instances with hc attrs:
     size_t getHCAttrsOffset();
     HCAttrs* getHCAttrsPtr();
-    void setDict(BoxedDict* d);
+    void setDictBacked(Box* d);
+    // For instances with dict attrs:
     BoxedDict* getDict();
+    void setDict(BoxedDict* d);
 
 
     void setattr(BoxedString* attr, Box* val, SetattrRewriteArgs* rewrite_args);

src/gc/collector.cpp

@@ -602,6 +602,7 @@ static void orderFinalizers() {
     std::vector<Box*> finalizer_marked;
 
     for (Box* obj : objects_with_ordered_finalizers) {
+        GC_TRACE_LOG("%p has an ordered finalizer\n", obj);
         GCAllocation* al = GCAllocation::fromUserData(obj);
 
         // We are only interested in object with finalizers that need to be garbage-collected.
@@ -920,9 +921,10 @@ static void openTraceFp(bool is_pre) {
         fclose(trace_fp);
 
     char tracefn_buf[80];
-    snprintf(tracefn_buf, sizeof(tracefn_buf), "gc_trace_%d.%03d%s.txt", getpid(), ncollections + is_pre,
+    snprintf(tracefn_buf, sizeof(tracefn_buf), "gc_trace_%d.%04d%s.txt", getpid(), ncollections + is_pre,
              is_pre ? "_pre" : "");
     trace_fp = fopen(tracefn_buf, "w");
+    assert(trace_fp);
 }
 
 static int _dummy() {
@@ -1001,17 +1003,29 @@ void runCollection() {
     }
 
     // We want to make sure that classes get freed before their metaclasses.
-    // Use a simple approach of only freeing one level of the hierarchy and then
-    // letting the next collection do the next one.
-    llvm::DenseSet<BoxedClass*> classes_to_not_free;
-    for (auto b : classes_to_free) {
-        classes_to_not_free.insert(b->cls);
-    }
+    // So, while there are still more classes to free, free any classes that are
+    // not the metaclass of another class we will free.  Then repeat.
+    //
+    // Note: our earlier approach of just deferring metaclasses to the next collection is
+    // not quite safe, since we will have freed everything that the class refers to.
+    while (!classes_to_free.empty()) {
+        llvm::DenseSet<BoxedClass*> classes_to_not_free;
+        for (auto b : classes_to_free) {
+            classes_to_not_free.insert(b->cls);
+        }
+
+        std::vector<BoxedClass*> deferred_classes;
+        for (auto b : classes_to_free) {
+            GC_TRACE_LOG("Dealing with the postponed free of class %p\n", b);
+            if (classes_to_not_free.count(b)) {
+                deferred_classes.push_back(b);
+                continue;
+            }
+            global_heap._setFree(GCAllocation::fromUserData(b));
+        }
 
-    for (auto b : classes_to_free) {
-        if (classes_to_not_free.count(b))
-            continue;
-        global_heap._setFree(GCAllocation::fromUserData(b));
+        assert(deferred_classes.size() < classes_to_free.size());
+        std::swap(deferred_classes, classes_to_free);
     }
 
     global_heap.cleanupAfterCollection();

src/gc/gc.h

@@ -28,14 +28,6 @@
 
 #define GC_KEEP_ALIVE(t) asm volatile("" : : "X"(t))
 
-#define TRACE_GC_MARKING 0
-#if TRACE_GC_MARKING
-extern FILE* trace_fp;
-#define GC_TRACE_LOG(...) fprintf(pyston::gc::trace_fp, __VA_ARGS__)
-#else
-#define GC_TRACE_LOG(...)
-#endif
-
 struct _PyWeakReference;
 typedef struct _PyWeakReference PyWeakReference;
 
@@ -45,6 +37,14 @@ class Box;
 
 namespace gc {
 class GCVisitable;
+
+#define TRACE_GC_MARKING 0
+#if TRACE_GC_MARKING
+extern FILE* trace_fp;
+#define GC_TRACE_LOG(...) fprintf(pyston::gc::trace_fp, __VA_ARGS__)
+#else
+#define GC_TRACE_LOG(...)
+#endif
 }
 
 namespace threading {

src/gc/gc_alloc.h

@@ -112,8 +112,6 @@ extern "C" inline void* gc_alloc(size_t bytes, GCKind kind_id) {
 // if (VERBOSITY()) printf("Allocated %ld bytes at [%p, %p)\n", bytes, r, (char*)r + bytes);
 #endif
 
-    GC_TRACE_LOG("Allocated %p\n", r);
-
 #if STAT_ALLOCATIONS
     gc_alloc_bytes.log(alloc_bytes);
     gc_alloc_bytes_typed[(int)kind_id].log(alloc_bytes);

src/gc/heap.cpp

@@ -186,7 +186,7 @@ __attribute__((always_inline)) bool _doFree(GCAllocation* al, std::vector<Box*>*
         if (unlikely(isWeaklyReferenced(b))) {
             assert(weakly_referenced && "attempting to free a weakly referenced object manually");
             weakly_referenced->push_back(b);
-            GC_TRACE_LOG("%p is weakly referenced\n", al->user_data);
+            GC_TRACE_LOG("doFree: %p is weakly referenced\n", al->user_data);
             return false;
         }
 
@@ -196,11 +196,12 @@ __attribute__((always_inline)) bool _doFree(GCAllocation* al, std::vector<Box*>*
         if (unlikely(PyType_Check(b))) {
             assert(classes_to_free);
             classes_to_free->push_back(static_cast<BoxedClass*>(b));
+            GC_TRACE_LOG("doFree: %p is a class object\n", al->user_data);
             return false;
         }
 
         if (b->cls->tp_dealloc != dealloc_null && b->cls->has_safe_tp_dealloc) {
-            GC_TRACE_LOG("running safe destructor for %p\n", b);
+            GC_TRACE_LOG("doFree: running safe destructor for %p\n", b);
             gc_safe_destructors.log();
 
             GCAllocation* al = GCAllocation::fromUserData(b);
@@ -687,8 +688,10 @@ GCAllocation* SmallArena::_alloc(size_t rounded_size, int bucket_idx) {
     while (true) {
         while (Block* cache_block = *cache_head) {
             GCAllocation* rtn = _allocFromBlock(cache_block);
-            if (rtn)
+            if (rtn) {
+                GC_TRACE_LOG("Allocated %p\n", rtn->user_data);
                 return rtn;
+            }
 
             removeFromLLAndNull(cache_block);
             insertIntoLL(&cache->cache_full_heads[bucket_idx], cache_block);

src/runtime/builtin_modules/builtins.cpp

@@ -598,7 +598,7 @@ Box* setattrFunc(Box* obj, Box* _str, Box* value) {
     _str = coerceUnicodeToStr<CXX>(_str);
 
     if (_str->cls != str_cls) {
-        raiseExcHelper(TypeError, "setattr(): attribute name must be string");
+        raiseExcHelper(TypeError, "attribute name must be string, not '%s'", _str->cls->tp_name);
     }
 
     BoxedString* str = static_cast<BoxedString*>(_str);

src/runtime/inline/link_forcer.cpp

@@ -113,6 +113,7 @@ void force() {
     FORCE(assertFailDerefNameDefined);
     FORCE(assertFail);
     FORCE(printExprHelper);
+    FORCE(printHelper);
 
     FORCE(strOrUnicode);
     FORCE(printFloat);

src/runtime/list.cpp

@@ -1282,6 +1282,7 @@ void setupList() {
                                            sizeof(BoxedListIterator), false, "listiterator");
     list_reverse_iterator_cls = BoxedClass::create(type_cls, object_cls, &BoxedListIterator::gcHandler, 0, 0,
                                                    sizeof(BoxedListIterator), false, "listreverseiterator");
+    list_iterator_cls->instances_are_nonzero = list_reverse_iterator_cls->instances_are_nonzero = true;
 
     list_cls->giveAttr("__len__", new BoxedFunction(boxRTFunction((void*)listLen, BOXED_INT, 1)));
 

src/runtime/objmodel.cpp

@@ -222,6 +222,9 @@ extern "C" void printHelper(Box* dest, Box* var, bool nl) {
     static BoxedString* newline_str = internStringImmortal("\n");
     static BoxedString* space_str = internStringImmortal(" ");
 
+    if (dest == None)
+        dest = getSysStdout();
+
     if (var) {
         // begin code for handling of softspace
         bool new_softspace = !nl;
@@ -229,11 +232,17 @@ extern "C" void printHelper(Box* dest, Box* var, bool nl) {
             callattrInternal<CXX>(dest, write_str, CLASS_OR_INST, 0, ArgPassSpec(1), space_str, 0, 0, 0, 0);
 
         Box* str_or_unicode_var = (var->cls == unicode_cls) ? var : str(var);
-        callattrInternal<CXX>(dest, write_str, CLASS_OR_INST, 0, ArgPassSpec(1), str_or_unicode_var, 0, 0, 0, 0);
+        Box* write_rtn
+            = callattrInternal<CXX>(dest, write_str, CLASS_OR_INST, 0, ArgPassSpec(1), str_or_unicode_var, 0, 0, 0, 0);
+        if (!write_rtn)
+            raiseAttributeError(dest, write_str->s());
     }
 
     if (nl) {
-        callattrInternal<CXX>(dest, write_str, CLASS_OR_INST, 0, ArgPassSpec(1), newline_str, 0, 0, 0, 0);
+        Box* write_rtn
+            = callattrInternal<CXX>(dest, write_str, CLASS_OR_INST, 0, ArgPassSpec(1), newline_str, 0, 0, 0, 0);
+        if (!write_rtn)
+            raiseAttributeError(dest, write_str->s());
         if (!var)
             softspace(dest, false);
     }
@@ -996,12 +1005,10 @@ Box* typeLookup(BoxedClass* cls, BoxedString* attr, GetattrRewriteArgs* rewrite_
                 rewrite_args->obj_shape_guarded = true;
             }
             val = base->getattr(attr, rewrite_args);
-            assert(rewrite_args->out_success);
             if (val)
                 return val;
         }
 
-        assert(rewrite_args->out_success);
         assert(!rewrite_args->out_rtn);
         rewrite_args->out_return_convention = GetattrRewriteArgs::NO_RETURN;
         return NULL;
@@ -2486,12 +2493,13 @@ extern "C" bool nonzero(Box* obj) {
             rewriter.reset();
 
         if (rtn == NULL) {
-            ASSERT(obj->cls->is_user_defined || obj->cls == classobj_cls || obj->cls == type_cls
-                       || isSubclass(obj->cls, Exception) || obj->cls == file_cls || obj->cls == traceback_cls
-                       || obj->cls == instancemethod_cls || obj->cls == module_cls || obj->cls == capifunc_cls
-                       || obj->cls == builtin_function_or_method_cls || obj->cls == method_cls || obj->cls == frame_cls
-                       || obj->cls == generator_cls || obj->cls == capi_getset_cls || obj->cls == pyston_getset_cls
-                       || obj->cls == wrapperdescr_cls || obj->cls == wrapperobject_cls,
+            ASSERT(obj->cls->is_user_defined || obj->cls->instances_are_nonzero || obj->cls == classobj_cls
+                       || obj->cls == type_cls || isSubclass(obj->cls, Exception) || obj->cls == file_cls
+                       || obj->cls == traceback_cls || obj->cls == instancemethod_cls || obj->cls == module_cls
+                       || obj->cls == capifunc_cls || obj->cls == builtin_function_or_method_cls
+                       || obj->cls == method_cls || obj->cls == frame_cls || obj->cls == generator_cls
+                       || obj->cls == capi_getset_cls || obj->cls == pyston_getset_cls || obj->cls == wrapperdescr_cls
+                       || obj->cls == wrapperobject_cls,
                    "%s.__nonzero__", getTypeName(obj)); // TODO
 
             if (rewriter.get()) {

src/runtime/types.cpp

@@ -1398,6 +1398,20 @@ static Box* typeSubDict(Box* obj, void* context) {
     abort();
 }
 
+void Box::setDictBacked(Box* val) {
+    assert(this->cls->instancesHaveHCAttrs());
+
+    RELEASE_ASSERT(val->cls == dict_cls || val->cls == attrwrapper_cls, "");
+
+    auto new_attr_list = (HCAttrs::AttrList*)gc_alloc(sizeof(HCAttrs::AttrList) + sizeof(Box*), gc::GCKind::PRECISE);
+    new_attr_list->attrs[0] = val;
+
+    HCAttrs* hcattrs = this->getHCAttrsPtr();
+
+    hcattrs->hcls = HiddenClass::dict_backed;
+    hcattrs->attr_list = new_attr_list;
+}
+
 static void typeSubSetDict(Box* obj, Box* val, void* context) {
     if (obj->cls->instancesHaveDictAttrs()) {
         RELEASE_ASSERT(val->cls == dict_cls, "");
@@ -1406,16 +1420,7 @@ static void typeSubSetDict(Box* obj, Box* val, void* context) {
     }
 
     if (obj->cls->instancesHaveHCAttrs()) {
-        RELEASE_ASSERT(val->cls == dict_cls || val->cls == attrwrapper_cls, "");
-
-        auto new_attr_list
-            = (HCAttrs::AttrList*)gc_alloc(sizeof(HCAttrs::AttrList) + sizeof(Box*), gc::GCKind::PRECISE);
-        new_attr_list->attrs[0] = val;
-
-        HCAttrs* hcattrs = obj->getHCAttrsPtr();
-
-        hcattrs->hcls = HiddenClass::dict_backed;
-        hcattrs->attr_list = new_attr_list;
+        obj->setDictBacked(val);
         return;
     }
 
@@ -2218,6 +2223,28 @@ class AttrWrapper : public Box {
 private:
     Box* b;
 
+    void convertToDictBacked() {
+        HCAttrs* attrs = this->b->getHCAttrsPtr();
+        if (attrs->hcls->type == HiddenClass::DICT_BACKED)
+            return;
+
+        BoxedDict* d = new BoxedDict();
+
+        RELEASE_ASSERT(attrs->hcls->type == HiddenClass::NORMAL || attrs->hcls->type == HiddenClass::SINGLETON, "");
+        for (const auto& p : attrs->hcls->getStrAttrOffsets()) {
+            d->d[p.first] = attrs->attr_list->attrs[p.second];
+        }
+
+        b->setDictBacked(d);
+    }
+
+    bool isDictBacked() { return b->getHCAttrsPtr()->hcls->type == HiddenClass::DICT_BACKED; }
+
+    Box* getDictBacking() {
+        assert(isDictBacked());
+        return b->getHCAttrsPtr()->attr_list->attrs[0];
+    }
+
 public:
     AttrWrapper(Box* b) : b(b) {
         assert(b->cls->instancesHaveHCAttrs());
@@ -2246,9 +2273,16 @@ public:
         RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
         AttrWrapper* self = static_cast<AttrWrapper*>(_self);
 
-        _key = coerceUnicodeToStr<CXX>(_key);
+        if (_key->cls != str_cls)
+            self->convertToDictBacked();
 
-        RELEASE_ASSERT(_key->cls == str_cls, "");
+        if (self->isDictBacked()) {
+            static BoxedString* setitem_str = internStringImmortal("__setitem__");
+            return callattrInternal<CXX>(self->getDictBacking(), setitem_str, LookupScope::CLASS_ONLY, NULL,
+                                         ArgPassSpec(2), _key, value, NULL, NULL, NULL);
+        }
+
+        assert(_key->cls == str_cls);
         BoxedString* key = static_cast<BoxedString*>(_key);
         internStringMortalInplace(key);
 
@@ -2276,9 +2310,16 @@ public:
         RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
         AttrWrapper* self = static_cast<AttrWrapper*>(_self);
 
-        _key = coerceUnicodeToStr<CXX>(_key);
+        if (_key->cls != str_cls)
+            self->convertToDictBacked();
 
-        RELEASE_ASSERT(_key->cls == str_cls, "");
+        if (self->isDictBacked()) {
+            static BoxedString* setdefault_str = internStringImmortal("setdefault");
+            return callattrInternal<CXX>(self->getDictBacking(), setdefault_str, LookupScope::CLASS_ONLY, NULL,
+                                         ArgPassSpec(2), _key, value, NULL, NULL, NULL);
+        }
+
+        assert(_key->cls == str_cls);
         BoxedString* key = static_cast<BoxedString*>(_key);
         internStringMortalInplace(key);
 
@@ -2313,7 +2354,7 @@ public:
         if (S == CAPI && !_key)
             return NULL;
 
-        RELEASE_ASSERT(_key->cls == str_cls, "%s", _key->cls->tp_name);
+        RELEASE_ASSERT(_key->cls == str_cls, "");
         BoxedString* key = static_cast<BoxedString*>(_key);
         internStringMortalInplace(key);
 
@@ -2354,7 +2395,7 @@ public:
 
         _key = coerceUnicodeToStr<CXX>(_key);
 
-        RELEASE_ASSERT(_key->cls == str_cls, "%s", _key->cls->tp_name);
+        RELEASE_ASSERT(_key->cls == str_cls, "");
         BoxedString* key = static_cast<BoxedString*>(_key);
         internStringMortalInplace(key);
 
@@ -2559,6 +2600,12 @@ public:
         RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
         AttrWrapper* self = static_cast<AttrWrapper*>(_self);
 
+        if (self->isDictBacked()) {
+            static BoxedString* iter_str = internStringImmortal("__iter__");
+            return callattrInternal<CXX>(self->getDictBacking(), iter_str, LookupScope::CLASS_ONLY, NULL,
+                                         ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
+        }
+
         return new AttrWrapperIter(self);
     }
 
@@ -3505,6 +3552,7 @@ void setupRuntime() {
     // XXX silly that we have to set this again
     new (&object_cls->attrs) HCAttrs(HiddenClass::makeSingleton());
     new (&type_cls->attrs) HCAttrs(HiddenClass::makeSingleton());
+    object_cls->instances_are_nonzero = true;
     object_cls->tp_getattro = PyObject_GenericGetAttr;
     object_cls->tp_setattro = PyObject_GenericSetAttr;
     object_cls->tp_init = object_init;

src/runtime/types.h

@@ -235,6 +235,12 @@ public:
     // that we can't rely on for extension classes.
     bool is_pyston_class;
 
+    // Just for debugging: whether instances of this class should always be considered nonzero.
+    // This is the default for anything that doesn't define a __nonzero__ or __len__ method, but
+    // for builtin types we have the extra check that we opted into this behavior rather than
+    // just forgot to add nonzero/len.
+    bool instances_are_nonzero;
+
     bool has___class__; // Has a custom __class__ attribute (ie different from object's __class__ descriptor)
     bool has_instancecheck;
     bool has_subclasscheck;

test/integration/sqlalchemy_smalltest.py

@@ -73,6 +73,7 @@ MODULES_TO_TEST = [
     'test.base.test_events',
     'test.base.test_except',
     'test.base.test_inspect',
+    'test.base.test_utils',
     'test.dialect.test_mxodbc',
     'test.dialect.test_pyodbc',
     'test.dialect.test_sybase',
@@ -103,7 +104,7 @@ for fn in test_files:
         m = __import__(mname, fromlist=["__all__"])
         for clsname in dir(m):
             cls = getattr(m, clsname)
-            if not clsname.endswith("Test") or not isinstance(cls, type):
+            if clsname.startswith('_') or not clsname.endswith("Test") or not isinstance(cls, type):
                 continue
             print "Running", cls
 

test/tests/closure_test.py

@@ -82,3 +82,15 @@ def f7(n):
 
 c = f7(5)()
 print c.foo()
+
+
+# Regression test: being included in a closure (ie in the parent scope) shouldn't ruin type analysis.
+def f8():
+    l = []
+
+    def g():
+        print l
+
+    l.append(1)
+for i in xrange(11000):
+    f8()

test/tests/dir.py

@@ -123,3 +123,24 @@ while True:
     except StopIteration:
         break
 print sorted(l)
+
+c = C1()
+print sorted(c.__dict__.items())
+# setattr() converts the attr name to a string:
+setattr(c, u'a', 1)
+print sorted(c.__dict__.items())
+# directly setting on the dict does not:
+c.__dict__[u'b'] = 2
+print sorted(c.__dict__.items())
+c.__dict__.update({u'c':3})
+print sorted(c.__dict__.items())
+
+# Can't set non-string-attrs with setattr, but can via dict:
+try:
+    setattr(c, 1, 1)
+    assert 0
+except TypeError as e:
+    print e
+print sorted(c.__dict__.items())
+c.__dict__[5] = 2
+print sorted(c.__dict__.items())

test/tests/hasattr_guarding.py

+# Test some weird hasattr guarding scenarios
+# I think this applies equally well to getattr
+
+def f(o):
+    print hasattr(o, "a")
+    print getattr(o, "a", None)
+
+class C(object):
+    def __getattr__(self, key):
+        print "getattr", key
+        raise AttributeError(key)
+
+for i in xrange(300):
+    print i
+    c = C()
+    try:
+        f(c)
+    except AttributeError as e:
+        print e
+    c.a = 1
+    setattr(c, str(i), i)
+    f(c)
+

test/tests/print.py

@@ -54,3 +54,9 @@ try:
     assert 0, "expected TypeError was not thrown"
 except TypeError:
     pass
+
+print >>None, "this should still print"
+try:
+    print >>1, "this should error"
+except AttributeError as e:
+    print e