Add 'runtime ics' to improve C++ runtime speed

An experimental way of making the runtime faster.  Previously, the C++
runtime was much slower than pure-Python code, since the Python code
can use the JIT to improve performance (through inline caches and type
speculation).  This commit adds "runtime ics", which are out-of-band
patchpoints that we can allocate from C++.  We insert them into the
same patchpoint system that gets used by the main JIT pipeline, so
they get rewritten like any other patchpoint, but the lifetime and use
gets controlled by the C++ code.

The trickiest part is getting the newly-allocated instruction regions
to work with our exception handling; to make this work we have to emit
.eh_frame sections.  (A custom libunwind-based exception unwinder
might not have this requirement.)

Otherwise it feels like a decent approach.  Right now just using it to
implement class slots and then only using them in the pyElements
iterator; it's probably not the most direct or efficient way of
implementing that particular feature, but it has the benefit of being
general and being useful for things such as binops which are not
directly expressible in terms of class slots.

This commit adds the support without turning the feature on (next
commit will turn it on), since there are a number of other changes
needed.

include/object.h

@@ -455,6 +455,7 @@ struct _typeobject {
     void* _hcls;
     void* _hcattrs;
     char _dep_getattrs[56]; // FIXME: this is hardcoding the size of this particular implementation of std::unordered_map
+    char _ics[48];
     void* _base;
     void* _gcvisit_func;
     int _attrs_offset;

src/asm_writing/assembler.cpp

@@ -634,6 +634,9 @@ void Assembler::callq(Register r) {
     emitByte(0xd3);
 }
 
+void Assembler::retq() {
+    emitByte(0xc3);
+}
 
 
 void Assembler::cmp(Register reg1, Register reg2) {

src/asm_writing/assembler.h

@@ -129,6 +129,7 @@ public:
     void inc(Indirect mem);
 
     void callq(Register reg);
+    void retq();
 
     void cmp(Register reg1, Register reg2);
     void cmp(Register reg, Immediate imm);

src/asm_writing/icinfo.cpp

@@ -196,9 +196,9 @@ ICInfo::ICInfo(void* start_addr, void* continue_addr, StackInfo stack_info, int
 }
 
 static std::unordered_map<void*, ICInfo*> ics_by_return_addr;
-void registerCompiledPatchpoint(CompiledFunction* cf, uint8_t* start_addr, uint8_t* slowpath_start_addr,
-                                uint8_t* continue_addr, uint8_t* slowpath_rtn_addr, const ICSetupInfo* ic,
-                                StackInfo stack_info, std::unordered_set<int> live_outs) {
+ICInfo* registerCompiledPatchpoint(uint8_t* start_addr, uint8_t* slowpath_start_addr, uint8_t* continue_addr,
+                                   uint8_t* slowpath_rtn_addr, const ICSetupInfo* ic, StackInfo stack_info,
+                                   std::unordered_set<int> live_outs) {
     assert(slowpath_start_addr - start_addr >= ic->num_slots * ic->slot_size);
     assert(slowpath_rtn_addr > slowpath_start_addr);
     assert(slowpath_rtn_addr <= start_addr + ic->totalSize());
@@ -241,8 +241,7 @@ void registerCompiledPatchpoint(CompiledFunction* cf, uint8_t* start_addr, uint8
 
     ics_by_return_addr[slowpath_rtn_addr] = icinfo;
 
-    assert(cf);
-    cf->ics.push_back(icinfo);
+    return icinfo;
 }
 
 ICInfo* getICInfo(void* rtn_addr) {

src/asm_writing/icinfo.h

@@ -129,9 +129,9 @@ public:
 
 class ICSetupInfo;
 struct CompiledFunction;
-void registerCompiledPatchpoint(CompiledFunction* cf, uint8_t* start_addr, uint8_t* slowpath_start_addr,
-                                uint8_t* continue_addr, uint8_t* slowpath_rtn_addr, const ICSetupInfo*,
-                                StackInfo stack_info, std::unordered_set<int> live_outs);
+ICInfo* registerCompiledPatchpoint(uint8_t* start_addr, uint8_t* slowpath_start_addr, uint8_t* continue_addr,
+                                   uint8_t* slowpath_rtn_addr, const ICSetupInfo*, StackInfo stack_info,
+                                   std::unordered_set<int> live_outs);
 
 ICInfo* getICInfo(void* rtn_addr);
 }

src/codegen/memmgr.cpp

@@ -230,4 +230,14 @@ PystonMemoryManager::~PystonMemoryManager() {
 llvm::RTDyldMemoryManager* createMemoryManager() {
     return new PystonMemoryManager();
 }
+
+// These functions exist as instance methods of the RTDyldMemoryManager class,
+// but it's tricky to access them since the class has pure-virtual methods.
+void registerEHFrames(uint8_t* addr, uint64_t load_addr, size_t size) {
+    PystonMemoryManager().registerEHFrames(addr, load_addr, size);
+}
+
+void deregisterEHFrames(uint8_t* addr, uint64_t load_addr, size_t size) {
+    PystonMemoryManager().deregisterEHFrames(addr, load_addr, size);
+}
 }

src/codegen/memmgr.h

@@ -15,12 +15,18 @@
 #ifndef PYSTON_CODEGEN_MEMMGR_H
 #define PYSTON_CODEGEN_MEMMGR_H
 
+#include <cstddef>
+#include <cstdint>
+
 namespace llvm {
 class RTDyldMemoryManager;
 }
 
 namespace pyston {
+
 llvm::RTDyldMemoryManager* createMemoryManager();
+void registerEHFrames(uint8_t* addr, uint64_t load_addr, size_t size);
+void deregisterEHFrames(uint8_t* addr, uint64_t load_addr, size_t size);
 }
 
 #endif

src/codegen/patchpoints.cpp

@@ -230,8 +230,12 @@ void processStackmap(CompiledFunction* cf, StackMap* stackmap) {
 
         assert(pp->numICStackmapArgs() == 0); // don't do anything with these for now
 
-        registerCompiledPatchpoint(cf, start_addr, slowpath_start, end_addr, slowpath_rtn_addr, ic,
-                                   StackInfo({ stack_size, scratch_size, scratch_rbp_offset }), std::move(live_outs));
+        ICInfo* icinfo = registerCompiledPatchpoint(start_addr, slowpath_start, end_addr, slowpath_rtn_addr, ic,
+                                                    StackInfo({ stack_size, scratch_size, scratch_rbp_offset }),
+                                                    std::move(live_outs));
+
+        assert(cf);
+        cf->ics.push_back(icinfo);
     }
 
     for (PatchpointInfo* pp : new_patchpoints) {

src/core/options.cpp

@@ -56,6 +56,7 @@ bool ENABLE_INLINING = 1 && _GLOBAL_ENABLE;
 bool ENABLE_REOPT = 1 && _GLOBAL_ENABLE;
 bool ENABLE_PYSTON_PASSES = 1 && _GLOBAL_ENABLE;
 bool ENABLE_TYPE_FEEDBACK = 1 && _GLOBAL_ENABLE;
+bool ENABLE_RUNTIME_ICS = 0 && _GLOBAL_ENABLE;
 
 bool ENABLE_FRAME_INTROSPECTION = 1;
 bool BOOLS_AS_I64 = ENABLE_FRAME_INTROSPECTION;

src/core/options.h

@@ -36,7 +36,7 @@ extern bool SHOW_DISASM, FORCE_OPTIMIZE, BENCH, PROFILE, DUMPJIT, TRAP, USE_STRI
 extern bool ENABLE_ICS, ENABLE_ICGENERICS, ENABLE_ICGETITEMS, ENABLE_ICSETITEMS, ENABLE_ICDELITEMS, ENABLE_ICBINEXPS,
     ENABLE_ICNONZEROS, ENABLE_ICCALLSITES, ENABLE_ICSETATTRS, ENABLE_ICGETATTRS, ENALBE_ICDELATTRS, ENABLE_ICGETGLOBALS,
     ENABLE_SPECULATION, ENABLE_OSR, ENABLE_LLVMOPTS, ENABLE_INLINING, ENABLE_REOPT, ENABLE_PYSTON_PASSES,
-    ENABLE_TYPE_FEEDBACK, ENABLE_FRAME_INTROSPECTION;
+    ENABLE_TYPE_FEEDBACK, ENABLE_FRAME_INTROSPECTION, ENABLE_RUNTIME_ICS;
 
 // Due to a temporary LLVM limitation, represent bools as i64's instead of i1's.
 extern bool BOOLS_AS_I64;

src/core/types.h

@@ -322,9 +322,17 @@ std::string getFullNameOfClass(BoxedClass* cls);
 
 class Rewriter;
 class RewriterVar;
+class RuntimeIC;
+class CallattrIC;
+class NonzeroIC;
+class BinopIC;
 
 class Box;
 class BoxIterator {
+private:
+    Box* iter;
+    Box* value;
+
 public:
     BoxIterator(Box* iter) : iter(iter), value(nullptr) {}
 
@@ -332,20 +340,11 @@ public:
     bool operator!=(BoxIterator const& rhs) const { return !(*this == rhs); }
 
     BoxIterator& operator++();
-    BoxIterator operator++(int) {
-        BoxIterator tmp(*this);
-        operator++();
-        return tmp;
-    }
 
     Box* operator*() const { return value; }
     Box* operator*() { return value; }
 
     void gcHandler(GCVisitor* v);
-
-private:
-    Box* iter;
-    Box* value;
 };
 
 namespace gc {
@@ -462,6 +461,13 @@ public:
     // to guard on anything about the class.
     ICInvalidator dependent_icgetattrs;
 
+    // TODO: these don't actually get deallocated right now
+    std::shared_ptr<CallattrIC> hasnext_ic, next_ic;
+    std::shared_ptr<NonzeroIC> nonzero_ic;
+    CallattrIC* getHasnextIC();
+    CallattrIC* getNextIC();
+    NonzeroIC* getNonzeroIC();
+
     // Only a single base supported for now.
     // Is NULL iff this is object_cls
     BoxedClass* base;

src/runtime/builtin_modules/builtins.cpp

@@ -24,6 +24,7 @@
 #include "core/ast.h"
 #include "core/types.h"
 #include "gc/collector.h"
+#include "runtime/ics.h"
 #include "runtime/inline/xrange.h"
 #include "runtime/list.h"
 #include "runtime/long.h"
@@ -179,9 +180,11 @@ extern "C" Box* sum(Box* container, Box* initial) {
     if (initial->cls == str_cls)
         raiseExcHelper(TypeError, "sum() can't sum strings [use ''.join(seq) instead]");
 
+    BinopIC pp;
+
     Box* cur = initial;
     for (Box* e : container->pyElements()) {
-        cur = binopInternal(cur, e, AST_TYPE::Add, false, NULL);
+        cur = pp.call(cur, e, AST_TYPE::Add);
     }
     return cur;
 }
@@ -550,7 +553,9 @@ public:
     static Box* next(Box* _self) {
         assert(_self->cls == enumerate_cls);
         BoxedEnumerate* self = static_cast<BoxedEnumerate*>(_self);
-        return new BoxedTuple({ boxInt(self->idx++), *self->iterator++ });
+        Box* val = *self->iterator;
+        ++self->iterator;
+        return new BoxedTuple({ boxInt(self->idx++), val });
     }
 
     static Box* hasnext(Box* _self) {
@@ -648,16 +653,19 @@ Box* print(BoxedTuple* args, BoxedDict* kwargs) {
         BoxedString* s = str(e);
 
         if (!first) {
-            Box* r = callattr(dest, &write_str, false, ArgPassSpec(1), space_box, NULL, NULL, NULL, NULL);
+            Box* r = callattr(dest, &write_str, CallattrFlags({.cls_only = false, .null_on_nonexistent = false }),
+                              ArgPassSpec(1), space_box, NULL, NULL, NULL, NULL);
             RELEASE_ASSERT(r, "");
         }
         first = false;
 
-        Box* r = callattr(dest, &write_str, false, ArgPassSpec(1), s, NULL, NULL, NULL, NULL);
+        Box* r = callattr(dest, &write_str, CallattrFlags({.cls_only = false, .null_on_nonexistent = false }),
+                          ArgPassSpec(1), s, NULL, NULL, NULL, NULL);
         RELEASE_ASSERT(r, "");
     }
 
-    Box* r = callattr(dest, &write_str, false, ArgPassSpec(1), end, NULL, NULL, NULL, NULL);
+    Box* r = callattr(dest, &write_str, CallattrFlags({.cls_only = false, .null_on_nonexistent = false }),
+                      ArgPassSpec(1), end, NULL, NULL, NULL, NULL);
     RELEASE_ASSERT(r, "");
 
     return None;

src/runtime/builtin_modules/sys.cpp

@@ -73,7 +73,8 @@ void appendToSysPath(const std::string& path) {
 void prependToSysPath(const std::string& path) {
     BoxedList* sys_path = getSysPath();
     static std::string attr = "insert";
-    callattr(sys_path, &attr, false, ArgPassSpec(2), boxInt(0), new BoxedString(path), NULL, NULL, NULL);
+    callattr(sys_path, &attr, CallattrFlags({.cls_only = false, .null_on_nonexistent = false }), ArgPassSpec(2),
+             boxInt(0), new BoxedString(path), NULL, NULL, NULL);
 }
 
 static BoxedClass* sys_flags_cls;

src/runtime/ics.cpp

+// Copyright (c) 2014 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "runtime/ics.h"
+
+#include "asm_writing/icinfo.h"
+#include "asm_writing/rewriter.h"
+#include "codegen/compvars.h"
+#include "codegen/memmgr.h"
+#include "codegen/patchpoints.h"
+#include "codegen/stackmaps.h"
+#include "core/common.h"
+#include "core/options.h"
+#include "core/stats.h"
+#include "core/types.h"
+
+namespace pyston {
+
+// I am not sure if we should use the equivalent of -fomit-frame-pointer for these trampolines.
+// If we don't use it, we break gdb backtraces.
+// If we do use it, we have an inconsistency with the rest of the code, which runs with -fno-omit-frame-pointer.
+//
+// I haven't detected anything go that wrong, so let's just enable it for now.
+#define RUNTIMEICS_OMIT_FRAME_PTR 1
+
+
+// Useful links for understanding the eh_frame format:
+// - http://www.dwarfstd.org/doc/Dwarf3.pdf
+// - https://refspecs.linuxfoundation.org/LSB_3.0.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html
+// - Generating with clang then "readelf -w"
+//
+// This template is generated from this C++ file:
+//
+// extern void foo();
+// int bar() {
+//   foo();
+//   return 1;
+// }
+//
+// objdump -s -j .eh_frame test
+// readelf -w test
+//
+
+#if RUNTIMEICS_OMIT_FRAME_PTR
+// clang++ test.cpp -o test -O3 -fomit-frame-pointer -c
+// The generated assembly is:
+//
+//  0:   50                      push   %rax
+//  1:   e8 00 00 00 00          callq  6 <_Z3barv+0x6>
+//  6:   b8 01 00 00 00          mov    $0x1,%eax
+//  b:   5a                      pop    %rdx
+//  c:   c3                      retq
+//
+//  (I believe the push/pop are for stack alignment)
+//
+static const char _eh_frame_template[] =
+    // CIE
+    "\x14\x00\x00\x00" // size of the CIE
+    "\x00\x00\x00\x00" // specifies this is an CIE
+    "\x03"             // version number
+    "\x7a\x52\x00"     // augmentation string "zR"
+    "\x01\x78\x10"     // code factor 1, data factor -8, return address 16
+    "\x01\x1b"         // augmentation data: 1b (CIE pointers as 4-byte-signed pcrel values)
+    "\x0c\x07\x08\x90\x01\x00\x00"
+    // Instructions:
+    // - DW_CFA_def_cfa: r7 (rsp) ofs 8
+    // - DW_CFA_offset: r16 (rip) at cfa-8
+    // - nop, nop
+
+    // FDE:
+    "\x14\x00\x00\x00" // size of the FDE
+    "\x1c\x00\x00\x00" // offset to the CIE
+    "\x00\x00\x00\x00" // prcel offset to function address [to be filled in]
+    "\x0d\x00\x00\x00" // function size [to be filled in]
+    "\x00"             // augmentation data (none)
+    "\x41\x0e\x10"
+    // Instructions:
+    // - DW_CFA_advance_loc: 1 to 00000001
+    // - DW_CFA_def_cfa_offset: 16
+    "\x00\x00\x00\x00" // padding
+
+    "\x00\x00\x00\x00" // terminator
+    ;
+#else
+// clang++ test.cpp -o test -O3 -fno-omit-frame-pointer -c
+// The generated assembly is:
+//
+//  0:   55                      push   %rbp
+//  1:   48 89 e5                mov    %rsp,%rbp
+//  4:   e8 00 00 00 00          callq  9 <_Z3barv+0x9>
+//  9:   b8 01 00 00 00          mov    $0x1,%eax
+//  e:   5d                      pop    %rbp
+//  f:   c3                      retq
+//
+static const char _eh_frame_template[] =
+    // CIE
+    "\x14\x00\x00\x00" // size of the CIE
+    "\x00\x00\x00\x00" // specifies this is an CIE
+    "\x03"             // version number
+    "\x7a\x52\x00"     // augmentation string "zR"
+    "\x01\x78\x10"     // code factor 1, data factor -8, return address 16
+    "\x01\x1b"         // augmentation data: 1b (CIE pointers as 4-byte-signed pcrel values)
+    "\x0c\x07\x08\x90\x01\x00\x00"
+    // Instructions:
+    // - DW_CFA_def_cfa: r7 (rsp) ofs 8
+    // - DW_CFA_offset: r16 (rip) at cfa-8
+    // - nop, nop
+
+    // FDE:
+    "\x1c\x00\x00\x00" // size of the FDE
+    "\x1c\x00\x00\x00" // offset to the CIE
+    "\x00\x00\x00\x00" // prcel offset to function address [to be filled in]
+    "\x10\x00\x00\x00" // function size [to be filled in]
+    "\x00"             // augmentation data (none)
+    "\x41\x0e\x10\x86\x02\x43\x0d\x06"
+    // Instructions:
+    // - DW_CFA_advance_loc: 1 to 00000001
+    // - DW_CFA_def_cfa_offset: 16
+    // - DW_CFA_offset: r6 (rbp) at cfa-16
+    // - DW_CFA_advance_loc: 3 to 00000004
+    // - DW_CFA_def_cfa_register: r6 (rbp)
+    // - nops
+    "\x00\x00\x00\x00\x00\x00\x00" // padding
+
+    "\x00\x00\x00\x00" // terminator
+    ;
+#endif
+#define EH_FRAME_SIZE sizeof(_eh_frame_template)
+
+static void writeTrivialEhFrame(void* eh_frame_addr, void* func_addr, uint64_t func_size) {
+    memcpy(eh_frame_addr, _eh_frame_template, sizeof(_eh_frame_template));
+
+    int32_t* offset_ptr = (int32_t*)((uint8_t*)eh_frame_addr + 0x20);
+    int32_t* size_ptr = (int32_t*)((uint8_t*)eh_frame_addr + 0x24);
+
+    int64_t offset = (int8_t*)func_addr - (int8_t*)offset_ptr;
+    assert(offset >= INT_MIN && offset <= INT_MAX);
+    *offset_ptr = offset;
+
+    assert(func_size <= UINT_MAX);
+    *size_ptr = func_size;
+}
+
+RuntimeIC::RuntimeIC(void* func_addr, int num_slots, int slot_size) {
+    static StatCounter sc("runtime_ics_num");
+    sc.log();
+
+    if (ENABLE_RUNTIME_ICS) {
+#if RUNTIMEICS_OMIT_FRAME_PTR
+        static const int PROLOGUE_SIZE = 1;
+#else
+        /*
+         * We emit a prologue since we want to align the stack pointer,
+         * and also use RBP.
+         * It's not clear if we need to use RBP or not, since we emit the .eh_frame section anyway.
+         *
+         * The prologue looks like:
+         * push %rbp # 55
+         * mov %rsp, %rbp # 48 89 e5
+         *
+         * The epilogue is:
+         * pop %rbp # 5d
+         * retq # c3
+         */
+        static const int PROLOGUE_SIZE = 4;
+#endif
+        static const int CALL_SIZE = 13;
+        static const int EPILOGUE_SIZE = 2;
+        int patchable_size = num_slots * slot_size;
+        int total_size = PROLOGUE_SIZE + patchable_size + CALL_SIZE + EPILOGUE_SIZE;
+        addr = malloc(total_size);
+
+        // printf("Allocated runtime IC at %p\n", addr);
+
+        std::unique_ptr<ICSetupInfo> setup_info(
+            ICSetupInfo::initialize(true, num_slots, slot_size, ICSetupInfo::Generic, NULL));
+        uint8_t* pp_start = (uint8_t*)addr + PROLOGUE_SIZE;
+        uint8_t* pp_end = pp_start + patchable_size + CALL_SIZE;
+
+
+        SpillMap _spill_map;
+        std::pair<uint8_t*, uint8_t*> p
+            = initializePatchpoint3(func_addr, pp_start, pp_end, 0 /* scratch_offset */, 0 /* scratch_size */,
+                                    std::unordered_set<int>(), _spill_map);
+        assert(_spill_map.size() == 0);
+        assert(p.first == pp_start + patchable_size);
+        assert(p.second == pp_end);
+
+        icinfo = registerCompiledPatchpoint(pp_start, pp_start + patchable_size, pp_end, pp_end, setup_info.get(),
+                                            StackInfo(), std::unordered_set<int>());
+
+        assembler::Assembler prologue_assem((uint8_t*)addr, PROLOGUE_SIZE);
+#if RUNTIMEICS_OMIT_FRAME_PTR
+        prologue_assem.push(assembler::RAX);
+#else
+        prologue_assem.push(assembler::RBP);
+        prologue_assem.mov(assembler::RSP, assembler::RBP);
+#endif
+        assert(!prologue_assem.hasFailed());
+        assert(prologue_assem.isExactlyFull());
+
+        assembler::Assembler epilogue_assem(pp_end, EPILOGUE_SIZE);
+#if RUNTIMEICS_OMIT_FRAME_PTR
+        epilogue_assem.pop(assembler::RDX);
+#else
+        epilogue_assem.pop(assembler::RBP);
+#endif
+        epilogue_assem.retq();
+        assert(!epilogue_assem.hasFailed());
+        assert(epilogue_assem.isExactlyFull());
+
+        // TODO: ideally would be more intelligent about allocation strategies.
+        // The code sections should be together and the eh sections together
+        eh_frame_addr = malloc(EH_FRAME_SIZE);
+        writeTrivialEhFrame(eh_frame_addr, addr, total_size);
+        registerEHFrames((uint8_t*)eh_frame_addr, (uint64_t)eh_frame_addr, EH_FRAME_SIZE);
+
+    } else {
+        addr = func_addr;
+    }
+}
+
+RuntimeIC::~RuntimeIC() {
+    if (ENABLE_RUNTIME_ICS) {
+        deregisterEHFrames((uint8_t*)eh_frame_addr, (uint64_t)eh_frame_addr, EH_FRAME_SIZE);
+        free(addr);
+        free(eh_frame_addr);
+    } else {
+    }
+}
+}

src/runtime/ics.h

+// Copyright (c) 2014 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef PYSTON_RUNTIME_ICS_H
+#define PYSTON_RUNTIME_ICS_H
+
+#include "core/common.h"
+#include "runtime/objmodel.h"
+
+namespace pyston {
+
+class ICInfo;
+
+class RuntimeIC {
+private:
+    void* addr;
+    void* eh_frame_addr;
+
+    ICInfo* icinfo;
+
+    RuntimeIC(const RuntimeIC&) = delete;
+    void operator=(const RuntimeIC&) = delete;
+
+protected:
+    RuntimeIC(void* addr, int num_slots, int slot_size);
+    ~RuntimeIC();
+
+    template <class... Args> uint64_t call_int(Args... args) {
+        return reinterpret_cast<uint64_t (*)(Args...)>(this->addr)(args...);
+    }
+
+    template <class... Args> bool call_bool(Args... args) {
+        return reinterpret_cast<bool (*)(Args...)>(this->addr)(args...);
+    }
+
+    template <class... Args> void* call_ptr(Args... args) {
+        return reinterpret_cast<void* (*)(Args...)>(this->addr)(args...);
+    }
+
+    template <class... Args> double call_double(Args... args) {
+        return reinterpret_cast<double (*)(Args...)>(this->addr)(args...);
+    }
+};
+
+class CallattrIC : public RuntimeIC {
+public:
+    CallattrIC() : RuntimeIC((void*)callattr, 1, 160) {}
+
+    Box* call(Box* obj, const std::string* attr, CallattrFlags flags, ArgPassSpec spec, Box* arg0, Box* arg1, Box* arg2,
+              Box** args, const std::vector<const std::string*>* keyword_names) {
+        return (Box*)call_ptr(obj, attr, flags, spec, arg0, arg1, arg2, args, keyword_names);
+    }
+};
+
+class BinopIC : public RuntimeIC {
+public:
+    BinopIC() : RuntimeIC((void*)binop, 1, 160) {}
+
+    Box* call(Box* lhs, Box* rhs, int op_type) { return (Box*)call_ptr(lhs, rhs, op_type); }
+};
+
+class NonzeroIC : public RuntimeIC {
+public:
+    NonzeroIC() : RuntimeIC((void*)nonzero, 1, 40) {}
+
+    bool call(Box* obj) { return call_bool(obj); }
+};
+
+} // namespace pyston
+
+#endif

src/runtime/objmodel.cpp

@@ -1842,6 +1842,10 @@ extern "C" void dump(void* p) {
         Box* b = (Box*)p;
         printf("Class: %s\n", getFullTypeName(b).c_str());
 
+        if (b->cls == bool_cls) {
+            printf("The %s object\n", b == True ? "True" : "False");
+        }
+
         if (isSubclass(b->cls, type_cls)) {
             printf("Type name: %s\n", getFullNameOfClass(static_cast<BoxedClass*>(b)).c_str());
         }
@@ -2034,8 +2038,10 @@ extern "C" Box* callattrInternal(Box* obj, const std::string* attr, LookupScope
     }
 }
 
-extern "C" Box* callattr(Box* obj, const std::string* attr, bool clsonly, ArgPassSpec argspec, Box* arg1, Box* arg2,
-                         Box* arg3, Box** args, const std::vector<const std::string*>* keyword_names) {
+extern "C" Box* callattr(Box* obj, const std::string* attr, CallattrFlags flags, ArgPassSpec argspec, Box* arg1,
+                         Box* arg2, Box* arg3, Box** args, const std::vector<const std::string*>* keyword_names) {
+    assert(gc::isValidGCObject(obj));
+
     int npassed_args = argspec.totalPassed();
 
     static StatCounter slowpath_callattr("slowpath_callattr");
@@ -2050,7 +2056,7 @@ extern "C" Box* callattr(Box* obj, const std::string* attr, bool clsonly, ArgPas
         __builtin_extract_return_addr(__builtin_return_address(0)), num_orig_args, "callattr"));
     Box* rtn;
 
-    LookupScope scope = clsonly ? CLASS_ONLY : CLASS_OR_INST;
+    LookupScope scope = flags.cls_only ? CLASS_ONLY : CLASS_OR_INST;
 
     if (rewriter.get()) {
         // TODO feel weird about doing this; it either isn't necessary
@@ -2077,7 +2083,7 @@ extern "C" Box* callattr(Box* obj, const std::string* attr, bool clsonly, ArgPas
         rtn = callattrInternal(obj, attr, scope, NULL, argspec, arg1, arg2, arg3, args, keyword_names);
     }
 
-    if (rtn == NULL) {
+    if (rtn == NULL && !flags.null_on_nonexistent) {
         raiseAttributeError(obj, attr->c_str());
     }
 

src/runtime/objmodel.h

@@ -48,7 +48,11 @@ extern "C" void setattr(Box* obj, const char* attr, Box* attr_val);
 extern "C" void delattr(Box* obj, const char* attr);
 extern "C" bool nonzero(Box* obj);
 extern "C" Box* runtimeCall(Box*, ArgPassSpec, Box*, Box*, Box*, Box**, const std::vector<const std::string*>*);
-extern "C" Box* callattr(Box*, const std::string*, bool, ArgPassSpec, Box*, Box*, Box*, Box**,
+struct CallattrFlags {
+    bool cls_only : 1;
+    bool null_on_nonexistent : 1;
+};
+extern "C" Box* callattr(Box*, const std::string*, CallattrFlags, ArgPassSpec, Box*, Box*, Box*, Box**,
                          const std::vector<const std::string*>*);
 extern "C" BoxedString* str(Box* obj);
 extern "C" BoxedString* repr(Box* obj);
@@ -81,6 +85,8 @@ extern "C" bool isSubclass(BoxedClass* child, BoxedClass* parent);
 extern "C" BoxedClosure* createClosure(BoxedClosure* parent_closure);
 extern "C" Box* getiter(Box* o);
 
+extern "C" void dump(void* p);
+
 struct SetattrRewriteArgs;
 void setattrInternal(Box* obj, const std::string& attr, Box* val, SetattrRewriteArgs* rewrite_args);
 

src/runtime/types.cpp

@@ -26,6 +26,7 @@
 #include "core/types.h"
 #include "gc/collector.h"
 #include "runtime/classobj.h"
+#include "runtime/ics.h"
 #include "runtime/iterobject.h"
 #include "runtime/long.h"
 #include "runtime/objmodel.h"
@@ -51,21 +52,54 @@ bool IN_SHUTDOWN = false;
 #define SLICE_STOP_OFFSET ((char*)&(((BoxedSlice*)0x01)->stop) - (char*)0x1)
 #define SLICE_STEP_OFFSET ((char*)&(((BoxedSlice*)0x01)->step) - (char*)0x1)
 
+CallattrIC* BoxedClass::getHasnextIC() {
+    auto ic = hasnext_ic.get();
+    if (!ic) {
+        ic = new CallattrIC();
+        hasnext_ic.reset(ic);
+    }
+    return ic;
+}
+CallattrIC* BoxedClass::getNextIC() {
+    auto ic = next_ic.get();
+    if (!ic) {
+        ic = new CallattrIC();
+        next_ic.reset(ic);
+    }
+    return ic;
+}
+NonzeroIC* BoxedClass::getNonzeroIC() {
+    auto ic = nonzero_ic.get();
+    if (!ic) {
+        ic = new NonzeroIC();
+        nonzero_ic.reset(ic);
+    }
+    return ic;
+}
+
 BoxIterator& BoxIterator::operator++() {
     static std::string hasnext_str("__hasnext__");
     static std::string next_str("next");
 
-    Box* hasnext = callattrInternal(iter, &hasnext_str, CLASS_ONLY, NULL, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
+    assert(iter);
+
+    Box* hasnext = iter->cls->getHasnextIC()->call(iter, &hasnext_str,
+                                                   CallattrFlags({.cls_only = true, .null_on_nonexistent = true }),
+                                                   ArgPassSpec(0), nullptr, nullptr, nullptr, nullptr, nullptr);
     if (hasnext) {
-        if (nonzero(hasnext)) {
-            value = callattrInternal(iter, &next_str, CLASS_ONLY, NULL, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
+        if (hasnext->cls->getNonzeroIC()->call(hasnext)) {
+            value = iter->cls->getNextIC()->call(iter, &next_str,
+                                                 CallattrFlags({.cls_only = true, .null_on_nonexistent = true }),
+                                                 ArgPassSpec(0), nullptr, nullptr, nullptr, nullptr, nullptr);
         } else {
             iter = nullptr;
             value = nullptr;
         }
     } else {
         try {
-            value = callattrInternal(iter, &next_str, CLASS_ONLY, NULL, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
+            value = iter->cls->getNextIC()->call(iter, &next_str,
+                                                 CallattrFlags({.cls_only = true, .null_on_nonexistent = true }),
+                                                 ArgPassSpec(0), nullptr, nullptr, nullptr, nullptr, nullptr);
         } catch (Box* e) {
             if ((e == StopIteration) || isSubclass(e->cls, StopIteration)) {
                 iter = nullptr;