Generate cxx "fixups" on-demand

fixups aka the stubs that decref whatever's needed when an exception is thrown

I looked into this because most (75%?) of the refcounting overhead
comes from the cxx fixups.  Previously we would always generate them in
the IRGenerator, regardless of whether they were needed.  Now they are
generated in the refcounter, which knows whether they are needed or not.

Unfortunately it looks like they are usually needed, so the gains here
aren't that great (saves about 10% llvm instructions whereas cxx fixups
in general added about 400% more llvm instructions).

I think this is still a good change because it's also necessary in order to use
Marius's EH stuff.

I think the cost of the fixups is mostly related to the cost of the decrefs
that it adds, so even though most of the refcounting overhead seems to be due to
adding the cxx fixups, reducing general decref overhead might reduce cxx fixup overhead

src/codegen/irgen.h

@@ -15,6 +15,7 @@
 #ifndef PYSTON_CODEGEN_IRGEN_H
 #define PYSTON_CODEGEN_IRGEN_H
 
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ExecutionEngine/ObjectCache.h"
 #include "llvm/IR/CallSite.h"
@@ -216,13 +217,16 @@ private:
     llvm::DenseMap<llvm::Instruction*, llvm::SmallVector<llvm::Value*, 4>> refs_consumed;
     llvm::DenseMap<llvm::Instruction*, llvm::SmallVector<llvm::Value*, 4>> refs_used;
     llvm::ValueMap<llvm::Value*, RefcountState> vars;
+    llvm::DenseSet<llvm::Instruction*> may_throw;
 
 public:
     llvm::Value* setType(llvm::Value* v, RefType reftype);
     llvm::Value* setNullable(llvm::Value* v, bool nullable = true);
     void refConsumed(llvm::Value* v, llvm::Instruction*);
     void refUsed(llvm::Value* v, llvm::Instruction*);
+    void setMayThrow(llvm::Instruction*);
     static void addRefcounts(IRGenState* state);
+    bool isNullable(llvm::Value* v);
 };
 }
 

src/codegen/irgen/irgenerator.cpp

@@ -447,7 +447,7 @@ private:
         if (unw_info.exc_dest == NO_CXX_INTERCEPTION) {
             needs_cxx_interception = false;
         } else {
-            bool needs_refcounting_fixup = true;
+            bool needs_refcounting_fixup = false;
             needs_cxx_interception = (target_exception_style == CXX && (needs_refcounting_fixup || unw_info.hasHandler()
                                                                         || irstate->getExceptionStyle() == CAPI));
         }
@@ -487,6 +487,9 @@ private:
         } else {
             llvm::CallInst* cs = getBuilder()->CreateCall(callee, args);
 
+            if (target_exception_style == CXX)
+                irstate->getRefcounts()->setMayThrow(cs);
+
             if (target_exception_style == CAPI)
                 checkAndPropagateCapiException(unw_info, cs, capi_exc_value);
 
@@ -1035,7 +1038,8 @@ private:
 
                 auto inst = emitter.createCall(UnwindInfo::cantUnwind(), g.funcs.setFrameExcInfo,
                                                { frame_info, converted_type->getValue(), converted_value->getValue(),
-                                                 converted_traceback->getValue() });
+                                                 converted_traceback->getValue() },
+                                               NOEXC);
                 emitter.refConsumed(converted_type->getValue(), inst);
                 emitter.refConsumed(converted_value->getValue(), inst);
                 emitter.refConsumed(converted_traceback->getValue(), inst);
@@ -3065,31 +3069,8 @@ public:
 
         emitter.getBuilder()->SetInsertPoint(cxx_exc_dest);
 
-        llvm::Function* _personality_func = g.stdlib_module->getFunction("__gxx_personality_v0");
-        assert(_personality_func);
-        llvm::Value* personality_func
-            = g.cur_module->getOrInsertFunction(_personality_func->getName(), _personality_func->getFunctionType());
-        assert(personality_func);
-        llvm::LandingPadInst* landing_pad = emitter.getBuilder()->CreateLandingPad(
-            llvm::StructType::create(std::vector<llvm::Type*>{ g.i8_ptr, g.i64 }), personality_func, 1);
-        landing_pad->addClause(getNullPtr(g.i8_ptr));
-
-        llvm::Value* cxaexc_pointer = emitter.getBuilder()->CreateExtractValue(landing_pad, { 0 });
-
-        llvm::Function* std_module_catch = g.stdlib_module->getFunction("__cxa_begin_catch");
-        auto begin_catch_func
-            = g.cur_module->getOrInsertFunction(std_module_catch->getName(), std_module_catch->getFunctionType());
-        assert(begin_catch_func);
-
-        llvm::Value* excinfo_pointer = emitter.getBuilder()->CreateCall(begin_catch_func, cxaexc_pointer);
-        llvm::Value* excinfo_pointer_casted
-            = emitter.getBuilder()->CreateBitCast(excinfo_pointer, g.llvm_excinfo_type->getPointerTo());
-
-        auto* builder = emitter.getBuilder();
-        llvm::Value* exc_type = builder->CreateLoad(builder->CreateConstInBoundsGEP2_32(excinfo_pointer_casted, 0, 0));
-        llvm::Value* exc_value = builder->CreateLoad(builder->CreateConstInBoundsGEP2_32(excinfo_pointer_casted, 0, 1));
-        llvm::Value* exc_traceback
-            = builder->CreateLoad(builder->CreateConstInBoundsGEP2_32(excinfo_pointer_casted, 0, 2));
+        llvm::Value* exc_type, *exc_value, *exc_traceback;
+        std::tie(exc_type, exc_value, exc_traceback) = createLandingpad(cxx_exc_dest);
         emitter.setType(exc_type, RefType::OWNED);
         emitter.setType(exc_value, RefType::OWNED);
         emitter.setType(exc_traceback, RefType::OWNED);
@@ -3101,23 +3082,24 @@ public:
                                                      new ConcreteCompilerVariable(UNKNOWN, exc_value),
                                                      new ConcreteCompilerVariable(UNKNOWN, exc_traceback)));
 
-            builder->CreateBr(final_dest);
+            emitter.getBuilder()->CreateBr(final_dest);
         } else if (irstate->getExceptionStyle() == CAPI) {
-            auto call_inst = builder->CreateCall3(g.funcs.PyErr_Restore, exc_type, exc_value, exc_traceback);
+            auto call_inst
+                = emitter.getBuilder()->CreateCall3(g.funcs.PyErr_Restore, exc_type, exc_value, exc_traceback);
             irstate->getRefcounts()->refConsumed(exc_type, call_inst);
             irstate->getRefcounts()->refConsumed(exc_value, call_inst);
             irstate->getRefcounts()->refConsumed(exc_traceback, call_inst);
-            builder->CreateCall(g.funcs.deinitFrame, irstate->getFrameInfoVar());
-            builder->CreateRet(getNullPtr(g.llvm_value_type_ptr));
+            emitter.getBuilder()->CreateCall(g.funcs.deinitFrame, irstate->getFrameInfoVar());
+            emitter.getBuilder()->CreateRet(getNullPtr(g.llvm_value_type_ptr));
         } else {
             // auto call_inst = emitter.createCall3(UnwindInfo(unw_info.current_stmt, NO_CXX_INTERCEPTION),
-            // g.funcs.rawThrow, exc_type, exc_value, exc_traceback);
-            auto call_inst = emitter.getBuilder()->CreateCall3(g.funcs.rawThrow, exc_type, exc_value, exc_traceback);
+            // g.funcs.rawReraise, exc_type, exc_value, exc_traceback);
+            auto call_inst = emitter.getBuilder()->CreateCall3(g.funcs.rawReraise, exc_type, exc_value, exc_traceback);
             irstate->getRefcounts()->refConsumed(exc_type, call_inst);
             irstate->getRefcounts()->refConsumed(exc_value, call_inst);
             irstate->getRefcounts()->refConsumed(exc_traceback, call_inst);
 
-            builder->CreateUnreachable();
+            emitter.getBuilder()->CreateUnreachable();
         }
 
         emitter.setCurrentBasicBlock(orig_block);
@@ -3135,6 +3117,37 @@ public:
     }
 };
 
+std::tuple<llvm::Value*, llvm::Value*, llvm::Value*> createLandingpad(llvm::BasicBlock* bb) {
+    assert(bb->begin() == bb->end());
+
+    llvm::IRBuilder<true> builder(bb);
+
+    llvm::Function* _personality_func = g.stdlib_module->getFunction("__gxx_personality_v0");
+    assert(_personality_func);
+    llvm::Value* personality_func
+        = g.cur_module->getOrInsertFunction(_personality_func->getName(), _personality_func->getFunctionType());
+    assert(personality_func);
+    llvm::LandingPadInst* landing_pad = builder.CreateLandingPad(
+        llvm::StructType::create(std::vector<llvm::Type*>{ g.i8_ptr, g.i64 }), personality_func, 1);
+    landing_pad->addClause(getNullPtr(g.i8_ptr));
+
+    llvm::Value* cxaexc_pointer = builder.CreateExtractValue(landing_pad, { 0 });
+
+    llvm::Function* std_module_catch = g.stdlib_module->getFunction("__cxa_begin_catch");
+    auto begin_catch_func
+        = g.cur_module->getOrInsertFunction(std_module_catch->getName(), std_module_catch->getFunctionType());
+    assert(begin_catch_func);
+
+    llvm::Value* excinfo_pointer = builder.CreateCall(begin_catch_func, cxaexc_pointer);
+    llvm::Value* excinfo_pointer_casted = builder.CreateBitCast(excinfo_pointer, g.llvm_excinfo_type->getPointerTo());
+
+    llvm::Value* exc_type = builder.CreateLoad(builder.CreateConstInBoundsGEP2_32(excinfo_pointer_casted, 0, 0));
+    llvm::Value* exc_value = builder.CreateLoad(builder.CreateConstInBoundsGEP2_32(excinfo_pointer_casted, 0, 1));
+    llvm::Value* exc_traceback = builder.CreateLoad(builder.CreateConstInBoundsGEP2_32(excinfo_pointer_casted, 0, 2));
+
+    return std::make_tuple(exc_type, exc_value, exc_traceback);
+}
+
 IRGenerator* createIRGenerator(IRGenState* irstate, std::unordered_map<CFGBlock*, llvm::BasicBlock*>& entry_blocks,
                                CFGBlock* myblock, TypeAnalysis* types) {
     return new IRGeneratorImpl(irstate, entry_blocks, myblock, types);

src/codegen/irgen/irgenerator.h

@@ -172,6 +172,8 @@ public:
     virtual CFGBlock* getCFGBlock() = 0;
 };
 
+std::tuple<llvm::Value*, llvm::Value*, llvm::Value*> createLandingpad(llvm::BasicBlock*);
+
 class IREmitter;
 class AST_Call;
 IREmitter* createIREmitter(IRGenState* irstate, llvm::BasicBlock*& curblock, IRGenerator* irgenerator = NULL);

src/codegen/irgen/refcounts.cpp

@@ -70,6 +70,11 @@ llvm::Value* RefcountTracker::setNullable(llvm::Value* v, bool nullable) {
     return v;
 }
 
+bool RefcountTracker::isNullable(llvm::Value* v) {
+    assert(vars.count(v));
+    return vars.lookup(v).nullable;
+}
+
 void RefcountTracker::refConsumed(llvm::Value* v, llvm::Instruction* inst) {
     assert(this->vars[v].reftype != RefType::UNKNOWN);
 
@@ -82,6 +87,12 @@ void RefcountTracker::refUsed(llvm::Value* v, llvm::Instruction* inst) {
     this->refs_used[inst].push_back(v);
 }
 
+void RefcountTracker::setMayThrow(llvm::Instruction* inst) {
+    assert(!may_throw.count(inst));
+    this->may_throw.insert(inst);
+}
+
+
 void remapPhis(llvm::BasicBlock* in_block, llvm::BasicBlock* from_block, llvm::BasicBlock* new_from_block) {
     for (llvm::Instruction& i : *in_block) {
         llvm::Instruction* I = &i;
@@ -227,6 +238,8 @@ void addDecrefs(llvm::Value* v, bool nullable, int num_refs, llvm::Instruction*
         raise(SIGTRAP);
     }
 
+    // TODO -- assert that v isn't a constant None?  Implies wasted extra increfs/decrefs
+
     if (isa<ConstantPointerNull>(v)) {
         assert(nullable);
         return;
@@ -319,6 +332,51 @@ void addDecrefs(llvm::Value* v, bool nullable, int num_refs, llvm::Instruction*
     builder.SetInsertPoint(continue_block);
 }
 
+void addCXXFixup(llvm::Instruction* inst, const llvm::SmallVector<llvm::TrackingVH<llvm::Value>, 4>& to_decref,
+                 RefcountTracker* rt) {
+    // inst->getParent()->getParent()->dump();
+    // inst->dump();
+
+    ASSERT(!llvm::isa<llvm::InvokeInst>(inst),
+           "don't need a fixup here!"); // could either not ask for the fixup or maybe just skip it here
+    assert(llvm::isa<llvm::CallInst>(inst));
+
+    llvm::CallInst* call = llvm::cast<llvm::CallInst>(inst);
+
+    llvm::BasicBlock* cur_block = inst->getParent();
+    llvm::BasicBlock* continue_block = cur_block->splitBasicBlock(inst);
+    llvm::BasicBlock* fixup_block
+        = llvm::BasicBlock::Create(g.context, "cxx_fixup", inst->getParent()->getParent(), continue_block);
+
+    assert(llvm::isa<llvm::BranchInst>(cur_block->getTerminator()));
+    cur_block->getTerminator()->eraseFromParent();
+
+    // llvm::SmallVector<llvm::Value*, 4> args(call->arg_begin(), call->arg_end());
+    llvm::SmallVector<llvm::Value*, 4> args(call->arg_operands().begin(), call->arg_operands().end());
+    llvm::InvokeInst* new_invoke
+        = InvokeInst::Create(call->getCalledValue(), continue_block, fixup_block, args, call->getName(), cur_block);
+    new_invoke->setAttributes(call->getAttributes());
+    new_invoke->setDebugLoc(call->getDebugLoc());
+    assert(!call->hasMetadataOtherThanDebugLoc());
+    call->replaceAllUsesWith(new_invoke);
+    call->eraseFromParent();
+
+    llvm::Value* exc_type, *exc_value, *exc_traceback;
+    std::tie(exc_type, exc_value, exc_traceback) = createLandingpad(fixup_block);
+
+    llvm::IRBuilder<true> builder(fixup_block);
+    auto rethrow = builder.CreateCall3(g.funcs.rawReraise, exc_type, exc_value, exc_traceback);
+    builder.CreateUnreachable();
+
+    // fixup_block->dump();
+
+    for (auto&& v : to_decref) {
+        addDecrefs(v, rt->isNullable(v), 1, rethrow);
+    }
+
+    // new_invoke->getParent()->getParent()->dump();
+}
+
 // TODO: this should be cleaned up and moved to src/core/
 template <typename K, typename V> class OrderedMap {
 private:
@@ -641,7 +699,7 @@ void RefcountTracker::addRefcounts(IRGenState* irstate) {
 #endif
 
     struct RefOp {
-        llvm::Value* operand;
+        llvm::TrackingVH<llvm::Value> operand;
         bool nullable;
         int num_refs;
 
@@ -660,6 +718,12 @@ void RefcountTracker::addRefcounts(IRGenState* irstate) {
 
         llvm::SmallVector<RefOp, 4> increfs;
         llvm::SmallVector<RefOp, 4> decrefs;
+
+        struct CXXFixup {
+            llvm::Instruction* inst;
+            llvm::SmallVector<llvm::TrackingVH<llvm::Value>, 4> to_decref;
+        };
+        llvm::SmallVector<CXXFixup, 4> cxx_fixups;
     };
     llvm::DenseMap<llvm::BasicBlock*, RefState> states;
 
@@ -668,12 +732,13 @@ void RefcountTracker::addRefcounts(IRGenState* irstate) {
         orderer.add(&BB);
     }
 
-    std::vector<llvm::Instruction*> tracked_instructions;
+    std::vector<llvm::InvokeInst*> invokes;
     for (auto&& II : llvm::inst_range(f)) {
         llvm::Instruction* inst = &II;
         if (!rt->vars.count(inst))
             continue;
-        tracked_instructions.push_back(inst);
+        if (auto ii = dyn_cast<InvokeInst>(inst))
+            invokes.push_back(ii);
     }
 
     while (llvm::BasicBlock* bb = orderer.pop()) {
@@ -703,6 +768,7 @@ void RefcountTracker::addRefcounts(IRGenState* irstate) {
         state.ending_refs.clear();
         state.increfs.clear();
         state.decrefs.clear();
+        state.cxx_fixups.clear();
 
         // Compute the incoming refstate based on the refstate of any successor nodes
         llvm::SmallVector<llvm::BasicBlock*, 4> successors;
@@ -782,9 +848,55 @@ void RefcountTracker::addRefcounts(IRGenState* irstate) {
             // - the phi-node-generator is supposed to handle that by putting a refConsumed on the terminator of the
             // previous block
             // - that refConsumed will caus a use as well.
+            auto inst = &I;
+
+            if (!isa<InvokeInst>(inst) && rt->vars.count(&I)) {
+                const auto&& rstate = rt->vars.lookup(inst);
+                int starting_refs = (rstate.reftype == RefType::OWNED ? 1 : 0);
+                if (state.ending_refs[inst] != starting_refs) {
+                    llvm::Instruction* insertion_pt = NULL;
+                    llvm::BasicBlock* insertion_block = NULL, * insertion_from_block = NULL;
+                    insertion_pt = inst->getNextNode();
+                    while (llvm::isa<llvm::PHINode>(insertion_pt)) {
+                        insertion_pt = insertion_pt->getNextNode();
+                    }
+
+                    if (state.ending_refs[inst] < starting_refs) {
+                        assert(rstate.reftype == RefType::OWNED);
+                        state.decrefs.push_back(RefOp({ inst, rstate.nullable, starting_refs - state.ending_refs[inst],
+                                                        insertion_pt, insertion_block, insertion_from_block }));
+                    } else {
+                        state.increfs.push_back(RefOp({ inst, rstate.nullable, state.ending_refs[inst] - starting_refs,
+                                                        insertion_pt, insertion_block, insertion_from_block }));
+                    }
+                }
+                state.ending_refs.erase(inst);
+            }
+
             if (llvm::isa<llvm::PHINode>(&I))
                 continue;
 
+
+            // If we are about to insert a CXX fixup, do the increfs after the call, rather than trying to push
+            // them before the call and having to insert decrefs on the fixup path.
+            if (rt->may_throw.count(&I)) {
+                llvm::SmallVector<llvm::Value*, 4> to_erase;
+                for (auto&& p : state.ending_refs) {
+                    int needed_refs = (rt->vars.lookup(p.first).reftype == RefType::OWNED ? 1 : 0);
+                    if (p.second > needed_refs) {
+                        state.increfs.push_back(RefOp({ p.first, rt->vars.lookup(p.first).nullable,
+                                                        p.second - needed_refs, I.getNextNode(), NULL, NULL }));
+                    }
+
+                    state.ending_refs[p.first] = needed_refs;
+                    if (needed_refs == 0)
+                        to_erase.push_back(p.first);
+                }
+
+                for (auto v : to_erase)
+                    state.ending_refs.erase(v);
+            }
+
             llvm::DenseMap<llvm::Value*, int> num_consumed_by_inst;
             OrderedMap<llvm::Value*, int> num_times_as_op;
 
@@ -807,6 +919,7 @@ void RefcountTracker::addRefcounts(IRGenState* irstate) {
                 num_times_as_op[op]++;
             }
 
+            // First, calculate anything we need to keep alive through the end of the function call:
             for (auto&& p : num_times_as_op) {
                 auto& op = p.first;
 
@@ -816,7 +929,7 @@ void RefcountTracker::addRefcounts(IRGenState* irstate) {
                     num_consumed = it->second;
 
                 if (num_times_as_op[op] > num_consumed) {
-                    if (rt->vars[op].reftype == RefType::OWNED) {
+                    if (rt->vars.lookup(op).reftype == RefType::OWNED) {
                         if (state.ending_refs[op] == 0) {
                             // llvm::outs() << "Last use of " << *op << " is at " << I << "; adding a decref after\n";
 
@@ -839,6 +952,34 @@ void RefcountTracker::addRefcounts(IRGenState* irstate) {
                         }
                     }
                 }
+            }
+
+            if (rt->may_throw.count(&I)) {
+                // TODO: pump out any increfs rather than pushing them before this.
+
+                state.cxx_fixups.emplace_back();
+                auto&& fixup = state.cxx_fixups.back();
+                fixup.inst = &I;
+                for (auto&& p : state.ending_refs) {
+                    for (int i = 0; i < p.second; i++) {
+                        assert(rt->vars.count(p.first));
+                        fixup.to_decref.push_back(p.first);
+                    }
+                }
+
+                if (fixup.to_decref.empty())
+                    state.cxx_fixups.pop_back();
+            }
+
+            // Lastly, take care of any stolen refs.  This happens regardless of whether an exception gets thrown,
+            // so it goes after that handling (since we are processing in reverse).
+            for (auto&& p : num_times_as_op) {
+                auto& op = p.first;
+
+                auto&& it = num_consumed_by_inst.find(op);
+                int num_consumed = 0;
+                if (it != num_consumed_by_inst.end())
+                    num_consumed = it->second;
 
                 if (num_consumed)
                     state.ending_refs[op] += num_consumed;
@@ -855,43 +996,32 @@ void RefcountTracker::addRefcounts(IRGenState* irstate) {
         }
 
 
-        // size_t hash = 0;
-        // Handle variables that were defined in this BB:
-        for (auto&& inst : tracked_instructions) {
-            const auto&& rstate = rt->vars.lookup(inst);
-
-            // hash = hash * 31 + std::hash<llvm::StringRef>()(inst->getName());
+        // Invokes are special.  Handle them here by treating them as if they happened in their normal-dest block.
+        for (InvokeInst* ii : invokes) {
+            const auto&& rstate = rt->vars.lookup(ii);
 
-            // Invokes are special.  Handle them here by treating them as if they happened in their normal-dest block.
-            llvm::InvokeInst* ii = llvm::dyn_cast<llvm::InvokeInst>(inst);
-            if ((!ii && inst->getParent() == &BB) || (ii && ii->getNormalDest() == &BB)) {
+            if (ii->getNormalDest() == &BB) {
+                // TODO: duplicated with the non-invoke code
                 int starting_refs = (rstate.reftype == RefType::OWNED ? 1 : 0);
-                if (state.ending_refs[inst] != starting_refs) {
+                if (state.ending_refs[ii] != starting_refs) {
                     llvm::Instruction* insertion_pt = NULL;
                     llvm::BasicBlock* insertion_block = NULL, * insertion_from_block = NULL;
-                    if (ii) {
-                        insertion_block = bb;
-                        insertion_from_block = inst->getParent();
-                    } else {
-                        insertion_pt = inst->getNextNode();
-                        while (llvm::isa<llvm::PHINode>(insertion_pt)) {
-                            insertion_pt = insertion_pt->getNextNode();
-                        }
-                    }
 
-                    if (state.ending_refs[inst] < starting_refs) {
+                    insertion_block = bb;
+                    insertion_from_block = ii->getParent();
+
+                    if (state.ending_refs[ii] < starting_refs) {
                         assert(rstate.reftype == RefType::OWNED);
-                        state.decrefs.push_back(RefOp({ inst, rstate.nullable, starting_refs - state.ending_refs[inst],
+                        state.decrefs.push_back(RefOp({ ii, rstate.nullable, starting_refs - state.ending_refs[ii],
                                                         insertion_pt, insertion_block, insertion_from_block }));
                     } else {
-                        state.increfs.push_back(RefOp({ inst, rstate.nullable, state.ending_refs[inst] - starting_refs,
+                        state.increfs.push_back(RefOp({ ii, rstate.nullable, state.ending_refs[ii] - starting_refs,
                                                         insertion_pt, insertion_block, insertion_from_block }));
                     }
                 }
-                state.ending_refs.erase(inst);
+                state.ending_refs.erase(ii);
             }
         }
-        // errs() << "DETERMINISM: rt.vars name hash: " << hash << '\n';
 
         // If this is the entry block, finish dealing with the ref state rather than handing off to a predecessor
         if (&BB == &BB.getParent()->front()) {
@@ -964,24 +1094,23 @@ void RefcountTracker::addRefcounts(IRGenState* irstate) {
     for (auto bb : basic_blocks) {
         auto&& state = states[bb];
         for (auto& op : state.increfs) {
+            assert(rt->vars.count(op.operand));
             auto insertion_pt = op.insertion_inst;
             if (!insertion_pt)
                 insertion_pt = findInsertionPoint(op.insertion_bb, op.insertion_from_bb, insertion_pts);
             addIncrefs(op.operand, op.nullable, op.num_refs, insertion_pt);
         }
         for (auto& op : state.decrefs) {
+            assert(rt->vars.count(op.operand));
             auto insertion_pt = op.insertion_inst;
             if (!insertion_pt)
                 insertion_pt = findInsertionPoint(op.insertion_bb, op.insertion_from_bb, insertion_pts);
             addDecrefs(op.operand, op.nullable, op.num_refs, insertion_pt);
         }
-    }
 
-    if (VERBOSITY() >= 2) {
-        fprintf(stderr, "After refcounts:\n");
-        fprintf(stderr, "\033[35m");
-        dumpPrettyIR(f);
-        fprintf(stderr, "\033[0m");
+        for (auto&& fixup : state.cxx_fixups) {
+            addCXXFixup(fixup.inst, fixup.to_decref, rt);
+        }
     }
 
     long us = _t.end();

src/codegen/runtime_hooks.cpp

@@ -315,7 +315,7 @@ void initGlobalFuncs(GlobalState& g) {
     GET(raise0_capi);
     GET(raise3);
     GET(raise3_capi);
-    GET(rawThrow);
+    GET(rawReraise);
     GET(PyErr_Fetch);
     GET(PyErr_NormalizeException);
     GET(PyErr_Restore);

src/codegen/runtime_hooks.h

@@ -51,7 +51,7 @@ struct GlobalFuncs {
     llvm::Value* boxedLocalsSet, *boxedLocalsGet, *boxedLocalsDel;
 
     llvm::Value* __cxa_end_catch;
-    llvm::Value* raise0, *raise0_capi, *raise3, *raise3_capi, *rawThrow;
+    llvm::Value* raise0, *raise0_capi, *raise3, *raise3_capi, *rawReraise;
     llvm::Value* PyErr_Fetch, *PyErr_NormalizeException, *PyErr_Restore, *caughtCapiException, *reraiseCapiExcAsCxx;
     llvm::Value* deopt;
     llvm::Value* checkRefs;

src/runtime/exceptions.cpp

@@ -282,8 +282,7 @@ extern "C" void reraiseCapiExcAsCxx() {
     throw e;
 }
 
-// XXX rename this
-extern "C" void rawThrow(Box* type, Box* value, Box* tb) {
+extern "C" void rawReraise(Box* type, Box* value, Box* tb) {
     startReraise();
     throw ExcInfo(type, value, tb);
 }

src/runtime/inline/link_forcer.cpp

@@ -130,7 +130,7 @@ void force() {
     FORCE(raise0_capi);
     FORCE(raise3);
     FORCE(raise3_capi);
-    FORCE(rawThrow);
+    FORCE(rawReraise);
     FORCE(PyErr_Fetch);
     FORCE(PyErr_NormalizeException);
     FORCE(PyErr_Restore);

src/runtime/objmodel.h

@@ -37,7 +37,7 @@ extern "C" void raise0(ExcInfo* frame_exc_info) __attribute__((__noreturn__));
 extern "C" void raise0_capi(ExcInfo* frame_exc_info) noexcept;
 extern "C" void raise3(Box*, Box*, Box*) __attribute__((__noreturn__));
 extern "C" void raise3_capi(Box*, Box*, Box*) noexcept;
-extern "C" void rawThrow(Box*, Box*, Box*) __attribute__((__noreturn__));
+extern "C" void rawReraise(Box*, Box*, Box*) __attribute__((__noreturn__));
 void raiseExc(STOLEN(Box*) exc_obj) __attribute__((__noreturn__));
 void _printStacktrace();