Merge pull request #790 from kmod/exceptions2

Make Raise and Subscript exprs use capi exceptions

src/codegen/compvars.cpp

@@ -337,6 +337,10 @@ public:
                               CompilerVariable* slice) override {
         ConcreteCompilerVariable* converted_slice = slice->makeConverted(emitter, slice->getBoxType());
 
+        ExceptionStyle target_exception_style = CXX;
+        if (FORCE_LLVM_CAPI || info.unw_info.capi_exc_dest)
+            target_exception_style = CAPI;
+
         bool do_patchpoint = ENABLE_ICGETITEMS;
         llvm::Value* rtn;
         if (do_patchpoint) {
@@ -346,13 +350,20 @@ public:
             llvm_args.push_back(var->getValue());
             llvm_args.push_back(converted_slice->getValue());
 
-            llvm::Value* uncasted = emitter.createIC(pp, (void*)pyston::getitem, llvm_args, info.unw_info);
+            llvm::Value* uncasted
+                = emitter.createIC(pp, (void*)(target_exception_style == CAPI ? pyston::getitem_capi : pyston::getitem),
+                                   llvm_args, info.unw_info, target_exception_style);
             rtn = emitter.getBuilder()->CreateIntToPtr(uncasted, g.llvm_value_type_ptr);
         } else {
-            rtn = emitter.createCall2(info.unw_info, g.funcs.getitem, var->getValue(), converted_slice->getValue());
+            rtn = emitter.createCall2(info.unw_info,
+                                      target_exception_style == CAPI ? g.funcs.getitem_capi : g.funcs.getitem,
+                                      var->getValue(), converted_slice->getValue(), target_exception_style);
         }
-
         converted_slice->decvref(emitter);
+
+        if (target_exception_style == CAPI)
+            emitter.checkAndPropagateCapiException(info.unw_info, rtn, getNullPtr(g.llvm_value_type_ptr));
+
         return new ConcreteCompilerVariable(UNKNOWN, rtn, true);
     }
 
@@ -490,7 +501,7 @@ CompilerVariable* UnknownType::getattr(IREmitter& emitter, const OpInfo& info, C
     llvm::Value* rtn_val = NULL;
 
     ExceptionStyle target_exception_style = CXX;
-    if (info.unw_info.capi_exc_dest)
+    if (info.unw_info.capi_exc_dest || (!cls_only && FORCE_LLVM_CAPI))
         target_exception_style = CAPI;
 
     llvm::Value* llvm_func;
@@ -1494,7 +1505,7 @@ public:
         if (canStaticallyResolveGetattrs()) {
             Box* rtattr = typeLookup(cls, attr, nullptr);
             if (rtattr == NULL) {
-                ExceptionStyle exception_style = info.unw_info.capi_exc_dest ? CAPI : CXX;
+                ExceptionStyle exception_style = (FORCE_LLVM_CAPI || info.unw_info.capi_exc_dest) ? CAPI : CXX;
                 llvm::Value* raise_func = exception_style == CXX ? g.funcs.raiseAttributeErrorStr
                                                                  : g.funcs.raiseAttributeErrorStrCapi;
                 llvm::CallSite call = emitter.createCall3(
@@ -1546,7 +1557,7 @@ public:
                                                   BoxedString* attr, bool clsonly, ArgPassSpec argspec,
                                                   const std::vector<CompilerVariable*>& args,
                                                   const std::vector<BoxedString*>* keyword_names,
-                                                  bool* no_attribute = NULL) {
+                                                  bool* no_attribute = NULL, ExceptionStyle exception_style = CXX) {
         if (!canStaticallyResolveGetattrs())
             return NULL;
 
@@ -1555,10 +1566,18 @@ public:
             if (no_attribute) {
                 *no_attribute = true;
             } else {
+                llvm::Value* raise_func = exception_style == CXX ? g.funcs.raiseAttributeErrorStr
+                                                                 : g.funcs.raiseAttributeErrorStrCapi;
+
                 llvm::CallSite call = emitter.createCall3(
-                    info.unw_info, g.funcs.raiseAttributeErrorStr, embedRelocatablePtr(cls->tp_name, g.i8_ptr),
-                    embedRelocatablePtr(attr->data(), g.i8_ptr), getConstantInt(attr->size(), g.i64));
-                call.setDoesNotReturn();
+                    info.unw_info, raise_func, embedRelocatablePtr(cls->tp_name, g.i8_ptr),
+                    embedRelocatablePtr(attr->data(), g.i8_ptr), getConstantInt(attr->size(), g.i64), exception_style);
+                if (exception_style == CAPI) {
+                    emitter.checkAndPropagateCapiException(info.unw_info, getNullPtr(g.llvm_value_type_ptr),
+                                                           getNullPtr(g.llvm_value_type_ptr));
+                } else {
+                    call.setDoesNotReturn();
+                }
             }
             return undefVariable();
         }
@@ -1589,6 +1608,9 @@ public:
             cf = cl->versions[i];
             assert(cf->spec->arg_types.size() == cl->numReceivedArgs());
 
+            if (cf->exception_style != exception_style)
+                continue;
+
             bool fits = true;
             for (int j = 0; j < args.size(); j++) {
                 if (!args[j]->canConvertTo(cf->spec->arg_types[j + 1])) {
@@ -1603,8 +1625,12 @@ public:
             break;
         }
 
-        assert(found);
-        assert(cf->code);
+        if (!found && exception_style == CAPI) {
+            std::string name = g.func_addr_registry.getFuncNameAtAddress(cl->versions[0]->code, true);
+            RELEASE_ASSERT(0, "Please define a capi variant for %s", name.c_str());
+        }
+        RELEASE_ASSERT(found, "");
+        RELEASE_ASSERT(cf->code, "");
 
         std::vector<llvm::Type*> arg_types;
         RELEASE_ASSERT(paramspec.num_args == cl->numReceivedArgs(), "");
@@ -1733,8 +1759,13 @@ public:
     CompilerVariable* getitem(IREmitter& emitter, const OpInfo& info, VAR* var, CompilerVariable* slice) override {
         static BoxedString* attr = internStringImmortal("__getitem__");
         bool no_attribute = false;
+
+        ExceptionStyle exception_style = CXX;
+        if (FORCE_LLVM_CAPI || info.unw_info.capi_exc_dest)
+            exception_style = CAPI;
+
         ConcreteCompilerVariable* called_constant = tryCallattrConstant(
-            emitter, info, var, attr, true, ArgPassSpec(1, 0, 0, 0), { slice }, NULL, &no_attribute);
+            emitter, info, var, attr, true, ArgPassSpec(1, 0, 0, 0), { slice }, NULL, &no_attribute, exception_style);
 
         if (no_attribute) {
             assert(called_constant->getType() == UNDEF);
@@ -2239,9 +2270,20 @@ public:
                     rtn->incvref();
                     return rtn;
                 } else {
-                    llvm::CallSite call = emitter.createCall(info.unw_info, g.funcs.raiseIndexErrorStr,
-                                                             embedConstantPtr("tuple", g.i8_ptr));
-                    call.setDoesNotReturn();
+                    ExceptionStyle target_exception_style = CXX;
+                    if (FORCE_LLVM_CAPI || info.unw_info.capi_exc_dest)
+                        target_exception_style = CAPI;
+
+                    if (target_exception_style == CAPI) {
+                        llvm::CallSite call = emitter.createCall(info.unw_info, g.funcs.raiseIndexErrorStrCapi,
+                                                                 embedConstantPtr("tuple", g.i8_ptr), CAPI);
+                        emitter.checkAndPropagateCapiException(info.unw_info, getNullPtr(g.llvm_value_type_ptr),
+                                                               getNullPtr(g.llvm_value_type_ptr));
+                    } else {
+                        llvm::CallSite call = emitter.createCall(info.unw_info, g.funcs.raiseIndexErrorStr,
+                                                                 embedConstantPtr("tuple", g.i8_ptr), CXX);
+                        call.setDoesNotReturn();
+                    }
                     return undefVariable();
                 }
             }

src/codegen/irgen/hooks.h

@@ -40,6 +40,8 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm);
 CompiledFunction* cfForMachineFunctionName(const std::string&);
 
 extern "C" void capiExcCaughtInJit(AST_stmt* current_stmt, void* source_info);
+// This is just meant for the use of the JIT (normal runtime code should call throwCAPIException)
+extern "C" void reraiseJitCapiExc() __attribute__((noreturn));
 
 extern "C" Box* exec(Box* boxedCode, Box* globals, Box* locals, FutureFlags caller_future_flags);
 extern "C" Box* eval(Box* boxedCode, Box* globals, Box* locals);

src/codegen/irgen/irgenerator.cpp

@@ -107,8 +107,15 @@ ExceptionStyle IRGenState::getLandingpadStyle(AST_Invoke* invoke) {
 
     r = CXX; // default
 
-    // print_ast(invoke);
-    // printf("\n");
+    if (invoke->stmt->type == AST_TYPE::Raise) {
+        AST_Raise* raise_stmt = ast_cast<AST_Raise>(invoke->stmt);
+        // Currently can't do a re-raise with a capi exception:
+        if (raise_stmt->arg0 && !raise_stmt->arg2) {
+            r = CAPI;
+            return r;
+        }
+    }
+
     AST_expr* expr = NULL;
 
     if (invoke->stmt->type == AST_TYPE::Assign) {
@@ -129,7 +136,7 @@ ExceptionStyle IRGenState::getLandingpadStyle(AST_Invoke* invoke) {
         return r;
     }
 
-    if (expr->type == AST_TYPE::Attribute) {
+    if (expr->type == AST_TYPE::Attribute || expr->type == AST_TYPE::Subscript) {
         r = CAPI;
         // printf("Doing a capi exception to %d\n", invoke->exc_dest->idx);
         return r;
@@ -298,42 +305,6 @@ private:
         }
     }
 
-    void checkAndPropagateCapiException(const UnwindInfo& unw_info, llvm::Value* returned_val, llvm::Value* exc_val,
-                                        bool double_check) override {
-        assert(!double_check); // need to call PyErr_Occurred
-
-        llvm::BasicBlock* normal_dest
-            = llvm::BasicBlock::Create(g.context, curblock->getName(), irstate->getLLVMFunction());
-        normal_dest->moveAfter(curblock);
-
-        llvm::BasicBlock* exc_dest;
-        bool exc_caught;
-        if (unw_info.hasHandler()) {
-            assert(unw_info.capi_exc_dest);
-            exc_dest = unw_info.capi_exc_dest;
-            exc_caught = true;
-        } else {
-            exc_dest = llvm::BasicBlock::Create(g.context, curblock->getName() + "_exc", irstate->getLLVMFunction());
-            exc_dest->moveAfter(curblock);
-            exc_caught = false;
-        }
-
-        assert(returned_val->getType() == exc_val->getType());
-        llvm::Value* check_val = getBuilder()->CreateICmpEQ(returned_val, exc_val);
-        llvm::BranchInst* nullcheck = getBuilder()->CreateCondBr(check_val, exc_dest, normal_dest);
-
-        setCurrentBasicBlock(exc_dest);
-        getBuilder()->CreateCall2(g.funcs.capiExcCaughtInJit,
-                                  embedRelocatablePtr(unw_info.current_stmt, g.llvm_aststmt_type_ptr),
-                                  embedRelocatablePtr(irstate->getSourceInfo(), g.i8_ptr));
-
-        if (!exc_caught) {
-            RELEASE_ASSERT(0, "need to implement this");
-        }
-
-        setCurrentBasicBlock(normal_dest);
-    }
-
     llvm::CallSite emitPatchpoint(llvm::Type* return_type, const ICSetupInfo* pp, llvm::Value* func,
                                   const std::vector<llvm::Value*>& args,
                                   const std::vector<llvm::Value*>& ic_stackmap_args, const UnwindInfo& unw_info,
@@ -507,6 +478,43 @@ public:
         return rtn.getInstruction();
     }
 
+    void checkAndPropagateCapiException(const UnwindInfo& unw_info, llvm::Value* returned_val, llvm::Value* exc_val,
+                                        bool double_check = false) override {
+        assert(!double_check); // need to call PyErr_Occurred
+
+        llvm::BasicBlock* normal_dest
+            = llvm::BasicBlock::Create(g.context, curblock->getName(), irstate->getLLVMFunction());
+        normal_dest->moveAfter(curblock);
+
+        llvm::BasicBlock* exc_dest;
+        bool exc_caught;
+        if (unw_info.hasHandler()) {
+            assert(unw_info.capi_exc_dest);
+            exc_dest = unw_info.capi_exc_dest;
+            exc_caught = true;
+        } else {
+            exc_dest = llvm::BasicBlock::Create(g.context, curblock->getName() + "_exc", irstate->getLLVMFunction());
+            exc_dest->moveAfter(curblock);
+            exc_caught = false;
+        }
+
+        assert(returned_val->getType() == exc_val->getType());
+        llvm::Value* check_val = getBuilder()->CreateICmpEQ(returned_val, exc_val);
+        llvm::BranchInst* nullcheck = getBuilder()->CreateCondBr(check_val, exc_dest, normal_dest);
+
+        setCurrentBasicBlock(exc_dest);
+        getBuilder()->CreateCall2(g.funcs.capiExcCaughtInJit,
+                                  embedRelocatablePtr(unw_info.current_stmt, g.llvm_aststmt_type_ptr),
+                                  embedRelocatablePtr(irstate->getSourceInfo(), g.i8_ptr));
+
+        if (!exc_caught) {
+            getBuilder()->CreateCall(g.funcs.reraiseJitCapiExc);
+            getBuilder()->CreateUnreachable();
+        }
+
+        setCurrentBasicBlock(normal_dest);
+    }
+
     Box* getIntConstant(int64_t n) override { return irstate->getSourceInfo()->parent_module->getIntConstant(n); }
 
     Box* getFloatConstant(double d) override { return irstate->getSourceInfo()->parent_module->getFloatConstant(d); }
@@ -2310,10 +2318,15 @@ private:
         // It looks like ommitting the second and third arguments are equivalent to passing None,
         // but ommitting the first argument is *not* the same as passing None.
 
+        ExceptionStyle target_exception_style = CXX;
+        if (unw_info.capi_exc_dest || (FORCE_LLVM_CAPI && node->arg0 && !node->arg2))
+            target_exception_style = CAPI;
+
         if (node->arg0 == NULL) {
             assert(!node->arg1);
             assert(!node->arg2);
 
+            assert(target_exception_style == CXX);
             emitter.createCall(unw_info, g.funcs.raise0, std::vector<llvm::Value*>());
             emitter.getBuilder()->CreateUnreachable();
 
@@ -2333,7 +2346,13 @@ private:
             }
         }
 
-        emitter.createCall(unw_info, g.funcs.raise3, args);
+        if (target_exception_style == CAPI) {
+            emitter.createCall(unw_info, g.funcs.raise3_capi, args, CAPI);
+            emitter.checkAndPropagateCapiException(unw_info, getNullPtr(g.llvm_value_type_ptr),
+                                                   getNullPtr(g.llvm_value_type_ptr));
+        } else {
+            emitter.createCall(unw_info, g.funcs.raise3, args, CXX);
+        }
         emitter.getBuilder()->CreateUnreachable();
 
         endBlock(DEAD);

src/codegen/runtime_hooks.cpp

@@ -199,6 +199,7 @@ void initGlobalFuncs(GlobalState& g) {
     GET(setattr);
     GET(delattr);
     GET(getitem);
+    GET(getitem_capi);
     GET(setitem);
     GET(delitem);
     GET(getGlobal);
@@ -227,6 +228,7 @@ void initGlobalFuncs(GlobalState& g) {
     GET(raiseAttributeErrorCapi);
     GET(raiseAttributeErrorStrCapi);
     GET(raiseIndexErrorStr);
+    GET(raiseIndexErrorStrCapi);
     GET(raiseNotIterableError);
     GET(assertNameDefined);
     GET(assertFailDerefNameDefined);
@@ -273,9 +275,11 @@ void initGlobalFuncs(GlobalState& g) {
     g.funcs.__cxa_end_catch = addFunc((void*)__cxa_end_catch, g.void_);
     GET(raise0);
     GET(raise3);
+    GET(raise3_capi);
     GET(PyErr_Fetch);
     GET(PyErr_NormalizeException);
     GET(capiExcCaughtInJit);
+    GET(reraiseJitCapiExc);
     GET(deopt);
 
     GET(div_float_float);

src/codegen/runtime_hooks.h

@@ -36,12 +36,12 @@ struct GlobalFuncs {
         *boxBool, *unboxBool, *createTuple, *createDict, *createList, *createSlice, *createUserClass, *createClosure,
         *createGenerator, *createSet;
     llvm::Value* getattr, *getattr_capi, *setattr, *delattr, *delitem, *delGlobal, *nonzero, *binop, *compare,
-        *augbinop, *unboxedLen, *getitem, *getclsattr, *getGlobal, *setitem, *unaryop, *import, *importFrom,
-        *importStar, *repr, *str, *strOrUnicode, *exceptionMatches, *yield, *getiterHelper, *hasnext;
+        *augbinop, *unboxedLen, *getitem, *getitem_capi, *getclsattr, *getGlobal, *setitem, *unaryop, *import,
+        *importFrom, *importStar, *repr, *str, *strOrUnicode, *exceptionMatches, *yield, *getiterHelper, *hasnext;
 
     llvm::Value* unpackIntoArray, *raiseAttributeError, *raiseAttributeErrorStr, *raiseAttributeErrorCapi,
-        *raiseAttributeErrorStrCapi, *raiseNotIterableError, *raiseIndexErrorStr, *assertNameDefined, *assertFail,
-        *assertFailDerefNameDefined;
+        *raiseAttributeErrorStrCapi, *raiseNotIterableError, *raiseIndexErrorStr, *raiseIndexErrorStrCapi,
+        *assertNameDefined, *assertFail, *assertFailDerefNameDefined;
     llvm::Value* printFloat, *listAppendInternal, *getSysStdout;
     llvm::Value* runtimeCall0, *runtimeCall1, *runtimeCall2, *runtimeCall3, *runtimeCall, *runtimeCallN;
     llvm::Value* callattr0, *callattr1, *callattr2, *callattr3, *callattr, *callattrN;
@@ -50,8 +50,8 @@ struct GlobalFuncs {
     llvm::Value* boxedLocalsSet, *boxedLocalsGet, *boxedLocalsDel;
 
     llvm::Value* __cxa_end_catch;
-    llvm::Value* raise0, *raise3;
-    llvm::Value* PyErr_Fetch, *PyErr_NormalizeException, *capiExcCaughtInJit;
+    llvm::Value* raise0, *raise3, *raise3_capi;
+    llvm::Value* PyErr_Fetch, *PyErr_NormalizeException, *capiExcCaughtInJit, *reraiseJitCapiExc;
     llvm::Value* deopt;
 
     llvm::Value* div_float_float, *floordiv_float_float, *mod_float_float, *pow_float_float;

src/codegen/unwinding.cpp

@@ -481,6 +481,16 @@ bool frameIsPythonFrame(unw_word_t ip, unw_word_t bp, unw_cursor_t* cursor, Pyth
     return true;
 }
 
+static const LineInfo lineInfoForFrame(PythonFrameIteratorImpl* frame_it) {
+    AST_stmt* current_stmt = frame_it->getCurrentStatement();
+    auto* cl = frame_it->getCL();
+    assert(cl);
+
+    auto source = cl->source.get();
+
+    return LineInfo(current_stmt->lineno, current_stmt->col_offset, source->fn, source->getName());
+}
+
 class PythonUnwindSession : public Box {
     ExcInfo exc_info;
     bool skip;
@@ -518,12 +528,13 @@ public:
         stat.log(t.end());
     }
 
-    void addTraceback(const LineInfo& line_info) {
+    void addTraceback(PythonFrameIteratorImpl& frame_iter) {
         RELEASE_ASSERT(is_active, "");
         if (exc_info.reraise) {
             exc_info.reraise = false;
             return;
         }
+        auto line_info = lineInfoForFrame(&frame_iter);
         BoxedTraceback::here(line_info, &exc_info.traceback);
     }
 
@@ -593,16 +604,6 @@ void throwingException(PythonUnwindSession* unwind) {
     unwind->logException();
 }
 
-static const LineInfo lineInfoForFrame(PythonFrameIteratorImpl* frame_it) {
-    AST_stmt* current_stmt = frame_it->getCurrentStatement();
-    auto* cl = frame_it->getCL();
-    assert(cl);
-
-    auto source = cl->source.get();
-
-    return LineInfo(current_stmt->lineno, current_stmt->col_offset, source->fn, source->getName());
-}
-
 extern "C" void capiExcCaughtInJit(AST_stmt* stmt, void* _source_info) {
     SourceInfo* source = static_cast<SourceInfo*>(_source_info);
     // TODO: handle reraise (currently on the ExcInfo object)
@@ -611,6 +612,16 @@ extern "C" void capiExcCaughtInJit(AST_stmt* stmt, void* _source_info) {
                          &tstate->curexc_traceback);
 }
 
+extern "C" void reraiseJitCapiExc() {
+    ensureCAPIExceptionSet();
+    // TODO: we are normalizing to many times?
+    ExcInfo e = excInfoForRaise(cur_thread_state.curexc_type, cur_thread_state.curexc_value,
+                                cur_thread_state.curexc_traceback);
+    PyErr_Clear();
+    e.reraise = true;
+    throw e;
+}
+
 void exceptionCaughtInInterpreter(LineInfo line_info, ExcInfo* exc_info) {
     static StatCounter frames_unwound("num_frames_unwound_python");
     frames_unwound.log();
@@ -647,7 +658,7 @@ void unwindingThroughFrame(PythonUnwindSession* unwind_session, unw_cursor_t* cu
         frames_unwound.log();
 
         if (!unwind_session->shouldSkipFrame())
-            unwind_session->addTraceback(lineInfoForFrame(&frame_iter));
+            unwind_session->addTraceback(frame_iter);
 
         // frame_iter->cf->entry_descriptor will be non-null for OSR frames.
         bool was_osr = (frame_iter.getId().type == PythonFrameId::COMPILED) && (frame_iter.cf->entry_descriptor);

src/core/options.cpp

@@ -44,6 +44,9 @@ bool ENABLE_PYPA_PARSER = true;
 bool USE_REGALLOC_BASIC = true;
 bool PAUSE_AT_ABORT = false;
 bool ENABLE_TRACEBACKS = true;
+// Forces the llvm jit to use capi exceptions whenever it can, as opposed to whenever it thinks
+// it is faster:
+bool FORCE_LLVM_CAPI = false;
 
 int OSR_THRESHOLD_INTERPRETER = 25;
 int REOPT_THRESHOLD_INTERPRETER = 25;

src/core/options.h

@@ -39,7 +39,7 @@ extern int MAX_OBJECT_CACHE_ENTRIES;
 
 extern bool SHOW_DISASM, FORCE_INTERPRETER, FORCE_OPTIMIZE, PROFILE, DUMPJIT, TRAP, USE_STRIPPED_STDLIB,
     CONTINUE_AFTER_FATAL, ENABLE_INTERPRETER, ENABLE_BASELINEJIT, ENABLE_PYPA_PARSER, USE_REGALLOC_BASIC,
-    PAUSE_AT_ABORT, ENABLE_TRACEBACKS, ASSEMBLY_LOGGING;
+    PAUSE_AT_ABORT, ENABLE_TRACEBACKS, ASSEMBLY_LOGGING, FORCE_LLVM_CAPI;
 
 extern bool ENABLE_ICS, ENABLE_ICGENERICS, ENABLE_ICGETITEMS, ENABLE_ICSETITEMS, ENABLE_ICDELITEMS, ENABLE_ICBINEXPS,
     ENABLE_ICNONZEROS, ENABLE_ICCALLSITES, ENABLE_ICSETATTRS, ENABLE_ICGETATTRS, ENALBE_ICDELATTRS, ENABLE_ICGETGLOBALS,

src/core/types.h

@@ -242,6 +242,7 @@ struct ParamReceiveSpec {
     bool operator!=(ParamReceiveSpec rhs) { return !(*this == rhs); }
 
     int totalReceived() { return num_args + (takes_varargs ? 1 : 0) + (takes_kwargs ? 1 : 0); }
+    int kwargsIndex() { return num_args + (takes_varargs ? 1 : 0); }
 };
 
 class ICInvalidator {

src/runtime/dict.cpp

@@ -227,7 +227,7 @@ template <enum ExceptionStyle S> Box* dictGetitem(BoxedDict* self, Box* k) noexc
         }
 
         if (S == CAPI) {
-            PyErr_SetObject(KeyError, k);
+            PyErr_SetObject(KeyError, BoxedTuple::create1(k));
             return NULL;
         } else
             raiseExcHelper(KeyError, k);
@@ -746,7 +746,9 @@ void setupDict() {
     dict_cls->giveAttr("setdefault",
                        new BoxedFunction(boxRTFunction((void*)dictSetdefault, UNKNOWN, 3, 1, false, false), { None }));
 
-    dict_cls->giveAttr("__getitem__", new BoxedFunction(boxRTFunction((void*)dictGetitem<CXX>, UNKNOWN, 2)));
+    auto dict_getitem = boxRTFunction((void*)dictGetitem<CXX>, UNKNOWN, 2, ParamNames::empty(), CXX);
+    addRTFunction(dict_getitem, (void*)dictGetitem<CAPI>, UNKNOWN, CAPI);
+    dict_cls->giveAttr("__getitem__", new BoxedFunction(dict_getitem));
     dict_cls->giveAttr("__setitem__", new BoxedFunction(boxRTFunction((void*)dictSetitem, NONE, 3)));
     dict_cls->giveAttr("__delitem__", new BoxedFunction(boxRTFunction((void*)dictDelitem, UNKNOWN, 2)));
     dict_cls->giveAttr("__contains__", new BoxedFunction(boxRTFunction((void*)dictContains, BOXED_BOOL, 2)));

src/runtime/inline/link_forcer.cpp

@@ -83,6 +83,7 @@ void force() {
     FORCE(augbinop);
     FORCE(unboxedLen);
     FORCE(getitem);
+    FORCE(getitem_capi);
     FORCE(getclsattr);
     FORCE(getGlobal);
     FORCE(delGlobal);
@@ -105,6 +106,7 @@ void force() {
     FORCE(raiseAttributeErrorCapi);
     FORCE(raiseAttributeErrorStrCapi);
     FORCE(raiseIndexErrorStr);
+    FORCE(raiseIndexErrorStrCapi);
     FORCE(raiseNotIterableError);
     FORCE(assertNameDefined);
     FORCE(assertFailDerefNameDefined);
@@ -120,9 +122,11 @@ void force() {
 
     FORCE(raise0);
     FORCE(raise3);
+    FORCE(raise3_capi);
     FORCE(PyErr_Fetch);
     FORCE(PyErr_NormalizeException);
     FORCE(capiExcCaughtInJit);
+    FORCE(reraiseJitCapiExc);
     FORCE(deopt);
 
     FORCE(div_i64_i64);

src/runtime/list.cpp

@@ -224,7 +224,16 @@ extern "C" PyObject* PyList_GetItem(PyObject* op, Py_ssize_t i) noexcept {
     }
 }
 
-extern "C" Box* listGetitemSlice(BoxedList* self, BoxedSlice* slice) {
+template <ExceptionStyle S> Box* listGetitemSlice(BoxedList* self, BoxedSlice* slice) {
+    if (S == CAPI) {
+        try {
+            return listGetitemSlice<CXX>(self, slice);
+        } catch (ExcInfo e) {
+            setCAPIException(e);
+            return NULL;
+        }
+    }
+
     assert(isSubclass(self->cls, list_cls));
     assert(slice->cls == slice_cls);
     i64 start, stop, step, length;
@@ -242,7 +251,16 @@ extern "C" Box* listGetslice(BoxedList* self, Box* boxedStart, Box* boxedStop) {
     return _listSlice(self, start, stop, 1, stop - start);
 }
 
-extern "C" Box* listGetitem(BoxedList* self, Box* slice) {
+template <ExceptionStyle S> Box* listGetitem(BoxedList* self, Box* slice) {
+    if (S == CAPI) {
+        try {
+            return listGetitem<CXX>(self, slice);
+        } catch (ExcInfo e) {
+            setCAPIException(e);
+            return NULL;
+        }
+    }
+
     assert(isSubclass(self->cls, list_cls));
     if (PyIndex_Check(slice)) {
         Py_ssize_t i = PyNumber_AsSsize_t(slice, PyExc_IndexError);
@@ -250,7 +268,7 @@ extern "C" Box* listGetitem(BoxedList* self, Box* slice) {
             throwCAPIException();
         return listGetitemUnboxed(self, i);
     } else if (slice->cls == slice_cls) {
-        return listGetitemSlice(self, static_cast<BoxedSlice*>(slice));
+        return listGetitemSlice<CXX>(self, static_cast<BoxedSlice*>(slice));
     } else {
         raiseExcHelper(TypeError, "list indices must be integers, not %s", getTypeName(slice));
     }
@@ -799,13 +817,8 @@ extern "C" int PyList_Sort(PyObject* v) noexcept {
 extern "C" Box* PyList_GetSlice(PyObject* a, Py_ssize_t ilow, Py_ssize_t ihigh) noexcept {
     assert(isSubclass(a->cls, list_cls));
     BoxedList* self = static_cast<BoxedList*>(a);
-    try {
-        // Lots of extra copies here; we can do better if we need to:
-        return listGetitemSlice(self, new BoxedSlice(boxInt(ilow), boxInt(ihigh), boxInt(1)));
-    } catch (ExcInfo e) {
-        setCAPIException(e);
-        return NULL;
-    }
+    // Lots of extra copies here; we can do better if we need to:
+    return listGetitemSlice<CAPI>(self, new BoxedSlice(boxInt(ilow), boxInt(ihigh), boxInt(1)));
 }
 
 Box* listContains(BoxedList* self, Box* elt) {
@@ -1092,8 +1105,13 @@ void setupList() {
 
     CLFunction* getitem = createRTFunction(2, 0, false, false);
     addRTFunction(getitem, (void*)listGetitemInt, UNKNOWN, std::vector<ConcreteCompilerType*>{ LIST, BOXED_INT });
-    addRTFunction(getitem, (void*)listGetitemSlice, LIST, std::vector<ConcreteCompilerType*>{ LIST, SLICE });
-    addRTFunction(getitem, (void*)listGetitem, UNKNOWN, std::vector<ConcreteCompilerType*>{ UNKNOWN, UNKNOWN });
+    addRTFunction(getitem, (void*)listGetitemSlice<CXX>, LIST, std::vector<ConcreteCompilerType*>{ LIST, SLICE }, CXX);
+    addRTFunction(getitem, (void*)listGetitem<CXX>, UNKNOWN, std::vector<ConcreteCompilerType*>{ UNKNOWN, UNKNOWN },
+                  CXX);
+    addRTFunction(getitem, (void*)listGetitemSlice<CAPI>, LIST, std::vector<ConcreteCompilerType*>{ LIST, SLICE },
+                  CAPI);
+    addRTFunction(getitem, (void*)listGetitem<CAPI>, UNKNOWN, std::vector<ConcreteCompilerType*>{ UNKNOWN, UNKNOWN },
+                  CAPI);
     list_cls->giveAttr("__getitem__", new BoxedFunction(getitem));
 
     list_cls->giveAttr("__getslice__", new BoxedFunction(boxRTFunction((void*)listGetslice, LIST, 3)));

src/runtime/objmodel.cpp

@@ -305,6 +305,10 @@ extern "C" void raiseIndexErrorStr(const char* typeName) {
     raiseExcHelper(IndexError, "%s index out of range", typeName);
 }
 
+extern "C" void raiseIndexErrorStrCapi(const char* typeName) noexcept {
+    PyErr_Format(IndexError, "%s index out of range", typeName);
+}
+
 extern "C" void raiseNotIterableError(const char* typeName) {
     raiseExcHelper(TypeError, "'%s' object is not iterable", typeName);
 }
@@ -1363,7 +1367,8 @@ Box* dataDescriptorInstanceSpecialCases(GetattrRewriteArgs* rewrite_args, BoxedS
 }
 
 // Helper function: make sure that a capi function either returned a non-error value, or
-// it set an exception.
+// it set an exception.  This is only needed in specialized situations; usually the "error
+// return without exception set" state can just be passed up to the caller.
 static void ensureValidCapiReturn(Box* r) {
     if (!r)
         ensureCAPIExceptionSet();
@@ -3011,8 +3016,27 @@ void rearrangeArguments(ParamReceiveSpec paramspec, const ParamNames* param_name
     // Fast path: if it's a simple-enough call, we don't have to do anything special.  On a simple
     // django-admin test this covers something like 93% of all calls to callFunc.
     if (argspec.num_keywords == 0 && argspec.has_starargs == paramspec.takes_varargs && !argspec.has_kwargs
-        && !paramspec.takes_kwargs && argspec.num_args == paramspec.num_args) {
-        assert(num_output_args == num_passed_args);
+        && argspec.num_args == paramspec.num_args && (!paramspec.takes_kwargs || paramspec.kwargsIndex() < 3)) {
+
+        // TODO could also do this for empty varargs
+        if (paramspec.takes_kwargs) {
+            assert(num_output_args == num_passed_args + 1);
+            int idx = paramspec.kwargsIndex();
+            assert(idx < 3);
+            getArg(idx, arg1, arg2, arg3, NULL) = NULL; // pass NULL for kwargs
+            if (rewrite_args) {
+                if (idx == 0)
+                    rewrite_args->arg1 = rewrite_args->rewriter->loadConst(0);
+                else if (idx == 1)
+                    rewrite_args->arg2 = rewrite_args->rewriter->loadConst(0);
+                else if (idx == 2)
+                    rewrite_args->arg3 = rewrite_args->rewriter->loadConst(0);
+                else
+                    abort();
+            }
+        } else {
+            assert(num_output_args == num_passed_args);
+        }
 
         // If the caller passed starargs, we can only pass those directly to the callee if it's a tuple,
         // since otherwise modifications by the callee would be visible to the caller (hence why varargs
@@ -4463,11 +4487,6 @@ Box* callItemOrSliceAttr(Box* target, BoxedString* item_str, BoxedString* slice_
 
 template <ExceptionStyle S>
 Box* getitemInternal(Box* target, Box* slice, GetitemRewriteArgs* rewrite_args) noexcept(S == CAPI) {
-    if (S == CAPI) {
-        assert(!rewrite_args && "implement me");
-        rewrite_args = NULL;
-    }
-
     // The PyObject_GetItem logic is:
     // - call mp_subscript if it exists
     // - if tp_as_sequence exists, try using that (with a number of conditions)
@@ -4479,7 +4498,6 @@ Box* getitemInternal(Box* target, Box* slice, GetitemRewriteArgs* rewrite_args)
     PyMappingMethods* m = target->cls->tp_as_mapping;
     if (m && m->mp_subscript && m->mp_subscript != slot_mp_subscript) {
         if (rewrite_args) {
-            assert(S == CXX);
             RewriterVar* r_obj = rewrite_args->target;
             RewriterVar* r_slice = rewrite_args->slice;
             RewriterVar* r_cls = r_obj->getAttr(offsetof(Box, cls));
@@ -4493,7 +4511,8 @@ Box* getitemInternal(Box* target, Box* slice, GetitemRewriteArgs* rewrite_args)
             // support calling a RewriterVar (can only call fixed function addresses).
             r_m->addAttrGuard(offsetof(PyMappingMethods, mp_subscript), (intptr_t)m->mp_subscript);
             RewriterVar* r_rtn = rewrite_args->rewriter->call(true, (void*)m->mp_subscript, r_obj, r_slice);
-            rewrite_args->rewriter->call(true, (void*)checkAndThrowCAPIException);
+            if (S == CXX)
+                rewrite_args->rewriter->call(true, (void*)checkAndThrowCAPIException);
             rewrite_args->out_success = true;
             rewrite_args->out_rtn = r_rtn;
         }
@@ -4503,6 +4522,11 @@ Box* getitemInternal(Box* target, Box* slice, GetitemRewriteArgs* rewrite_args)
         return r;
     }
 
+    if (S == CAPI) {
+        // assert(!rewrite_args && "implement me");
+        rewrite_args = NULL;
+    }
+
     static BoxedString* getitem_str = internStringImmortal("__getitem__");
     static BoxedString* getslice_str = internStringImmortal("__getslice__");
 
@@ -4596,6 +4620,39 @@ extern "C" Box* getitem(Box* target, Box* slice) {
     return rtn;
 }
 
+// target[slice]
+extern "C" Box* getitem_capi(Box* target, Box* slice) noexcept {
+    STAT_TIMER(t0, "us_timer_slowpath_getitem", 10);
+
+    // This possibly could just be represented as a single callattr; the only tricky part
+    // are the error messages.
+    // Ex "(1)[1]" and "(1).__getitem__(1)" give different error messages.
+
+    static StatCounter slowpath_getitem("slowpath_getitem");
+    slowpath_getitem.log();
+
+    std::unique_ptr<Rewriter> rewriter(
+        Rewriter::createRewriter(__builtin_extract_return_addr(__builtin_return_address(0)), 2, "getitem"));
+
+    Box* rtn;
+    if (rewriter.get()) {
+        GetitemRewriteArgs rewrite_args(rewriter.get(), rewriter->getArg(0), rewriter->getArg(1),
+                                        rewriter->getReturnDestination());
+
+        rtn = getitemInternal<CAPI>(target, slice, &rewrite_args);
+
+        if (!rewrite_args.out_success) {
+            rewriter.reset(NULL);
+        } else {
+            rewriter->commitReturning(rewrite_args.out_rtn);
+        }
+    } else {
+        rtn = getitemInternal<CAPI>(target, slice, NULL);
+    }
+
+    return rtn;
+}
+
 // target[slice] = value
 extern "C" void setitem(Box* target, Box* slice, Box* value) {
     STAT_TIMER(t0, "us_timer_slowpath_setitem", 10);

src/runtime/objmodel.h

@@ -35,6 +35,7 @@ class BoxedTuple;
 ExcInfo excInfoForRaise(Box*, Box*, Box*);
 extern "C" void raise0() __attribute__((__noreturn__));
 extern "C" void raise3(Box*, Box*, Box*) __attribute__((__noreturn__));
+extern "C" void raise3_capi(Box*, Box*, Box*) noexcept;
 void raiseExc(Box* exc_obj) __attribute__((__noreturn__));
 void _printStacktrace();
 
@@ -82,6 +83,7 @@ extern "C" i64 unboxedLen(Box* obj);
 extern "C" Box* binop(Box* lhs, Box* rhs, int op_type);
 extern "C" Box* augbinop(Box* lhs, Box* rhs, int op_type);
 extern "C" Box* getitem(Box* value, Box* slice);
+extern "C" Box* getitem_capi(Box* value, Box* slice) noexcept;
 extern "C" void setitem(Box* target, Box* slice, Box* value);
 extern "C" void delitem(Box* target, Box* slice);
 extern "C" Box* getclsattr(Box* obj, BoxedString* attr);
@@ -158,6 +160,7 @@ extern "C" void raiseAttributeErrorStrCapi(const char* typeName, llvm::StringRef
 extern "C" void raiseAttributeErrorCapi(Box* obj, llvm::StringRef attr) noexcept;
 extern "C" void raiseNotIterableError(const char* typeName) __attribute__((__noreturn__));
 extern "C" void raiseIndexErrorStr(const char* typeName) __attribute__((__noreturn__));
+extern "C" void raiseIndexErrorStrCapi(const char* typeName) noexcept;
 
 Box* typeCall(Box*, BoxedTuple*, BoxedDict*);
 Box* type_new(BoxedClass* metatype, Box* args, Box* kwds) noexcept;

src/runtime/stacktrace.cpp

@@ -272,6 +272,21 @@ extern "C" void raise3(Box* arg0, Box* arg1, Box* arg2) {
     throw exc_info;
 }
 
+extern "C" void raise3_capi(Box* arg0, Box* arg1, Box* arg2) noexcept {
+    bool reraise = arg2 != NULL && arg2 != None;
+
+    ExcInfo exc_info(NULL, NULL, NULL);
+    try {
+        exc_info = excInfoForRaise(arg0, arg1, arg2);
+        exc_info.reraise = reraise;
+    } catch (ExcInfo e) {
+        exc_info = e;
+    }
+
+    assert(!exc_info.reraise); // would get thrown away
+    PyErr_Restore(exc_info.type, exc_info.value, exc_info.traceback);
+}
+
 void raiseExcHelper(BoxedClass* cls, Box* arg) {
     Box* exc_obj = runtimeCall(cls, ArgPassSpec(1), arg, NULL, NULL, NULL, NULL);
     raiseExc(exc_obj);

src/runtime/str.cpp

@@ -2216,7 +2216,15 @@ Box* strEncode(BoxedString* self, Box* encoding, Box* error) {
     return result;
 }
 
-extern "C" Box* strGetitem(BoxedString* self, Box* slice) {
+template <ExceptionStyle S> Box* strGetitem(BoxedString* self, Box* slice) {
+    if (S == CAPI) {
+        try {
+            return strGetitem<CXX>(self, slice);
+        } catch (ExcInfo e) {
+            setCAPIException(e);
+            return NULL;
+        }
+    }
     assert(PyString_Check(self));
 
     if (PyIndex_Check(slice)) {
@@ -2760,7 +2768,9 @@ void setupStr() {
     str_cls->giveAttr("center",
                       new BoxedFunction(boxRTFunction((void*)strCenter, UNKNOWN, 3, 1, false, false), { spaceChar }));
 
-    str_cls->giveAttr("__getitem__", new BoxedFunction(boxRTFunction((void*)strGetitem, STR, 2)));
+    auto str_getitem = boxRTFunction((void*)strGetitem<CXX>, STR, 2, ParamNames::empty(), CXX);
+    addRTFunction(str_getitem, (void*)strGetitem<CAPI>, STR, CAPI);
+    str_cls->giveAttr("__getitem__", new BoxedFunction(str_getitem));
 
     str_cls->giveAttr("__getslice__", new BoxedFunction(boxRTFunction((void*)strGetslice, STR, 3)));
 

src/runtime/traceback.cpp

@@ -114,7 +114,7 @@ Box* BoxedTraceback::getLines(Box* b) {
 }
 
 void BoxedTraceback::here(LineInfo lineInfo, Box** tb) {
-    *tb = new BoxedTraceback(lineInfo, *tb);
+    *tb = new BoxedTraceback(std::move(lineInfo), *tb);
 }
 
 void setupTraceback() {

src/runtime/traceback.h

@@ -31,7 +31,7 @@ public:
     LineInfo line;
     Box* py_lines;
 
-    BoxedTraceback(LineInfo line, Box* tb_next) : tb_next(tb_next), line(line), py_lines(NULL) {}
+    BoxedTraceback(LineInfo line, Box* tb_next) : tb_next(tb_next), line(std::move(line)), py_lines(NULL) {}
 
     DEFAULT_CLASS(traceback_cls);
 

src/runtime/tuple.cpp

@@ -135,7 +135,16 @@ int BoxedTuple::Resize(BoxedTuple** pv, size_t newsize) noexcept {
     return 0;
 }
 
-Box* tupleGetitem(BoxedTuple* self, Box* slice) {
+template <ExceptionStyle S> Box* tupleGetitem(BoxedTuple* self, Box* slice) {
+    if (S == CAPI) {
+        try {
+            return tupleGetitem<CXX>(self, slice);
+        } catch (ExcInfo e) {
+            setCAPIException(e);
+            return NULL;
+        }
+    }
+
     assert(self->cls == tuple_cls);
 
     if (PyIndex_Check(slice)) {
@@ -555,7 +564,10 @@ void setupTuple() {
     CLFunction* getitem = createRTFunction(2, 0, 0, 0);
     addRTFunction(getitem, (void*)tupleGetitemInt, UNKNOWN, std::vector<ConcreteCompilerType*>{ UNKNOWN, BOXED_INT });
     addRTFunction(getitem, (void*)tupleGetitemSlice, UNKNOWN, std::vector<ConcreteCompilerType*>{ UNKNOWN, SLICE });
-    addRTFunction(getitem, (void*)tupleGetitem, UNKNOWN, std::vector<ConcreteCompilerType*>{ UNKNOWN, UNKNOWN });
+    addRTFunction(getitem, (void*)tupleGetitem<CXX>, UNKNOWN, std::vector<ConcreteCompilerType*>{ UNKNOWN, UNKNOWN },
+                  CXX);
+    addRTFunction(getitem, (void*)tupleGetitem<CAPI>, UNKNOWN, std::vector<ConcreteCompilerType*>{ UNKNOWN, UNKNOWN },
+                  CAPI);
     tuple_cls->giveAttr("__getitem__", new BoxedFunction(getitem));
 
     tuple_cls->giveAttr("__contains__", new BoxedFunction(boxRTFunction((void*)tupleContains, BOXED_BOOL, 2)));

test/tests/exception_subclasscheck2.py

+# skip-if: True
 # expected: fail
 # - CPython calls subclasscheck twice, while we call it once.
 #   Looks like this is because CPython calls PyErr_NormalizeException