Add a testing mode to force-use capi exceptions

And fix a bunch of resulting issues (mostly just some pretty benign
assertion errors).

src/codegen/compvars.cpp

@@ -338,7 +338,7 @@ public:
         ConcreteCompilerVariable* converted_slice = slice->makeConverted(emitter, slice->getBoxType());
 
         ExceptionStyle target_exception_style = CXX;
-        if (info.unw_info.capi_exc_dest)
+        if (FORCE_LLVM_CAPI || info.unw_info.capi_exc_dest)
             target_exception_style = CAPI;
 
         bool do_patchpoint = ENABLE_ICGETITEMS;
@@ -501,7 +501,7 @@ CompilerVariable* UnknownType::getattr(IREmitter& emitter, const OpInfo& info, C
     llvm::Value* rtn_val = NULL;
 
     ExceptionStyle target_exception_style = CXX;
-    if (info.unw_info.capi_exc_dest)
+    if (info.unw_info.capi_exc_dest || (!cls_only && FORCE_LLVM_CAPI))
         target_exception_style = CAPI;
 
     llvm::Value* llvm_func;
@@ -1505,7 +1505,7 @@ public:
         if (canStaticallyResolveGetattrs()) {
             Box* rtattr = typeLookup(cls, attr, nullptr);
             if (rtattr == NULL) {
-                ExceptionStyle exception_style = info.unw_info.capi_exc_dest ? CAPI : CXX;
+                ExceptionStyle exception_style = (FORCE_LLVM_CAPI || info.unw_info.capi_exc_dest) ? CAPI : CXX;
                 llvm::Value* raise_func = exception_style == CXX ? g.funcs.raiseAttributeErrorStr
                                                                  : g.funcs.raiseAttributeErrorStrCapi;
                 llvm::CallSite call = emitter.createCall3(
@@ -1557,14 +1557,10 @@ public:
                                                   BoxedString* attr, bool clsonly, ArgPassSpec argspec,
                                                   const std::vector<CompilerVariable*>& args,
                                                   const std::vector<BoxedString*>* keyword_names,
-                                                  bool* no_attribute = NULL) {
+                                                  bool* no_attribute = NULL, ExceptionStyle exception_style = CXX) {
         if (!canStaticallyResolveGetattrs())
             return NULL;
 
-        ExceptionStyle exception_style = CXX;
-        if (info.unw_info.capi_exc_dest)
-            exception_style = CAPI;
-
         Box* rtattr = cls->getattr(attr);
         if (rtattr == NULL) {
             if (no_attribute) {
@@ -1763,8 +1759,13 @@ public:
     CompilerVariable* getitem(IREmitter& emitter, const OpInfo& info, VAR* var, CompilerVariable* slice) override {
         static BoxedString* attr = internStringImmortal("__getitem__");
         bool no_attribute = false;
+
+        ExceptionStyle exception_style = CXX;
+        if (FORCE_LLVM_CAPI || info.unw_info.capi_exc_dest)
+            exception_style = CAPI;
+
         ConcreteCompilerVariable* called_constant = tryCallattrConstant(
-            emitter, info, var, attr, true, ArgPassSpec(1, 0, 0, 0), { slice }, NULL, &no_attribute);
+            emitter, info, var, attr, true, ArgPassSpec(1, 0, 0, 0), { slice }, NULL, &no_attribute, exception_style);
 
         if (no_attribute) {
             assert(called_constant->getType() == UNDEF);
@@ -2269,9 +2270,20 @@ public:
                     rtn->incvref();
                     return rtn;
                 } else {
-                    llvm::CallSite call = emitter.createCall(info.unw_info, g.funcs.raiseIndexErrorStr,
-                                                             embedConstantPtr("tuple", g.i8_ptr));
-                    call.setDoesNotReturn();
+                    ExceptionStyle target_exception_style = CXX;
+                    if (FORCE_LLVM_CAPI || info.unw_info.capi_exc_dest)
+                        target_exception_style = CAPI;
+
+                    if (target_exception_style == CAPI) {
+                        llvm::CallSite call = emitter.createCall(info.unw_info, g.funcs.raiseIndexErrorStrCapi,
+                                                                 embedConstantPtr("tuple", g.i8_ptr), CAPI);
+                        emitter.checkAndPropagateCapiException(info.unw_info, getNullPtr(g.llvm_value_type_ptr),
+                                                               getNullPtr(g.llvm_value_type_ptr));
+                    } else {
+                        llvm::CallSite call = emitter.createCall(info.unw_info, g.funcs.raiseIndexErrorStr,
+                                                                 embedConstantPtr("tuple", g.i8_ptr), CXX);
+                        call.setDoesNotReturn();
+                    }
                     return undefVariable();
                 }
             }

src/codegen/irgen/hooks.h

@@ -40,6 +40,8 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm);
 CompiledFunction* cfForMachineFunctionName(const std::string&);
 
 extern "C" void capiExcCaughtInJit(AST_stmt* current_stmt, void* source_info);
+// This is just meant for the use of the JIT (normal runtime code should call throwCAPIException)
+extern "C" void reraiseJitCapiExc() __attribute__((noreturn));
 
 extern "C" Box* exec(Box* boxedCode, Box* globals, Box* locals, FutureFlags caller_future_flags);
 extern "C" Box* eval(Box* boxedCode, Box* globals, Box* locals);

src/codegen/irgen/irgenerator.cpp

@@ -508,7 +508,8 @@ public:
                                   embedRelocatablePtr(irstate->getSourceInfo(), g.i8_ptr));
 
         if (!exc_caught) {
-            RELEASE_ASSERT(0, "need to implement this");
+            getBuilder()->CreateCall(g.funcs.reraiseJitCapiExc);
+            getBuilder()->CreateUnreachable();
         }
 
         setCurrentBasicBlock(normal_dest);
@@ -2318,7 +2319,7 @@ private:
         // but ommitting the first argument is *not* the same as passing None.
 
         ExceptionStyle target_exception_style = CXX;
-        if (unw_info.capi_exc_dest)
+        if (unw_info.capi_exc_dest || (FORCE_LLVM_CAPI && node->arg0 && !node->arg2))
             target_exception_style = CAPI;
 
         if (node->arg0 == NULL) {

src/codegen/runtime_hooks.cpp

@@ -228,6 +228,7 @@ void initGlobalFuncs(GlobalState& g) {
     GET(raiseAttributeErrorCapi);
     GET(raiseAttributeErrorStrCapi);
     GET(raiseIndexErrorStr);
+    GET(raiseIndexErrorStrCapi);
     GET(raiseNotIterableError);
     GET(assertNameDefined);
     GET(assertFailDerefNameDefined);
@@ -278,6 +279,7 @@ void initGlobalFuncs(GlobalState& g) {
     GET(PyErr_Fetch);
     GET(PyErr_NormalizeException);
     GET(capiExcCaughtInJit);
+    GET(reraiseJitCapiExc);
     GET(deopt);
 
     GET(div_float_float);

src/codegen/runtime_hooks.h

@@ -40,8 +40,8 @@ struct GlobalFuncs {
         *importFrom, *importStar, *repr, *str, *strOrUnicode, *exceptionMatches, *yield, *getiterHelper, *hasnext;
 
     llvm::Value* unpackIntoArray, *raiseAttributeError, *raiseAttributeErrorStr, *raiseAttributeErrorCapi,
-        *raiseAttributeErrorStrCapi, *raiseNotIterableError, *raiseIndexErrorStr, *assertNameDefined, *assertFail,
-        *assertFailDerefNameDefined;
+        *raiseAttributeErrorStrCapi, *raiseNotIterableError, *raiseIndexErrorStr, *raiseIndexErrorStrCapi,
+        *assertNameDefined, *assertFail, *assertFailDerefNameDefined;
     llvm::Value* printFloat, *listAppendInternal, *getSysStdout;
     llvm::Value* runtimeCall0, *runtimeCall1, *runtimeCall2, *runtimeCall3, *runtimeCall, *runtimeCallN;
     llvm::Value* callattr0, *callattr1, *callattr2, *callattr3, *callattr, *callattrN;
@@ -51,7 +51,7 @@ struct GlobalFuncs {
 
     llvm::Value* __cxa_end_catch;
     llvm::Value* raise0, *raise3, *raise3_capi;
-    llvm::Value* PyErr_Fetch, *PyErr_NormalizeException, *capiExcCaughtInJit;
+    llvm::Value* PyErr_Fetch, *PyErr_NormalizeException, *capiExcCaughtInJit, *reraiseJitCapiExc;
     llvm::Value* deopt;
 
     llvm::Value* div_float_float, *floordiv_float_float, *mod_float_float, *pow_float_float;

src/codegen/unwinding.cpp

@@ -481,6 +481,16 @@ bool frameIsPythonFrame(unw_word_t ip, unw_word_t bp, unw_cursor_t* cursor, Pyth
     return true;
 }
 
+static const LineInfo lineInfoForFrame(PythonFrameIteratorImpl* frame_it) {
+    AST_stmt* current_stmt = frame_it->getCurrentStatement();
+    auto* cl = frame_it->getCL();
+    assert(cl);
+
+    auto source = cl->source.get();
+
+    return LineInfo(current_stmt->lineno, current_stmt->col_offset, source->fn, source->getName());
+}
+
 class PythonUnwindSession : public Box {
     ExcInfo exc_info;
     bool skip;
@@ -518,12 +528,13 @@ public:
         stat.log(t.end());
     }
 
-    void addTraceback(const LineInfo& line_info) {
+    void addTraceback(PythonFrameIteratorImpl& frame_iter) {
         RELEASE_ASSERT(is_active, "");
         if (exc_info.reraise) {
             exc_info.reraise = false;
             return;
         }
+        auto line_info = lineInfoForFrame(&frame_iter);
         BoxedTraceback::here(line_info, &exc_info.traceback);
     }
 
@@ -593,16 +604,6 @@ void throwingException(PythonUnwindSession* unwind) {
     unwind->logException();
 }
 
-static const LineInfo lineInfoForFrame(PythonFrameIteratorImpl* frame_it) {
-    AST_stmt* current_stmt = frame_it->getCurrentStatement();
-    auto* cl = frame_it->getCL();
-    assert(cl);
-
-    auto source = cl->source.get();
-
-    return LineInfo(current_stmt->lineno, current_stmt->col_offset, source->fn, source->getName());
-}
-
 extern "C" void capiExcCaughtInJit(AST_stmt* stmt, void* _source_info) {
     SourceInfo* source = static_cast<SourceInfo*>(_source_info);
     // TODO: handle reraise (currently on the ExcInfo object)
@@ -611,6 +612,16 @@ extern "C" void capiExcCaughtInJit(AST_stmt* stmt, void* _source_info) {
                          &tstate->curexc_traceback);
 }
 
+extern "C" void reraiseJitCapiExc() {
+    ensureCAPIExceptionSet();
+    // TODO: we are normalizing to many times?
+    ExcInfo e = excInfoForRaise(cur_thread_state.curexc_type, cur_thread_state.curexc_value,
+                                cur_thread_state.curexc_traceback);
+    PyErr_Clear();
+    e.reraise = true;
+    throw e;
+}
+
 void exceptionCaughtInInterpreter(LineInfo line_info, ExcInfo* exc_info) {
     static StatCounter frames_unwound("num_frames_unwound_python");
     frames_unwound.log();
@@ -647,7 +658,7 @@ void unwindingThroughFrame(PythonUnwindSession* unwind_session, unw_cursor_t* cu
         frames_unwound.log();
 
         if (!unwind_session->shouldSkipFrame())
-            unwind_session->addTraceback(lineInfoForFrame(&frame_iter));
+            unwind_session->addTraceback(frame_iter);
 
         // frame_iter->cf->entry_descriptor will be non-null for OSR frames.
         bool was_osr = (frame_iter.getId().type == PythonFrameId::COMPILED) && (frame_iter.cf->entry_descriptor);

src/core/options.cpp

@@ -44,6 +44,9 @@ bool ENABLE_PYPA_PARSER = true;
 bool USE_REGALLOC_BASIC = true;
 bool PAUSE_AT_ABORT = false;
 bool ENABLE_TRACEBACKS = true;
+// Forces the llvm jit to use capi exceptions whenever it can, as opposed to whenever it thinks
+// it is faster:
+bool FORCE_LLVM_CAPI = false;
 
 int OSR_THRESHOLD_INTERPRETER = 25;
 int REOPT_THRESHOLD_INTERPRETER = 25;

src/core/options.h

@@ -39,7 +39,7 @@ extern int MAX_OBJECT_CACHE_ENTRIES;
 
 extern bool SHOW_DISASM, FORCE_INTERPRETER, FORCE_OPTIMIZE, PROFILE, DUMPJIT, TRAP, USE_STRIPPED_STDLIB,
     CONTINUE_AFTER_FATAL, ENABLE_INTERPRETER, ENABLE_BASELINEJIT, ENABLE_PYPA_PARSER, USE_REGALLOC_BASIC,
-    PAUSE_AT_ABORT, ENABLE_TRACEBACKS, ASSEMBLY_LOGGING;
+    PAUSE_AT_ABORT, ENABLE_TRACEBACKS, ASSEMBLY_LOGGING, FORCE_LLVM_CAPI;
 
 extern bool ENABLE_ICS, ENABLE_ICGENERICS, ENABLE_ICGETITEMS, ENABLE_ICSETITEMS, ENABLE_ICDELITEMS, ENABLE_ICBINEXPS,
     ENABLE_ICNONZEROS, ENABLE_ICCALLSITES, ENABLE_ICSETATTRS, ENABLE_ICGETATTRS, ENALBE_ICDELATTRS, ENABLE_ICGETGLOBALS,

src/runtime/inline/link_forcer.cpp

@@ -106,6 +106,7 @@ void force() {
     FORCE(raiseAttributeErrorCapi);
     FORCE(raiseAttributeErrorStrCapi);
     FORCE(raiseIndexErrorStr);
+    FORCE(raiseIndexErrorStrCapi);
     FORCE(raiseNotIterableError);
     FORCE(assertNameDefined);
     FORCE(assertFailDerefNameDefined);
@@ -125,6 +126,7 @@ void force() {
     FORCE(PyErr_Fetch);
     FORCE(PyErr_NormalizeException);
     FORCE(capiExcCaughtInJit);
+    FORCE(reraiseJitCapiExc);
     FORCE(deopt);
 
     FORCE(div_i64_i64);

src/runtime/list.cpp

@@ -224,7 +224,16 @@ extern "C" PyObject* PyList_GetItem(PyObject* op, Py_ssize_t i) noexcept {
     }
 }
 
-extern "C" Box* listGetitemSlice(BoxedList* self, BoxedSlice* slice) {
+template <ExceptionStyle S> Box* listGetitemSlice(BoxedList* self, BoxedSlice* slice) {
+    if (S == CAPI) {
+        try {
+            return listGetitemSlice<CXX>(self, slice);
+        } catch (ExcInfo e) {
+            setCAPIException(e);
+            return NULL;
+        }
+    }
+
     assert(isSubclass(self->cls, list_cls));
     assert(slice->cls == slice_cls);
     i64 start, stop, step, length;
@@ -242,7 +251,16 @@ extern "C" Box* listGetslice(BoxedList* self, Box* boxedStart, Box* boxedStop) {
     return _listSlice(self, start, stop, 1, stop - start);
 }
 
-extern "C" Box* listGetitem(BoxedList* self, Box* slice) {
+template <ExceptionStyle S> Box* listGetitem(BoxedList* self, Box* slice) {
+    if (S == CAPI) {
+        try {
+            return listGetitem<CXX>(self, slice);
+        } catch (ExcInfo e) {
+            setCAPIException(e);
+            return NULL;
+        }
+    }
+
     assert(isSubclass(self->cls, list_cls));
     if (PyIndex_Check(slice)) {
         Py_ssize_t i = PyNumber_AsSsize_t(slice, PyExc_IndexError);
@@ -250,7 +268,7 @@ extern "C" Box* listGetitem(BoxedList* self, Box* slice) {
             throwCAPIException();
         return listGetitemUnboxed(self, i);
     } else if (slice->cls == slice_cls) {
-        return listGetitemSlice(self, static_cast<BoxedSlice*>(slice));
+        return listGetitemSlice<CXX>(self, static_cast<BoxedSlice*>(slice));
     } else {
         raiseExcHelper(TypeError, "list indices must be integers, not %s", getTypeName(slice));
     }
@@ -799,13 +817,8 @@ extern "C" int PyList_Sort(PyObject* v) noexcept {
 extern "C" Box* PyList_GetSlice(PyObject* a, Py_ssize_t ilow, Py_ssize_t ihigh) noexcept {
     assert(isSubclass(a->cls, list_cls));
     BoxedList* self = static_cast<BoxedList*>(a);
-    try {
-        // Lots of extra copies here; we can do better if we need to:
-        return listGetitemSlice(self, new BoxedSlice(boxInt(ilow), boxInt(ihigh), boxInt(1)));
-    } catch (ExcInfo e) {
-        setCAPIException(e);
-        return NULL;
-    }
+    // Lots of extra copies here; we can do better if we need to:
+    return listGetitemSlice<CAPI>(self, new BoxedSlice(boxInt(ilow), boxInt(ihigh), boxInt(1)));
 }
 
 Box* listContains(BoxedList* self, Box* elt) {
@@ -1092,8 +1105,13 @@ void setupList() {
 
     CLFunction* getitem = createRTFunction(2, 0, false, false);
     addRTFunction(getitem, (void*)listGetitemInt, UNKNOWN, std::vector<ConcreteCompilerType*>{ LIST, BOXED_INT });
-    addRTFunction(getitem, (void*)listGetitemSlice, LIST, std::vector<ConcreteCompilerType*>{ LIST, SLICE });
-    addRTFunction(getitem, (void*)listGetitem, UNKNOWN, std::vector<ConcreteCompilerType*>{ UNKNOWN, UNKNOWN });
+    addRTFunction(getitem, (void*)listGetitemSlice<CXX>, LIST, std::vector<ConcreteCompilerType*>{ LIST, SLICE }, CXX);
+    addRTFunction(getitem, (void*)listGetitem<CXX>, UNKNOWN, std::vector<ConcreteCompilerType*>{ UNKNOWN, UNKNOWN },
+                  CXX);
+    addRTFunction(getitem, (void*)listGetitemSlice<CAPI>, LIST, std::vector<ConcreteCompilerType*>{ LIST, SLICE },
+                  CAPI);
+    addRTFunction(getitem, (void*)listGetitem<CAPI>, UNKNOWN, std::vector<ConcreteCompilerType*>{ UNKNOWN, UNKNOWN },
+                  CAPI);
     list_cls->giveAttr("__getitem__", new BoxedFunction(getitem));
 
     list_cls->giveAttr("__getslice__", new BoxedFunction(boxRTFunction((void*)listGetslice, LIST, 3)));

src/runtime/objmodel.cpp

@@ -305,6 +305,10 @@ extern "C" void raiseIndexErrorStr(const char* typeName) {
     raiseExcHelper(IndexError, "%s index out of range", typeName);
 }
 
+extern "C" void raiseIndexErrorStrCapi(const char* typeName) noexcept {
+    PyErr_Format(IndexError, "%s index out of range", typeName);
+}
+
 extern "C" void raiseNotIterableError(const char* typeName) {
     raiseExcHelper(TypeError, "'%s' object is not iterable", typeName);
 }
@@ -4517,7 +4521,7 @@ Box* getitemInternal(Box* target, Box* slice, GetitemRewriteArgs* rewrite_args)
     }
 
     if (S == CAPI) {
-        assert(!rewrite_args && "implement me");
+        // assert(!rewrite_args && "implement me");
         rewrite_args = NULL;
     }
 

src/runtime/objmodel.h

@@ -160,6 +160,7 @@ extern "C" void raiseAttributeErrorStrCapi(const char* typeName, llvm::StringRef
 extern "C" void raiseAttributeErrorCapi(Box* obj, llvm::StringRef attr) noexcept;
 extern "C" void raiseNotIterableError(const char* typeName) __attribute__((__noreturn__));
 extern "C" void raiseIndexErrorStr(const char* typeName) __attribute__((__noreturn__));
+extern "C" void raiseIndexErrorStrCapi(const char* typeName) noexcept;
 
 Box* typeCall(Box*, BoxedTuple*, BoxedDict*);
 Box* type_new(BoxedClass* metatype, Box* args, Box* kwds) noexcept;

src/runtime/str.cpp

@@ -2216,7 +2216,15 @@ Box* strEncode(BoxedString* self, Box* encoding, Box* error) {
     return result;
 }
 
-extern "C" Box* strGetitem(BoxedString* self, Box* slice) {
+template <ExceptionStyle S> Box* strGetitem(BoxedString* self, Box* slice) {
+    if (S == CAPI) {
+        try {
+            return strGetitem<CXX>(self, slice);
+        } catch (ExcInfo e) {
+            setCAPIException(e);
+            return NULL;
+        }
+    }
     assert(PyString_Check(self));
 
     if (PyIndex_Check(slice)) {
@@ -2760,7 +2768,9 @@ void setupStr() {
     str_cls->giveAttr("center",
                       new BoxedFunction(boxRTFunction((void*)strCenter, UNKNOWN, 3, 1, false, false), { spaceChar }));
 
-    str_cls->giveAttr("__getitem__", new BoxedFunction(boxRTFunction((void*)strGetitem, STR, 2)));
+    auto str_getitem = boxRTFunction((void*)strGetitem<CXX>, STR, 2, ParamNames::empty(), CXX);
+    addRTFunction(str_getitem, (void*)strGetitem<CAPI>, STR, CAPI);
+    str_cls->giveAttr("__getitem__", new BoxedFunction(str_getitem));
 
     str_cls->giveAttr("__getslice__", new BoxedFunction(boxRTFunction((void*)strGetslice, STR, 3)));
 

src/runtime/tuple.cpp

@@ -135,7 +135,16 @@ int BoxedTuple::Resize(BoxedTuple** pv, size_t newsize) noexcept {
     return 0;
 }
 
-Box* tupleGetitem(BoxedTuple* self, Box* slice) {
+template <ExceptionStyle S> Box* tupleGetitem(BoxedTuple* self, Box* slice) {
+    if (S == CAPI) {
+        try {
+            return tupleGetitem<CXX>(self, slice);
+        } catch (ExcInfo e) {
+            setCAPIException(e);
+            return NULL;
+        }
+    }
+
     assert(self->cls == tuple_cls);
 
     if (PyIndex_Check(slice)) {
@@ -555,7 +564,10 @@ void setupTuple() {
     CLFunction* getitem = createRTFunction(2, 0, 0, 0);
     addRTFunction(getitem, (void*)tupleGetitemInt, UNKNOWN, std::vector<ConcreteCompilerType*>{ UNKNOWN, BOXED_INT });
     addRTFunction(getitem, (void*)tupleGetitemSlice, UNKNOWN, std::vector<ConcreteCompilerType*>{ UNKNOWN, SLICE });
-    addRTFunction(getitem, (void*)tupleGetitem, UNKNOWN, std::vector<ConcreteCompilerType*>{ UNKNOWN, UNKNOWN });
+    addRTFunction(getitem, (void*)tupleGetitem<CXX>, UNKNOWN, std::vector<ConcreteCompilerType*>{ UNKNOWN, UNKNOWN },
+                  CXX);
+    addRTFunction(getitem, (void*)tupleGetitem<CAPI>, UNKNOWN, std::vector<ConcreteCompilerType*>{ UNKNOWN, UNKNOWN },
+                  CAPI);
     tuple_cls->giveAttr("__getitem__", new BoxedFunction(getitem));
 
     tuple_cls->giveAttr("__contains__", new BoxedFunction(boxRTFunction((void*)tupleContains, BOXED_BOOL, 2)));