add frame intospection for exited frames

this works by copying the last locals state into the BoxedFrame when we exit a function.
As a perf improvement we only create BoxedFrames if _getframe got called or an exception got thrown.

It turned out that our python frame retrieving / updating code was really slow (using libunwind) I replaced it now with
a approach which manually keeps track of the python frames (calling initFrame/deinitFrame every time a functions gets called / exits).
Sadly we can't inline this functions in the llvm tier currently because it uses thread local storage which the JIT does not support.
But I have a prototype which works around the issue with inline asm.
We also store now always the current stmt inside the FrameInfo and in addition the FunctionMetadata.
This made it possible to switch away from libunwind frame introspection for most cases (except deopt and c++ unwinding where we need to access all register)

I think we can further improve the patch in several ways:
 - only copying the vregs array instead of generating the dict which takes more time (because of space overhead and hashing?).
 - refcounting should remove uneccesarry locals updating when no one keeps track of the BoxedFrame anymore.
 - switch to cpythons traceback implementation

from_cpython/Include/pystate.h

@@ -111,6 +111,8 @@ typedef struct _ts {
 } PyThreadState;
 #endif
 typedef struct _ts {
+    void* frame_info; // This points to top python FrameInfo object
+
     int recursion_depth;
     int gilstate_counter;
 

src/codegen/ast_interpreter.cpp

@@ -138,7 +138,6 @@ private:
     CFGBlock* next_block, *current_block;
     FrameInfo frame_info;
 
-    FunctionMetadata* md;
     SourceInfo* source_info;
     ScopeInfo* scope_info;
     PhiAnalysis* phis;
@@ -170,7 +169,7 @@ public:
         return frame_info.globals;
     }
 
-    FunctionMetadata* getMD() { return md; }
+    FunctionMetadata* getMD() { return frame_info.md; }
     FrameInfo* getFrameInfo() { return &frame_info; }
     BoxedClosure* getPassedClosure() { return frame_info.passed_closure; }
     Box** getVRegs() { return vregs; }
@@ -217,7 +216,9 @@ void ASTInterpreter::setBoxedLocals(Box* boxedLocals) {
 }
 
 void ASTInterpreter::setFrameInfo(const FrameInfo* frame_info) {
+    Box** vregs = this->frame_info.vregs;
     this->frame_info = *frame_info;
+    this->frame_info.vregs = vregs;
 }
 
 void ASTInterpreter::setGlobals(Box* globals) {
@@ -228,7 +229,6 @@ void ASTInterpreter::setGlobals(Box* globals) {
 ASTInterpreter::ASTInterpreter(FunctionMetadata* md, Box** vregs)
     : current_block(0),
       frame_info(ExcInfo(NULL, NULL, NULL)),
-      md(md),
       source_info(md->source.get()),
       scope_info(0),
       phis(NULL),
@@ -242,6 +242,7 @@ ASTInterpreter::ASTInterpreter(FunctionMetadata* md, Box** vregs)
 
     scope_info = source_info->getScopeInfo();
     frame_info.vregs = vregs;
+    frame_info.md = md;
 
     assert(scope_info);
 }
@@ -254,7 +255,7 @@ void ASTInterpreter::initArguments(BoxedClosure* _closure, BoxedGenerator* _gene
     if (scope_info->createsClosure())
         created_closure = createClosure(_closure, scope_info->getClosureSize());
 
-    const ParamNames& param_names = md->param_names;
+    const ParamNames& param_names = getMD()->param_names;
 
     // make sure the AST_Name nodes are set
     assert(param_names.args.size() == param_names.arg_names.size());
@@ -282,7 +283,7 @@ void ASTInterpreter::startJITing(CFGBlock* block, int exit_offset) {
     assert(ENABLE_BASELINEJIT);
     assert(!jit);
 
-    auto& code_blocks = md->code_blocks;
+    auto& code_blocks = getMD()->code_blocks;
     JitCodeBlock* code_block = NULL;
     if (!code_blocks.empty())
         code_block = code_blocks[code_blocks.size() - 1].get();
@@ -326,6 +327,8 @@ Box* ASTInterpreter::execJITedBlock(CFGBlock* b) {
         if (stmt->type != AST_TYPE::Invoke)
             throw e;
 
+        assert(getPythonFrameInfo(0) == getFrameInfo());
+
         auto source = getMD()->source.get();
         stmt->cxx_exception_count++;
         caughtCxxException(LineInfo(stmt->lineno, stmt->col_offset, source->getFn(), source->getName()), &e);
@@ -369,7 +372,7 @@ Box* ASTInterpreter::executeInner(ASTInterpreter& interpreter, CFGBlock* start_b
         interpreter.next_block = start_block;
     }
 
-    if (ENABLE_BASELINEJIT && interpreter.md->times_interpreted >= REOPT_THRESHOLD_INTERPRETER)
+    if (ENABLE_BASELINEJIT && interpreter.getMD()->times_interpreted >= REOPT_THRESHOLD_INTERPRETER)
         interpreter.should_jit = true;
 
     while (interpreter.next_block) {
@@ -612,7 +615,7 @@ Value ASTInterpreter::visit_jump(AST_Jump* node) {
 
         // we may have started JITing because the OSR thresholds got triggered in this case we don't want to jit
         // additional blocks ouside of the loop if the function is cold.
-        if (md->times_interpreted < REOPT_THRESHOLD_INTERPRETER)
+        if (getMD()->times_interpreted < REOPT_THRESHOLD_INTERPRETER)
             should_jit = false;
     }
 
@@ -643,7 +646,8 @@ Box* ASTInterpreter::doOSR(AST_Jump* node) {
     ast_osrs.log();
 
     LivenessAnalysis* liveness = source_info->getLiveness();
-    std::unique_ptr<PhiAnalysis> phis = computeRequiredPhis(md->param_names, source_info->cfg, liveness, scope_info);
+    std::unique_ptr<PhiAnalysis> phis
+        = computeRequiredPhis(getMD()->param_names, source_info->cfg, liveness, scope_info);
 
     llvm::DenseMap<int, InternedString> offset_name_map;
     for (auto&& v : getSymVRegMap()) {
@@ -665,7 +669,7 @@ Box* ASTInterpreter::doOSR(AST_Jump* node) {
     }
 
     const OSREntryDescriptor* found_entry = nullptr;
-    for (auto& p : md->osr_versions) {
+    for (auto& p : getMD()->osr_versions) {
         if (p.first->backedge != node)
             continue;
 
@@ -718,7 +722,7 @@ Box* ASTInterpreter::doOSR(AST_Jump* node) {
     sorted_symbol_table[source_info->getInternedStrings().get(FRAME_INFO_PTR_NAME)] = (Box*)&frame_info;
 
     if (found_entry == nullptr) {
-        OSREntryDescriptor* entry = OSREntryDescriptor::create(md, node, CXX);
+        OSREntryDescriptor* entry = OSREntryDescriptor::create(getMD(), node, CXX);
 
         for (auto& it : sorted_symbol_table) {
             if (isIsDefinedName(it.first))
@@ -775,6 +779,8 @@ Value ASTInterpreter::visit_invoke(AST_Invoke* node) {
     } catch (ExcInfo e) {
         abortJITing();
 
+        assert(getPythonFrameInfo(0) == getFrameInfo());
+
         auto source = getMD()->source.get();
         node->cxx_exception_count++;
         caughtCxxException(LineInfo(node->lineno, node->col_offset, source->getFn(), source->getName()), &e);
@@ -1677,7 +1683,10 @@ const void* interpreter_instr_addr = (void*)&executeInnerAndSetupFrame;
 
 // small wrapper around executeInner because we can not directly call the member function from asm.
 extern "C" Box* executeInnerFromASM(ASTInterpreter& interpreter, CFGBlock* start_block, AST_stmt* start_at) {
-    return ASTInterpreter::executeInner(interpreter, start_block, start_at);
+    initFrame(interpreter.getFrameInfo());
+    Box* rtn = ASTInterpreter::executeInner(interpreter, start_block, start_at);
+    deinitFrame(interpreter.getFrameInfo());
+    return rtn;
 }
 
 Box* astInterpretFunction(FunctionMetadata* md, Box* closure, Box* generator, Box* globals, Box* arg1, Box* arg2,
@@ -1828,6 +1837,9 @@ static Box* astInterpretDeoptInner(FunctionMetadata* md, AST_expr* after_expr, A
 
     SourceInfo* source_info = md->source.get();
 
+    // We can't reuse the existing vregs from the LLVM tier because they only contain the user visible ones this means
+    // there wouldn't be enough space for the compiler generated ones which the interpreter (+bjit) stores inside the
+    // vreg array.
     Box** vregs = NULL;
     int num_vregs = md->calculateNumVRegs();
     if (num_vregs > 0) {
@@ -1905,6 +1917,11 @@ static Box* astInterpretDeoptInner(FunctionMetadata* md, AST_expr* after_expr, A
         assert(starting_statement);
     }
 
+    // We need to remove the old python frame created in the LLVM tier otherwise we would have a duplicate frame because
+    // the interpreter will set the new state before executing the first statement.
+    RELEASE_ASSERT(cur_thread_state.frame_info == frame_state.frame_info, "");
+    cur_thread_state.frame_info = frame_state.frame_info->back;
+
     Box* v = ASTInterpreter::execute(interpreter, start_block, starting_statement);
     return v ? v : None;
 }
@@ -1927,34 +1944,9 @@ static ASTInterpreter* getInterpreterFromFramePtr(void* frame_ptr) {
     return *ptr;
 }
 
-FunctionMetadata* getMDForInterpretedFrame(void* frame_ptr) {
-    ASTInterpreter* interpreter = getInterpreterFromFramePtr(frame_ptr);
-    assert(interpreter);
-    return interpreter->getMD();
-}
-
 FrameInfo* getFrameInfoForInterpretedFrame(void* frame_ptr) {
     ASTInterpreter* interpreter = getInterpreterFromFramePtr(frame_ptr);
     assert(interpreter);
     return interpreter->getFrameInfo();
 }
-
-BoxedDict* localsForInterpretedFrame(Box** vregs, CFG* cfg) {
-    BoxedDict* rtn = new BoxedDict();
-    for (auto& l : cfg->sym_vreg_map_user_visible) {
-        Box* val = vregs[l.second];
-        if (val) {
-            assert(gc::isValidGCObject(val));
-            rtn->d[l.first.getBox()] = val;
-        }
-    }
-
-    return rtn;
-}
-
-BoxedDict* localsForInterpretedFrame(void* frame_ptr) {
-    ASTInterpreter* interpreter = getInterpreterFromFramePtr(frame_ptr);
-    assert(interpreter);
-    return localsForInterpretedFrame(interpreter->getVRegs(), interpreter->getMD()->source->cfg);
-}
 }

src/codegen/ast_interpreter.h

@@ -76,13 +76,9 @@ Box* astInterpretFunctionEval(FunctionMetadata* cf, Box* globals, Box* boxedLoca
 Box* astInterpretDeopt(FunctionMetadata* cf, AST_expr* after_expr, AST_stmt* enclosing_stmt, Box* expr_val,
                        FrameStackState frame_state);
 
-FunctionMetadata* getMDForInterpretedFrame(void* frame_ptr);
 struct FrameInfo;
 FrameInfo* getFrameInfoForInterpretedFrame(void* frame_ptr);
 
-BoxedDict* localsForInterpretedFrame(Box** vregs, CFG* cfg);
-BoxedDict* localsForInterpretedFrame(void* frame_ptr);
-
 // Executes the equivalent of CPython's PRINT_EXPR opcode (call sys.displayhook)
 extern "C" void printExprHelper(Box* b);
 }

src/codegen/irgen/irgenerator.cpp

@@ -174,6 +174,11 @@ template <typename Builder> static llvm::Value* getGlobalsGep(Builder& builder,
     return builder.CreateConstInBoundsGEP2_32(v, 0, 6);
 }
 
+template <typename Builder> static llvm::Value* getMDGep(Builder& builder, llvm::Value* v) {
+    static_assert(offsetof(FrameInfo, md) == 64 + 16, "");
+    return builder.CreateConstInBoundsGEP2_32(v, 0, 8);
+}
+
 void IRGenState::setupFrameInfoVar(llvm::Value* passed_closure, llvm::Value* passed_globals,
                                    llvm::Value* frame_info_arg) {
     /*
@@ -273,9 +278,12 @@ void IRGenState::setupFrameInfoVar(llvm::Value* passed_closure, llvm::Value* pas
         builder.CreateStore(passed_globals, getGlobalsGep(builder, al));
         // set frame_info.vregs
         builder.CreateStore(vregs, getVRegsGep(builder, al));
+        builder.CreateStore(embedRelocatablePtr(getMD(), g.llvm_functionmetadata_type_ptr), getMDGep(builder, al));
 
         this->frame_info = al;
         this->globals = passed_globals;
+
+        builder.CreateCall(g.funcs.initFrame, this->frame_info);
     }
 
     stmt = getStmtGep(builder, frame_info);
@@ -2126,6 +2134,11 @@ private:
         rtn->ensureGrabbed(emitter);
         val->decvref(emitter);
 
+
+        // Don't call deinitFrame when this is a OSR function because the interpreter will call it
+        if (!irstate->getCurFunction()->entry_descriptor)
+            emitter.createCall(unw_info, g.funcs.deinitFrame, irstate->getFrameInfoVar());
+
         for (auto& p : symbol_table) {
             p.second->decvref(emitter);
         }
@@ -2889,8 +2902,8 @@ public:
             assert(!phi_node);
             phi_node = emitter.getBuilder()->CreatePHI(g.llvm_aststmt_type_ptr, 0);
 
-            emitter.getBuilder()->CreateCall2(g.funcs.caughtCapiException, phi_node,
-                                              embedRelocatablePtr(irstate->getSourceInfo(), g.i8_ptr));
+            emitter.createCall(UnwindInfo(current_stmt, NULL), g.funcs.caughtCapiException,
+                               { phi_node, embedRelocatablePtr(irstate->getSourceInfo(), g.i8_ptr) });
 
             if (!final_dest) {
                 // Propagate the exception out of the function:
@@ -2898,6 +2911,7 @@ public:
                     emitter.getBuilder()->CreateCall(g.funcs.reraiseCapiExcAsCxx);
                     emitter.getBuilder()->CreateUnreachable();
                 } else {
+                    emitter.createCall(UnwindInfo(current_stmt, NULL), g.funcs.deinitFrame, irstate->getFrameInfoVar());
                     emitter.getBuilder()->CreateRet(getNullPtr(g.llvm_value_type_ptr));
                 }
             } else {

src/codegen/runtime_hooks.cpp

@@ -198,6 +198,8 @@ void initGlobalFuncs(GlobalState& g) {
     GET(createClosure);
     GET(createGenerator);
     GET(createSet);
+    GET(initFrame);
+    GET(deinitFrame);
 
     GET(getattr);
     GET(getattr_capi);

src/codegen/runtime_hooks.h

@@ -34,7 +34,7 @@ struct GlobalFuncs {
 
     llvm::Value* boxInt, *unboxInt, *boxFloat, *unboxFloat, *createFunctionFromMetadata, *getFunctionMetadata,
         *boxInstanceMethod, *boxBool, *unboxBool, *createTuple, *createDict, *createList, *createSlice,
-        *createUserClass, *createClosure, *createGenerator, *createSet;
+        *createUserClass, *createClosure, *createGenerator, *createSet, *initFrame, *deinitFrame;
     llvm::Value* getattr, *getattr_capi, *setattr, *delattr, *delitem, *delGlobal, *nonzero, *binop, *compare,
         *augbinop, *unboxedLen, *getitem, *getitem_capi, *getclsattr, *getGlobal, *setitem, *unaryop, *import,
         *importFrom, *importStar, *repr, *exceptionMatches, *yield, *getiterHelper, *hasnext, *setGlobal, *apply_slice;

src/codegen/unwinding.h

@@ -80,28 +80,15 @@ public:
     FrameInfo* getFrameInfo();
     bool exists() { return impl.get() != NULL; }
     AST_stmt* getCurrentStatement();
-    Box* fastLocalsToBoxedLocals();
     Box* getGlobalsDict();
 
-    // Gets the "current version" of this frame: if the frame has executed since
-    // the iterator was obtained, the methods may return old values. This returns
-    // an updated copy that returns the updated values.
-    // The "current version" will live at the same stack location, but any other
-    // similarities need to be verified by the caller, ie it is up to the caller
-    // to determine that we didn't leave and reenter the stack frame.
-    // This function can only be called from the thread that created this object.
-    PythonFrameIterator getCurrentVersion();
-
-    // Assuming this is a valid frame iterator, return the next frame back (ie older).
-    PythonFrameIterator back();
-
     PythonFrameIterator(PythonFrameIterator&& rhs);
     void operator=(PythonFrameIterator&& rhs);
     PythonFrameIterator(std::unique_ptr<PythonFrameIteratorImpl> impl);
     ~PythonFrameIterator();
 };
 
-PythonFrameIterator getPythonFrame(int depth);
+FrameInfo* getPythonFrameInfo(int depth);
 
 // Fetches a writeable pointer to the frame-local excinfo object,
 // calculating it if necessary (from previous frames).

src/core/threading.cpp

@@ -38,7 +38,7 @@ std::unordered_set<PerThreadSetBase*> PerThreadSetBase::all_instances;
 
 extern "C" {
 __thread PyThreadState cur_thread_state
-    = { 0, 1, NULL, NULL, NULL, NULL }; // not sure if we need to explicitly request zero-initialization
+    = { NULL, 0, 1, NULL, NULL, NULL, NULL }; // not sure if we need to explicitly request zero-initialization
 }
 
 PthreadFastMutex threading_lock;

src/core/types.h

@@ -893,14 +893,16 @@ struct FrameInfo {
     BoxedClosure* passed_closure;
 
     Box** vregs;
-    // Current statement
-    // Caution the llvm tier only updates this information on direct external calls but not for patchpoints.
-    // This means if a patchpoint "current_stmt" info is available it must be used instead of this field.
-    AST_stmt* stmt;
+    AST_stmt* stmt; // current statement
     // This is either a module or a dict
     Box* globals;
 
-    FrameInfo(ExcInfo exc) : exc(exc), boxedLocals(NULL), frame_obj(0), passed_closure(0), vregs(0), stmt(0), globals(0) {}
+    FrameInfo* back;
+    FunctionMetadata* md;
+
+    Box* updateBoxedLocals();
+
+    FrameInfo(ExcInfo exc) : exc(exc), boxedLocals(NULL), frame_obj(0), passed_closure(0), vregs(0), stmt(0), globals(0), back(0), md(0) {}
 
     void gcVisit(GCVisitor* visitor);
 };

src/runtime/exceptions.cpp

@@ -51,7 +51,7 @@ void raiseSyntaxError(const char* msg, int lineno, int col_offset, llvm::StringR
     } else {
         // This is more like how the parser handles it:
         exc = runtimeCall(SyntaxError, ArgPassSpec(1), boxString(msg), NULL, NULL, NULL, NULL);
-        tb = new BoxedTraceback(LineInfo(lineno, col_offset, boxString(file), boxString(func)), None);
+        tb = new BoxedTraceback(LineInfo(lineno, col_offset, boxString(file), boxString(func)), None, getFrame(0));
     }
 
     assert(!PyErr_Occurred());
@@ -299,7 +299,7 @@ bool exceptionAtLineCheck() {
 
 void exceptionAtLine(LineInfo line_info, Box** traceback) {
     if (exceptionAtLineCheck())
-        BoxedTraceback::here(line_info, traceback);
+        BoxedTraceback::here(line_info, traceback, getFrame((FrameInfo*)cur_thread_state.frame_info));
 }
 
 void startReraise() {

src/runtime/frame.cpp

@@ -30,25 +30,24 @@ BoxedClass* frame_cls;
 class BoxedFrame : public Box {
 private:
     // Call boxFrame to get a BoxedFrame object.
-    BoxedFrame(PythonFrameIterator it) __attribute__((visibility("default")))
-    : it(std::move(it)), thread_id(PyThread_get_thread_ident()) {}
+    BoxedFrame(FrameInfo* frame_info) __attribute__((visibility("default")))
+    : frame_info(frame_info), _back(NULL), _code(NULL), _globals(NULL), _locals(NULL), _stmt(NULL) {
+        assert(frame_info);
+    }
 
 public:
-    PythonFrameIterator it;
-    long thread_id;
+    FrameInfo* frame_info;
 
-    Box* _globals;
+    Box* _back;
     Box* _code;
+    Box* _globals;
+    Box* _locals;
+
+    AST_stmt* _stmt;
+
+
+    bool hasExited() const { return frame_info == NULL; }
 
-    void update() {
-        // This makes sense as an exception, but who knows how the user program would react
-        // (it might swallow it and do something different)
-        RELEASE_ASSERT(thread_id == PyThread_get_thread_ident(),
-                       "frame objects can only be accessed from the same thread");
-        PythonFrameIterator new_it = it.getCurrentVersion();
-        RELEASE_ASSERT(new_it.exists() && new_it.getFrameInfo()->frame_obj == this, "frame has exited");
-        it = std::move(new_it);
-    }
 
     // cpython frame objects have the following attributes
 
@@ -82,84 +81,117 @@ public:
 
         auto f = static_cast<BoxedFrame*>(b);
 
+        v->visit(&f->_back);
         v->visit(&f->_code);
         v->visit(&f->_globals);
-    }
-
-    static void simpleDestructor(Box* b) {
-        auto f = static_cast<BoxedFrame*>(b);
-
-        f->it.~PythonFrameIterator();
+        v->visit(&f->_locals);
     }
 
     static Box* code(Box* obj, void*) {
         auto f = static_cast<BoxedFrame*>(obj);
+
+        if (!f->_code)
+            f->_code = (Box*)f->frame_info->md->getCode();
         return f->_code;
     }
 
     static Box* locals(Box* obj, void*) {
         auto f = static_cast<BoxedFrame*>(obj);
-        f->update();
-        return f->it.fastLocalsToBoxedLocals();
+
+        if (f->hasExited())
+            return f->_locals;
+        return f->frame_info->updateBoxedLocals();
     }
 
     static Box* globals(Box* obj, void*) {
         auto f = static_cast<BoxedFrame*>(obj);
+
+        if (!f->_globals) {
+            f->_globals = f->frame_info->globals;
+            if (f->_globals && PyModule_Check(f->_globals))
+                f->_globals = f->_globals->getAttrWrapper();
+        }
         return f->_globals;
     }
 
     static Box* back(Box* obj, void*) {
         auto f = static_cast<BoxedFrame*>(obj);
-        f->update();
 
-        PythonFrameIterator it = f->it.back();
-        if (!it.exists())
-            return None;
-        return BoxedFrame::boxFrame(std::move(it));
+        if (!f->_back) {
+            if (!f->frame_info->back)
+                f->_back = None;
+            else
+                f->_back = BoxedFrame::boxFrame(f->frame_info->back);
+        }
+        return f->_back;
     }
 
     static Box* lineno(Box* obj, void*) {
         auto f = static_cast<BoxedFrame*>(obj);
-        f->update();
-        AST_stmt* stmt = f->it.getCurrentStatement();
+
+        if (f->hasExited())
+            return boxInt(f->_stmt->lineno);
+
+        AST_stmt* stmt = f->frame_info->stmt;
         return boxInt(stmt->lineno);
     }
 
-    DEFAULT_CLASS(frame_cls);
+    void handleFrameExit() {
+        if (hasExited())
+            return;
 
-    static Box* boxFrame(PythonFrameIterator it) {
-        FrameInfo* fi = it.getFrameInfo();
-        if (fi->frame_obj == NULL) {
-            auto md = it.getMD();
-            Box* globals = it.getGlobalsDict();
-            BoxedFrame* f = fi->frame_obj = new BoxedFrame(std::move(it));
-            f->_globals = globals;
-            f->_code = (Box*)md->getCode();
-        }
+        _back = back(this, NULL);
+        _code = code(this, NULL);
+        _globals = globals(this, NULL);
+        _locals = locals(this, NULL);
+        _stmt = frame_info->stmt;
+
+        frame_info = NULL; // this means exited == true
+        assert(hasExited());
+    }
+
+    DEFAULT_CLASS(frame_cls);
 
+    static Box* boxFrame(FrameInfo* fi) {
+        if (fi->frame_obj == NULL)
+            fi->frame_obj = new BoxedFrame(fi);
+        assert(fi->frame_obj->cls == frame_cls);
         return fi->frame_obj;
     }
 };
 
+
+Box* getFrame(FrameInfo* frame_info) {
+    return BoxedFrame::boxFrame(frame_info);
+}
+
 Box* getFrame(int depth) {
-    auto it = getPythonFrame(depth);
-    if (!it.exists())
+    FrameInfo* frame_info = getPythonFrameInfo(depth);
+    if (!frame_info)
         return NULL;
+    return BoxedFrame::boxFrame(frame_info);
+}
+
+void frameInvalidateBack(BoxedFrame* frame) {
+    RELEASE_ASSERT(!frame->hasExited(), "should not happen");
+    frame->_back = NULL;
+}
+
+extern "C" void initFrame(FrameInfo* frame_info) {
+    frame_info->back = (FrameInfo*)(cur_thread_state.frame_info);
+    cur_thread_state.frame_info = frame_info;
+}
 
-    return BoxedFrame::boxFrame(std::move(it));
+extern "C" void deinitFrame(FrameInfo* frame_info) {
+    cur_thread_state.frame_info = frame_info->back;
+    BoxedFrame* frame = frame_info->frame_obj;
+    if (frame)
+        frame->handleFrameExit();
 }
 
 extern "C" int PyFrame_GetLineNumber(PyFrameObject* _f) noexcept {
-    // TODO remove this when we are able to introspect exited frames:
-    // We check if the frame exited and only return the correct line number when it is still available.
-    // Because of a limitation in out current frame introspection we can also not inspect OSRed frames.
-    BoxedFrame* f = (BoxedFrame*)_f;
-    PythonFrameIterator new_it = f->it.getCurrentVersion();
-    if (new_it.exists() && new_it.getFrameInfo()->frame_obj == f) {
-        BoxedInt* lineno = (BoxedInt*)BoxedFrame::lineno((Box*)f, NULL);
-        return lineno->n;
-    }
-    return -1;
+    BoxedInt* lineno = (BoxedInt*)BoxedFrame::lineno((Box*)_f, NULL);
+    return lineno->n;
 }
 
 extern "C" PyObject* PyFrame_GetGlobals(PyFrameObject* f) noexcept {
@@ -173,7 +205,6 @@ extern "C" PyFrameObject* PyFrame_ForStackLevel(int stack_level) noexcept {
 void setupFrame() {
     frame_cls = BoxedClass::create(type_cls, object_cls, &BoxedFrame::gchandler, 0, 0, sizeof(BoxedFrame), false,
                                    "frame", false);
-    frame_cls->tp_dealloc = BoxedFrame::simpleDestructor;
     frame_cls->has_safe_tp_dealloc = true;
 
     frame_cls->giveAttrDescriptor("f_code", BoxedFrame::code, NULL);

src/runtime/generator.cpp

@@ -93,6 +93,8 @@ void generatorEntry(BoxedGenerator* g) {
         try {
             RegisterHelper context_registerer(g, __builtin_frame_address(0));
 
+            g->top_caller_frame_info = (FrameInfo*)cur_thread_state.frame_info;
+
             // call body of the generator
             BoxedFunctionBase* func = g->function;
 
@@ -109,6 +111,7 @@ void generatorEntry(BoxedGenerator* g) {
         g->entryExited = true;
         threading::popGenerator();
     }
+    assert(g->top_caller_frame_info == cur_thread_state.frame_info);
     swapContext(&g->context, g->returnContext, 0);
 }
 
@@ -155,8 +158,11 @@ template <ExceptionStyle S> static bool generatorSendInternal(BoxedGenerator* se
     else
         self->prev_stack = StatTimer::swapStack(self->prev_stack);
 #endif
-
+    auto* top_caller_frame_info = (FrameInfo*)cur_thread_state.frame_info;
     swapContext(&self->returnContext, self->context, (intptr_t)self);
+    assert(cur_thread_state.frame_info == top_caller_frame_info
+           && "the generator should reset the frame info before the swapContext");
+
 
 #if STAT_TIMERS
     self->prev_stack = StatTimer::swapStack(self->prev_stack);
@@ -314,7 +320,28 @@ extern "C" Box* yield(BoxedGenerator* obj, Box* value) {
     self->returnValue = value;
 
     threading::popGenerator();
+
+    FrameInfo* generator_frame_info = (FrameInfo*)cur_thread_state.frame_info;
+    // a generator will only switch back (yield/unhandled exception) to its caller when it is one frame away from the
+    // caller
+    assert(self->top_caller_frame_info == generator_frame_info->back);
+
+    // reset current frame to the caller tops frame --> removes the frame the generator added
+    cur_thread_state.frame_info = self->top_caller_frame_info;
     swapContext(&self->context, self->returnContext, 0);
+    FrameInfo* top_new_caller_frame_info = (FrameInfo*)cur_thread_state.frame_info;
+
+    // the caller of the generator can change between yield statements that means we can't just restore the top of the
+    // frame to the point before the yield instead we have to update it.
+    if (top_new_caller_frame_info != self->top_caller_frame_info) {
+        // caller changed
+        self->top_caller_frame_info = top_new_caller_frame_info;
+        generator_frame_info->back = top_new_caller_frame_info;
+        if (generator_frame_info->frame_obj)
+            frameInvalidateBack(generator_frame_info->frame_obj);
+    }
+    cur_thread_state.frame_info = generator_frame_info;
+
     threading::pushGenerator(obj, obj->stack_begin, obj->returnContext);
 
     // if the generator receives a exception from the caller we have to throw it
@@ -347,7 +374,8 @@ extern "C" BoxedGenerator::BoxedGenerator(BoxedFunctionBase* function, Box* arg1
       returnValue(nullptr),
       exception(nullptr, nullptr, nullptr),
       context(nullptr),
-      returnContext(nullptr)
+      returnContext(nullptr),
+      top_caller_frame_info(nullptr)
 #if STAT_TIMERS
       ,
       prev_stack(NULL),

src/runtime/inline/link_forcer.cpp

@@ -72,6 +72,8 @@ void force() {
     FORCE(createPureImaginary);
     FORCE(createSet);
     FORCE(decodeUTF8StringPtr);
+    FORCE(initFrame);
+    FORCE(deinitFrame);
 
     FORCE(getattr);
     FORCE(getattr_capi);

src/runtime/traceback.cpp

@@ -39,11 +39,8 @@ void BoxedTraceback::gcHandler(GCVisitor* v, Box* b) {
     assert(b->cls == traceback_cls);
     BoxedTraceback* self = static_cast<BoxedTraceback*>(b);
 
-    if (self->py_lines)
-        v->visit(&self->py_lines);
-    if (self->tb_next)
-        v->visit(&self->tb_next);
-
+    v->visit(&self->tb_next);
+    v->visit(&self->tb_frame);
     v->visit(&self->line.file);
     v->visit(&self->line.func);
 
@@ -97,33 +94,8 @@ void printTraceback(Box* b) {
     }
 }
 
-Box* BoxedTraceback::getLines(Box* b) {
-    assert(b->cls == traceback_cls);
-
-    BoxedTraceback* tb = static_cast<BoxedTraceback*>(b);
-
-    if (!tb->py_lines) {
-        BoxedList* lines = new BoxedList();
-        for (BoxedTraceback* wtb = tb; wtb && wtb != None; wtb = static_cast<BoxedTraceback*>(wtb->tb_next)) {
-            auto& line = wtb->line;
-            auto l = BoxedTuple::create({ line.file, line.func, boxInt(line.line) });
-            listAppendInternal(lines, l);
-        }
-        tb->py_lines = lines;
-    }
-
-    return tb->py_lines;
-}
-
-void BoxedTraceback::here(LineInfo lineInfo, Box** tb) {
-    *tb = new BoxedTraceback(std::move(lineInfo), *tb);
-}
-
-static Box* traceback_tb_next(Box* self, void*) {
-    assert(self->cls == traceback_cls);
-
-    BoxedTraceback* traceback = static_cast<BoxedTraceback*>(self);
-    return traceback->tb_next;
+void BoxedTraceback::here(LineInfo lineInfo, Box** tb, Box* frame) {
+    *tb = new BoxedTraceback(std::move(lineInfo), *tb, frame);
 }
 
 extern "C" int _Py_DisplaySourceLine(PyObject* f, const char* filename, int lineno, int indent) noexcept {
@@ -134,17 +106,15 @@ void setupTraceback() {
     traceback_cls = BoxedClass::create(type_cls, object_cls, BoxedTraceback::gcHandler, 0, 0, sizeof(BoxedTraceback),
                                        false, "traceback");
 
-    traceback_cls->giveAttr("getLines",
-                            new BoxedFunction(FunctionMetadata::create((void*)BoxedTraceback::getLines, UNKNOWN, 1)));
-
     /*
      * Currently not supported.
-    traceback_cls->giveAttr("tb_frame", new (pyston_getset_cls) BoxedGetsetDescriptor(traceback_tb_frame, NULL, NULL));
     traceback_cls->giveAttr("tb_lasti", new (pyston_getset_cls) BoxedGetsetDescriptor(traceback_tb_lasti, NULL, NULL));
-    traceback_cls->giveAttr("tb_lineno", new (pyston_getset_cls) BoxedGetsetDescriptor(traceback_tb_lineno, NULL,
-    NULL));
     */
-    traceback_cls->giveAttrDescriptor("tb_next", traceback_tb_next, NULL);
+
+    traceback_cls->giveAttr(
+        "tb_frame", new BoxedMemberDescriptor(BoxedMemberDescriptor::OBJECT, offsetof(BoxedTraceback, tb_frame)));
+    traceback_cls->giveAttr(
+        "tb_next", new BoxedMemberDescriptor(BoxedMemberDescriptor::OBJECT, offsetof(BoxedTraceback, tb_next)));
     traceback_cls->giveAttr("tb_lineno",
                             new BoxedMemberDescriptor(BoxedMemberDescriptor::INT,
                                                       offsetof(BoxedTraceback, line) + offsetof(LineInfo, line)));

src/runtime/traceback.h

@@ -27,19 +27,26 @@ class GCVisitor;
 extern "C" BoxedClass* traceback_cls;
 class BoxedTraceback : public Box {
 public:
-    Box* tb_next;
     LineInfo line;
-    Box* py_lines;
-    BoxedTraceback(LineInfo line, Box* tb_next) : tb_next(tb_next), line(std::move(line)), py_lines(NULL) {}
+    Box* tb_next;
+    Box* tb_frame;
+
+    BoxedTraceback(LineInfo line, Box* tb_next, Box* tb_frame)
+        : line(std::move(line)), tb_next(tb_next), tb_frame(tb_frame) {
+        if (!tb_frame)
+            this->tb_frame = None;
+        else
+            assert(tb_frame->cls == frame_cls);
+    }
 
     DEFAULT_CLASS(traceback_cls);
 
-    static Box* getLines(Box* b);
+    static Box* lineno(Box* obj, void*);
 
     static void gcHandler(gc::GCVisitor* v, Box* b);
 
     // somewhat equivalent to PyTraceBack_Here
-    static void here(LineInfo lineInfo, Box** tb);
+    static void here(LineInfo lineInfo, Box** tb, Box* frame);
 };
 
 void printTraceback(Box* b);

src/runtime/types.cpp

@@ -105,6 +105,9 @@ void FrameInfo::gcVisit(GCVisitor* visitor) {
     visitor->visit(&exc.type);
     visitor->visit(&exc.value);
     visitor->visit(&frame_obj);
+
+    if (back)
+        ((FrameInfo*)back)->gcVisit(visitor);
 }
 
 // Analogue of PyType_GenericAlloc (default tp_alloc), but should only be used for Pyston classes!

src/runtime/types.h

@@ -1041,6 +1041,7 @@ public:
 
     struct Context* context, *returnContext;
     void* stack_begin;
+    FrameInfo* top_caller_frame_info;
 
 #if STAT_TIMERS
     StatTimer* prev_stack;
@@ -1146,7 +1147,11 @@ Box* codeForFunction(BoxedFunction*);
 Box* codeForFunctionMetadata(FunctionMetadata*);
 FunctionMetadata* metadataFromCode(Box* code);
 
+Box* getFrame(FrameInfo* frame_info);
 Box* getFrame(int depth);
+void frameInvalidateBack(BoxedFrame* frame);
+extern "C" void deinitFrame(FrameInfo* frame_info);
+extern "C" void initFrame(FrameInfo* frame_info);
 
 inline BoxedString* boxString(llvm::StringRef s) {
     if (s.size() <= 1) {

test/tests/generator_threads2.py

+import threading
+import traceback, sys
+
+def exc():
+    1/0
+
+def G():
+    traceback.print_stack(limit=2)
+    yield 1
+    traceback.print_stack(limit=2)
+    yield 2
+    exc()
+
+def f1(x):
+    print x.next()
+def f2(x):
+    print x.next()
+def f3(x):
+    try:
+        print x.next()
+    except:
+        print "exc"
+        traceback.print_tb(sys.exc_info()[2])
+def run(nthreads=4):
+    g = G()
+    def t(f):
+        return threading.Thread(target=f, args=(g,))
+    for t in [t(f1), t(f2), t(f3)]:
+        t.start()
+        t.join()
+run()

test/tests/sys_getframe_exited.py

-# expected: fail
-# - we don't (yet?) support looking at frame objects after
-#   their frame has exited
 import sys
 def f():
+    var = 42
     return sys._getframe(0)
 
 fr = f()
 print fr.f_locals
+
+
+def f():
+    var = 0
+    fr = sys._getframe(0)
+    var += 1
+    return fr
+fr = f()
+print fr.f_locals["var"]