Merge commit '30d451' into refcounting

some merge conflicts, about to get to

from_cpython/Include/pystate.h

@@ -111,6 +111,8 @@ typedef struct _ts {
 } PyThreadState;
 #endif
 typedef struct _ts {
+    void* frame_info; // This points to top python FrameInfo object
+
     int recursion_depth;
     int gilstate_counter;
 

from_cpython/Lib/logging/__init__.py

@@ -71,9 +71,15 @@ else:
     _srcfile = __file__
 _srcfile = os.path.normcase(_srcfile)
 
-# pyston changes: we don't support tb_frame or f_back, so always use sys._getframe
-currentframe = lambda: sys._getframe(4)
-start_getframe = 4
+# next bit filched from 1.5.2's inspect.py
+def currentframe():
+    """Return the frame object for the caller's stack frame."""
+    try:
+        raise Exception
+    except:
+        return sys.exc_info()[2].tb_frame.f_back
+
+if hasattr(sys, '_getframe'): currentframe = lambda: sys._getframe(3)
 # done filching
 
 # _srcfile is only used in conjunction with sys._getframe().
@@ -1222,23 +1228,17 @@ class Logger(Filterer):
         Find the stack frame of the caller so that we can note the source
         file name, line number and function name.
         """
-
         f = currentframe()
-        # pyston change: we can't use f_back to walk back up the stack, so increment a counter of
-        # frames to skip and repeatedly call sys._getframe
-        fn = start_getframe
         #On some versions of IronPython, currentframe() returns None if
         #IronPython isn't run with -X:Frames.
-        #if f is not None:
-        #    f = f.f_back
+        if f is not None:
+            f = f.f_back
         rv = "(unknown file)", 0, "(unknown function)"
         while hasattr(f, "f_code"):
             co = f.f_code
             filename = os.path.normcase(co.co_filename)
             if filename == _srcfile:
-                fn += 1
-                f = sys._getframe(fn);
-                #f = f.f_back
+                f = f.f_back
                 continue
             rv = (co.co_filename, f.f_lineno, co.co_name)
             break

from_cpython/Lib/traceback.py

 """Extract, format and print information about Python stack traces."""
 
-# This module has been heavily modified for Pyston, since we don't provide the
-# same traceback objects as CPython.
-
 import linecache
 import sys
 import types
@@ -59,20 +56,7 @@ def print_tb(tb, limit=None, file=None):
     if limit is None:
         if hasattr(sys, 'tracebacklimit'):
             limit = sys.tracebacklimit
-
     n = 0
-    # Pyston change:
-    for (filename, name, lineno) in tb.getLines():
-        if limit and n >= limit:
-            break
-
-        _print(file,
-               '  File "%s", line %d, in %s' % (filename, lineno, name))
-        linecache.checkcache(filename)
-        line = linecache.getline(filename, lineno, None)
-        if line: _print(file, '    ' + line.strip())
-        n = n+1
-    """
     while tb is not None and (limit is None or n < limit):
         f = tb.tb_frame
         lineno = tb.tb_lineno
@@ -86,7 +70,6 @@ def print_tb(tb, limit=None, file=None):
         if line: _print(file, '    ' + line.strip())
         tb = tb.tb_next
         n = n+1
-    """
 
 def format_tb(tb, limit = None):
     """A shorthand for 'format_list(extract_tb(tb, limit))'."""
@@ -108,18 +91,6 @@ def extract_tb(tb, limit = None):
             limit = sys.tracebacklimit
     list = []
     n = 0
-    # Pyston change:
-    for (filename, name, lineno) in tb.getLines():
-        if limit and n >= limit:
-            break
-
-        linecache.checkcache(filename)
-        line = linecache.getline(filename, lineno, None)
-        if line: line = line.strip()
-        else: line = None
-        list.append((filename, lineno, name, line))
-        n = n+1
-    """
     while tb is not None and (limit is None or n < limit):
         f = tb.tb_frame
         lineno = tb.tb_lineno
@@ -133,7 +104,6 @@ def extract_tb(tb, limit = None):
         list.append((filename, lineno, name, line))
         tb = tb.tb_next
         n = n+1
-    """
     return list
 
 
@@ -293,28 +263,19 @@ def print_stack(f=None, limit=None, file=None):
     arguments have the same meaning as for print_exception().
     """
     if f is None:
-        # Pyston change:
-        """
         try:
             raise ZeroDivisionError
         except ZeroDivisionError:
             f = sys.exc_info()[2].tb_frame.f_back
-        """
-        f = sys._getframe(1)
     print_list(extract_stack(f, limit), file)
 
 def format_stack(f=None, limit=None):
     """Shorthand for 'format_list(extract_stack(f, limit))'."""
-
     if f is None:
-        # Pyston change:
-        """
         try:
             raise ZeroDivisionError
         except ZeroDivisionError:
             f = sys.exc_info()[2].tb_frame.f_back
-        """
-        f = sys._getframe(1)
     return format_list(extract_stack(f, limit))
 
 def extract_stack(f=None, limit = None):
@@ -326,16 +287,11 @@ def extract_stack(f=None, limit = None):
     line number, function name, text), and the entries are in order
     from oldest to newest stack frame.
     """
-
     if f is None:
-        # Pyston change:
-        """
         try:
             raise ZeroDivisionError
         except ZeroDivisionError:
             f = sys.exc_info()[2].tb_frame.f_back
-        """
-        f = sys._getframe(1)
     if limit is None:
         if hasattr(sys, 'tracebacklimit'):
             limit = sys.tracebacklimit
@@ -361,6 +317,4 @@ def tb_lineno(tb):
 
     Obsolete in 2.3.
     """
-
-    raise NotImplementedError("This function is currently not implemented in Pyston")
     return tb.tb_lineno

src/codegen/ast_interpreter.cpp

@@ -138,7 +138,6 @@ private:
     CFGBlock* next_block, *current_block;
     FrameInfo frame_info;
 
-    FunctionMetadata* md;
     SourceInfo* source_info;
     ScopeInfo* scope_info;
     PhiAnalysis* phis;
@@ -156,7 +155,7 @@ public:
     ~ASTInterpreter() {
         Py_XDECREF(frame_info.boxedLocals);
 
-        int nvregs = md->calculateNumVRegs();
+        int nvregs = getMD()->calculateNumVRegs();
 
         for (int i = 0; i < nvregs; i++) {
             Py_XDECREF(vregs[i]);
@@ -184,7 +183,7 @@ public:
         return incref(frame_info.globals);
     }
 
-    FunctionMetadata* getMD() { return md; }
+    FunctionMetadata* getMD() { return frame_info.md; }
     FrameInfo* getFrameInfo() { return &frame_info; }
     BoxedClosure* getPassedClosure() { return frame_info.passed_closure; }
     Box** getVRegs() { return vregs; }
@@ -231,7 +230,9 @@ void ASTInterpreter::setBoxedLocals(Box* boxedLocals) {
 }
 
 void ASTInterpreter::setFrameInfo(const FrameInfo* frame_info) {
+    Box** vregs = this->frame_info.vregs;
     this->frame_info = *frame_info;
+    this->frame_info.vregs = vregs;
 }
 
 void ASTInterpreter::setGlobals(Box* globals) {
@@ -243,7 +244,6 @@ void ASTInterpreter::setGlobals(Box* globals) {
 ASTInterpreter::ASTInterpreter(FunctionMetadata* md, Box** vregs)
     : current_block(0),
       frame_info(ExcInfo(NULL, NULL, NULL)),
-      md(md),
       source_info(md->source.get()),
       scope_info(0),
       phis(NULL),
@@ -257,6 +257,7 @@ ASTInterpreter::ASTInterpreter(FunctionMetadata* md, Box** vregs)
 
     scope_info = source_info->getScopeInfo();
     frame_info.vregs = vregs;
+    frame_info.md = md;
 
     assert(scope_info);
 }
@@ -269,7 +270,7 @@ void ASTInterpreter::initArguments(BoxedClosure* _closure, BoxedGenerator* _gene
     if (scope_info->createsClosure())
         created_closure = createClosure(_closure, scope_info->getClosureSize());
 
-    const ParamNames& param_names = md->param_names;
+    const ParamNames& param_names = getMD()->param_names;
 
     // make sure the AST_Name nodes are set
     assert(param_names.args.size() == param_names.arg_names.size());
@@ -299,7 +300,7 @@ void ASTInterpreter::startJITing(CFGBlock* block, int exit_offset) {
     assert(ENABLE_BASELINEJIT);
     assert(!jit);
 
-    auto& code_blocks = md->code_blocks;
+    auto& code_blocks = getMD()->code_blocks;
     JitCodeBlock* code_block = NULL;
     if (!code_blocks.empty())
         code_block = code_blocks[code_blocks.size() - 1].get();
@@ -343,6 +344,8 @@ Box* ASTInterpreter::execJITedBlock(CFGBlock* b) {
         if (stmt->type != AST_TYPE::Invoke)
             throw e;
 
+        assert(getPythonFrameInfo(0) == getFrameInfo());
+
         auto source = getMD()->source.get();
         stmt->cxx_exception_count++;
         caughtCxxException(LineInfo(stmt->lineno, stmt->col_offset, source->getFn(), source->getName()), &e);
@@ -387,7 +390,7 @@ Box* ASTInterpreter::executeInner(ASTInterpreter& interpreter, CFGBlock* start_b
         interpreter.next_block = start_block;
     }
 
-    if (ENABLE_BASELINEJIT && interpreter.md->times_interpreted >= REOPT_THRESHOLD_INTERPRETER)
+    if (ENABLE_BASELINEJIT && interpreter.getMD()->times_interpreted >= REOPT_THRESHOLD_INTERPRETER)
         interpreter.should_jit = true;
 
     while (interpreter.next_block) {
@@ -662,7 +665,8 @@ Value ASTInterpreter::visit_jump(AST_Jump* node) {
 
         // we may have started JITing because the OSR thresholds got triggered in this case we don't want to jit
         // additional blocks ouside of the loop if the function is cold.
-        //if (md->times_interpreted < REOPT_THRESHOLD_INTERPRETER)
+        // XXX reenable this
+        //if (getMD()->times_interpreted < REOPT_THRESHOLD_INTERPRETER)
             //should_jit = false;
     }
 
@@ -693,7 +697,8 @@ Box* ASTInterpreter::doOSR(AST_Jump* node) {
     ast_osrs.log();
 
     LivenessAnalysis* liveness = source_info->getLiveness();
-    std::unique_ptr<PhiAnalysis> phis = computeRequiredPhis(md->param_names, source_info->cfg, liveness, scope_info);
+    std::unique_ptr<PhiAnalysis> phis
+        = computeRequiredPhis(getMD()->param_names, source_info->cfg, liveness, scope_info);
 
     llvm::DenseMap<int, InternedString> offset_name_map;
     for (auto&& v : getSymVRegMap()) {
@@ -715,7 +720,7 @@ Box* ASTInterpreter::doOSR(AST_Jump* node) {
     }
 
     const OSREntryDescriptor* found_entry = nullptr;
-    for (auto& p : md->osr_versions) {
+    for (auto& p : getMD()->osr_versions) {
         if (p.first->backedge != node)
             continue;
 
@@ -767,7 +772,7 @@ Box* ASTInterpreter::doOSR(AST_Jump* node) {
     sorted_symbol_table[source_info->getInternedStrings().get(FRAME_INFO_PTR_NAME)] = (Box*)&frame_info;
 
     if (found_entry == nullptr) {
-        OSREntryDescriptor* entry = OSREntryDescriptor::create(md, node, CXX);
+        OSREntryDescriptor* entry = OSREntryDescriptor::create(getMD(), node, CXX);
 
         for (auto& it : sorted_symbol_table) {
             if (isIsDefinedName(it.first))
@@ -824,6 +829,8 @@ Value ASTInterpreter::visit_invoke(AST_Invoke* node) {
     } catch (ExcInfo e) {
         abortJITing();
 
+        assert(getPythonFrameInfo(0) == getFrameInfo());
+
         auto source = getMD()->source.get();
         node->cxx_exception_count++;
         caughtCxxException(LineInfo(node->lineno, node->col_offset, source->getFn(), source->getName()), &e);
@@ -1773,7 +1780,10 @@ const void* interpreter_instr_addr = (void*)&executeInnerAndSetupFrame;
 
 // small wrapper around executeInner because we can not directly call the member function from asm.
 extern "C" Box* executeInnerFromASM(ASTInterpreter& interpreter, CFGBlock* start_block, AST_stmt* start_at) {
-    return ASTInterpreter::executeInner(interpreter, start_block, start_at);
+    initFrame(interpreter.getFrameInfo());
+    Box* rtn = ASTInterpreter::executeInner(interpreter, start_block, start_at);
+    deinitFrame(interpreter.getFrameInfo());
+    return rtn;
 }
 
 Box* astInterpretFunction(FunctionMetadata* md, Box* closure, Box* generator, Box* globals, Box* arg1, Box* arg2,
@@ -1924,6 +1934,9 @@ static Box* astInterpretDeoptInner(FunctionMetadata* md, AST_expr* after_expr, A
 
     SourceInfo* source_info = md->source.get();
 
+    // We can't reuse the existing vregs from the LLVM tier because they only contain the user visible ones this means
+    // there wouldn't be enough space for the compiler generated ones which the interpreter (+bjit) stores inside the
+    // vreg array.
     Box** vregs = NULL;
     int num_vregs = md->calculateNumVRegs();
     if (num_vregs > 0) {
@@ -2001,6 +2014,11 @@ static Box* astInterpretDeoptInner(FunctionMetadata* md, AST_expr* after_expr, A
         assert(starting_statement);
     }
 
+    // We need to remove the old python frame created in the LLVM tier otherwise we would have a duplicate frame because
+    // the interpreter will set the new state before executing the first statement.
+    RELEASE_ASSERT(cur_thread_state.frame_info == frame_state.frame_info, "");
+    cur_thread_state.frame_info = frame_state.frame_info->back;
+
     Box* v = ASTInterpreter::execute(interpreter, start_block, starting_statement);
     return v ? v : None;
 }
@@ -2023,33 +2041,9 @@ static ASTInterpreter* getInterpreterFromFramePtr(void* frame_ptr) {
     return *ptr;
 }
 
-FunctionMetadata* getMDForInterpretedFrame(void* frame_ptr) {
-    ASTInterpreter* interpreter = getInterpreterFromFramePtr(frame_ptr);
-    assert(interpreter);
-    return interpreter->getMD();
-}
-
 FrameInfo* getFrameInfoForInterpretedFrame(void* frame_ptr) {
     ASTInterpreter* interpreter = getInterpreterFromFramePtr(frame_ptr);
     assert(interpreter);
     return interpreter->getFrameInfo();
 }
-
-BoxedDict* localsForInterpretedFrame(Box** vregs, CFG* cfg) {
-    BoxedDict* rtn = new BoxedDict();
-    for (auto& l : cfg->sym_vreg_map_user_visible) {
-        Box* val = vregs[l.second];
-        if (val) {
-            rtn->d[l.first.getBox()] = val;
-        }
-    }
-
-    return rtn;
-}
-
-BoxedDict* localsForInterpretedFrame(void* frame_ptr) {
-    ASTInterpreter* interpreter = getInterpreterFromFramePtr(frame_ptr);
-    assert(interpreter);
-    return localsForInterpretedFrame(interpreter->getVRegs(), interpreter->getMD()->source->cfg);
-}
 }

src/codegen/ast_interpreter.h

@@ -76,13 +76,9 @@ Box* astInterpretFunctionEval(FunctionMetadata* cf, Box* globals, Box* boxedLoca
 Box* astInterpretDeopt(FunctionMetadata* cf, AST_expr* after_expr, AST_stmt* enclosing_stmt, Box* expr_val,
                        FrameStackState frame_state);
 
-FunctionMetadata* getMDForInterpretedFrame(void* frame_ptr);
 struct FrameInfo;
 FrameInfo* getFrameInfoForInterpretedFrame(void* frame_ptr);
 
-BoxedDict* localsForInterpretedFrame(Box** vregs, CFG* cfg);
-BoxedDict* localsForInterpretedFrame(void* frame_ptr);
-
 // Executes the equivalent of CPython's PRINT_EXPR opcode (call sys.displayhook)
 extern "C" void printExprHelper(Box* b);
 }

src/codegen/irgen/irgenerator.cpp

@@ -175,6 +175,11 @@ template <typename Builder> static llvm::Value* getGlobalsGep(Builder& builder,
     return builder.CreateConstInBoundsGEP2_32(v, 0, 6);
 }
 
+template <typename Builder> static llvm::Value* getMDGep(Builder& builder, llvm::Value* v) {
+    static_assert(offsetof(FrameInfo, md) == 64 + 16, "");
+    return builder.CreateConstInBoundsGEP2_32(v, 0, 8);
+}
+
 void IRGenState::setupFrameInfoVar(llvm::Value* passed_closure, llvm::Value* passed_globals,
                                    llvm::Value* frame_info_arg) {
     /*
@@ -279,9 +284,12 @@ void IRGenState::setupFrameInfoVar(llvm::Value* passed_closure, llvm::Value* pas
         builder.CreateStore(passed_globals, getGlobalsGep(builder, al));
         // set frame_info.vregs
         builder.CreateStore(vregs, getVRegsGep(builder, al));
+        builder.CreateStore(embedRelocatablePtr(getMD(), g.llvm_functionmetadata_type_ptr), getMDGep(builder, al));
 
         this->frame_info = al;
         this->globals = passed_globals;
+
+        builder.CreateCall(g.funcs.initFrame, this->frame_info);
     }
 
     stmt = getStmtGep(builder, frame_info);
@@ -350,6 +358,10 @@ private:
 
     llvm::CallSite emitCall(const UnwindInfo& unw_info, llvm::Value* callee, const std::vector<llvm::Value*>& args,
                             ExceptionStyle target_exception_style) {
+        llvm::Value* stmt = unw_info.current_stmt ? embedRelocatablePtr(unw_info.current_stmt, g.llvm_aststmt_type_ptr)
+                                                  : getNullPtr(g.llvm_aststmt_type_ptr);
+        getBuilder()->CreateStore(stmt, irstate->getStmtVar());
+
         bool needs_cxx_interception;
         if (unw_info.exc_dest == NO_CXX_INTERCEPTION) {
             needs_cxx_interception = false;
@@ -430,7 +442,7 @@ private:
 
         pp_args.insert(pp_args.end(), ic_stackmap_args.begin(), ic_stackmap_args.end());
 
-        irgenerator->addFrameStackmapArgs(info, unw_info.current_stmt, pp_args);
+        irgenerator->addFrameStackmapArgs(info, pp_args);
 
         llvm::Intrinsic::ID intrinsic_id;
         if (return_type->isIntegerTy() || return_type->isPointerTy()) {
@@ -444,7 +456,6 @@ private:
             abort();
         }
         llvm::Function* patchpoint = this->getIntrinsic(intrinsic_id);
-
         llvm::CallSite rtn = this->emitCall(unw_info, patchpoint, pp_args, target_exception_style);
         return rtn;
     }
@@ -503,10 +514,6 @@ public:
             }
         }
 #endif
-
-        llvm::Value* stmt = unw_info.current_stmt ? embedRelocatablePtr(unw_info.current_stmt, g.llvm_aststmt_type_ptr)
-                                                  : getNullPtr(g.llvm_aststmt_type_ptr);
-        getBuilder()->CreateStore(stmt, irstate->getStmtVar());
         return emitCall(unw_info, callee, args, target_exception_style).getInstruction();
     }
 
@@ -2098,6 +2105,9 @@ private:
 
         ConcreteCompilerVariable* rtn = val->makeConverted(emitter, opt_rtn_type);
 
+        if (!irstate->getCurFunction()->entry_descriptor)
+            emitter.createCall(unw_info, g.funcs.deinitFrame, irstate->getFrameInfoVar());
+
         assert(rtn->getValue());
         auto ret_inst = emitter.getBuilder()->CreateRet(rtn->getValue());
 
@@ -2546,24 +2556,9 @@ private:
     }
 
 public:
-    void addFrameStackmapArgs(PatchpointInfo* pp, AST_stmt* current_stmt,
-                              std::vector<llvm::Value*>& stackmap_args) override {
+    void addFrameStackmapArgs(PatchpointInfo* pp, std::vector<llvm::Value*>& stackmap_args) override {
         int initial_args = stackmap_args.size();
 
-        assert(UNBOXED_INT->llvmType() == g.i64);
-        if (ENABLE_JIT_OBJECT_CACHE) {
-            llvm::Value* v;
-            if (current_stmt)
-                v = emitter.getBuilder()->CreatePtrToInt(embedRelocatablePtr(current_stmt, g.i8_ptr), g.i64);
-            else
-                v = getConstantInt(0, g.i64);
-            stackmap_args.push_back(v);
-        } else {
-            stackmap_args.push_back(getConstantInt((uint64_t)current_stmt, g.i64));
-        }
-
-        pp->addFrameVar("!current_stmt", UNBOXED_INT);
-
         // For deopts we need to add the compiler created names to the stackmap
         if (ENABLE_FRAME_INTROSPECTION && pp->isDeopt()) {
             // TODO: don't need to use a sorted symbol table if we're explicitly recording the names!
@@ -2867,8 +2862,9 @@ public:
             emitter.setCurrentBasicBlock(capi_exc_dest);
             assert(!phi_node);
             phi_node = emitter.getBuilder()->CreatePHI(g.llvm_aststmt_type_ptr, 0);
-            emitter.getBuilder()->CreateCall2(g.funcs.caughtCapiException, phi_node,
-                                              embedRelocatablePtr(irstate->getSourceInfo(), g.i8_ptr));
+
+            emitter.createCall(UnwindInfo(current_stmt, NULL), g.funcs.caughtCapiException,
+                               { phi_node, embedRelocatablePtr(irstate->getSourceInfo(), g.i8_ptr) });
 
             if (!final_dest) {
                 // Propagate the exception out of the function:
@@ -2876,6 +2872,7 @@ public:
                     emitter.getBuilder()->CreateCall(g.funcs.reraiseCapiExcAsCxx);
                     emitter.getBuilder()->CreateUnreachable();
                 } else {
+                    emitter.createCall(UnwindInfo(current_stmt, NULL), g.funcs.deinitFrame, irstate->getFrameInfoVar());
                     emitter.getBuilder()->CreateRet(getNullPtr(g.llvm_value_type_ptr));
                 }
             } else {

src/codegen/irgen/irgenerator.h

@@ -163,8 +163,7 @@ public:
     virtual void run(const CFGBlock* block) = 0; // primary entry point
     virtual EndingState getEndingSymbolTable() = 0;
     virtual void doSafePoint(AST_stmt* next_statement) = 0;
-    virtual void addFrameStackmapArgs(PatchpointInfo* pp, AST_stmt* current_stmt,
-                                      std::vector<llvm::Value*>& stackmap_args) = 0;
+    virtual void addFrameStackmapArgs(PatchpointInfo* pp, std::vector<llvm::Value*>& stackmap_args) = 0;
     virtual void addOutgoingExceptionState(ExceptionState exception_state) = 0;
     virtual void setIncomingExceptionState(llvm::SmallVector<ExceptionState, 2> exc_state) = 0;
     virtual llvm::BasicBlock* getCXXExcDest(const UnwindInfo&) = 0;

src/codegen/runtime_hooks.cpp

@@ -198,6 +198,8 @@ void initGlobalFuncs(GlobalState& g) {
     GET(createClosure);
     GET(createGenerator);
     GET(createSet);
+    GET(initFrame);
+    GET(deinitFrame);
 
     GET(getattr);
     GET(getattr_capi);

src/codegen/runtime_hooks.h

@@ -34,7 +34,7 @@ struct GlobalFuncs {
 
     llvm::Value* boxInt, *unboxInt, *boxFloat, *unboxFloat, *createFunctionFromMetadata, *getFunctionMetadata,
         *boxInstanceMethod, *boxBool, *unboxBool, *createTuple, *createDict, *createList, *createSlice,
-        *createUserClass, *createClosure, *createGenerator, *createSet;
+        *createUserClass, *createClosure, *createGenerator, *createSet, *initFrame, *deinitFrame;
     llvm::Value* getattr, *getattr_capi, *setattr, *delattr, *delitem, *delGlobal, *nonzero, *binop, *compare,
         *augbinop, *unboxedLen, *getitem, *getitem_capi, *getclsattr, *getGlobal, *setitem, *unaryop, *import,
         *importFrom, *importStar, *repr, *exceptionMatches, *yield, *getiterHelper, *hasnext, *setGlobal, *apply_slice;

src/codegen/unwinding.h

@@ -80,28 +80,15 @@ public:
     FrameInfo* getFrameInfo();
     bool exists() { return impl.get() != NULL; }
     AST_stmt* getCurrentStatement();
-    Box* fastLocalsToBoxedLocals();
     Box* getGlobalsDict();
 
-    // Gets the "current version" of this frame: if the frame has executed since
-    // the iterator was obtained, the methods may return old values. This returns
-    // an updated copy that returns the updated values.
-    // The "current version" will live at the same stack location, but any other
-    // similarities need to be verified by the caller, ie it is up to the caller
-    // to determine that we didn't leave and reenter the stack frame.
-    // This function can only be called from the thread that created this object.
-    PythonFrameIterator getCurrentVersion();
-
-    // Assuming this is a valid frame iterator, return the next frame back (ie older).
-    PythonFrameIterator back();
-
     PythonFrameIterator(PythonFrameIterator&& rhs);
     void operator=(PythonFrameIterator&& rhs);
     PythonFrameIterator(std::unique_ptr<PythonFrameIteratorImpl> impl);
     ~PythonFrameIterator();
 };
 
-PythonFrameIterator getPythonFrame(int depth);
+FrameInfo* getPythonFrameInfo(int depth);
 
 // Fetches a writeable pointer to the frame-local excinfo object,
 // calculating it if necessary (from previous frames).

src/core/threading.cpp

@@ -39,7 +39,7 @@ std::unordered_set<PerThreadSetBase*> PerThreadSetBase::all_instances;
 
 extern "C" {
 __thread PyThreadState cur_thread_state
-    = { 0, 1, NULL, NULL, NULL, NULL, 0, NULL }; // not sure if we need to explicitly request zero-initialization
+    = { NULL, 0, 1, NULL, NULL, NULL, NULL, 0, NULL }; // not sure if we need to explicitly request zero-initialization
 }
 
 PthreadFastMutex threading_lock;

src/core/types.h

@@ -947,14 +947,16 @@ struct FrameInfo {
     BoxedClosure* passed_closure;
 
     Box** vregs;
-    // Current statement
-    // Caution the llvm tier only updates this information on direct external calls but not for patchpoints.
-    // This means if a patchpoint "current_stmt" info is available it must be used instead of this field.
-    AST_stmt* stmt;
+    AST_stmt* stmt; // current statement
     // This is either a module or a dict
     Box* globals;
 
-    FrameInfo(ExcInfo exc) : exc(exc), boxedLocals(NULL), frame_obj(0), passed_closure(0), vregs(0), stmt(0), globals(0) {}
+    FrameInfo* back;
+    FunctionMetadata* md;
+
+    Box* updateBoxedLocals();
+
+    FrameInfo(ExcInfo exc) : exc(exc), boxedLocals(NULL), frame_obj(0), passed_closure(0), vregs(0), stmt(0), globals(0), back(0), md(0) {}
 };
 
 // callattr() takes a number of flags and arguments, and for performance we pack them into a single register:

src/runtime/exceptions.cpp

@@ -51,7 +51,7 @@ void raiseSyntaxError(const char* msg, int lineno, int col_offset, llvm::StringR
     } else {
         // This is more like how the parser handles it:
         exc = runtimeCall(SyntaxError, ArgPassSpec(1), boxString(msg), NULL, NULL, NULL, NULL);
-        tb = new BoxedTraceback(LineInfo(lineno, col_offset, boxString(file), boxString(func)), None);
+        tb = new BoxedTraceback(LineInfo(lineno, col_offset, boxString(file), boxString(func)), None, getFrame(0));
     }
 
     assert(!PyErr_Occurred());
@@ -304,7 +304,7 @@ bool exceptionAtLineCheck() {
 
 void exceptionAtLine(LineInfo line_info, Box** traceback) {
     if (exceptionAtLineCheck())
-        BoxedTraceback::here(line_info, traceback);
+        BoxedTraceback::here(line_info, traceback, getFrame((FrameInfo*)cur_thread_state.frame_info));
 }
 
 void startReraise() {

src/runtime/frame.cpp

@@ -30,25 +30,24 @@ BoxedClass* frame_cls;
 class BoxedFrame : public Box {
 private:
     // Call boxFrame to get a BoxedFrame object.
-    BoxedFrame(PythonFrameIterator it) __attribute__((visibility("default")))
-    : it(std::move(it)), thread_id(PyThread_get_thread_ident()) {}
+    BoxedFrame(FrameInfo* frame_info) __attribute__((visibility("default")))
+    : frame_info(frame_info), _back(NULL), _code(NULL), _globals(NULL), _locals(NULL), _stmt(NULL) {
+        assert(frame_info);
+    }
 
 public:
-    PythonFrameIterator it;
-    long thread_id;
+    FrameInfo* frame_info;
 
-    Box* _globals;
+    Box* _back;
     Box* _code;
+    Box* _globals;
+    Box* _locals;
+
+    AST_stmt* _stmt;
+
+
+    bool hasExited() const { return frame_info == NULL; }
 
-    void update() {
-        // This makes sense as an exception, but who knows how the user program would react
-        // (it might swallow it and do something different)
-        RELEASE_ASSERT(thread_id == PyThread_get_thread_ident(),
-                       "frame objects can only be accessed from the same thread");
-        PythonFrameIterator new_it = it.getCurrentVersion();
-        RELEASE_ASSERT(new_it.exists() && new_it.getFrameInfo()->frame_obj == this, "frame has exited");
-        it = std::move(new_it);
-    }
 
     // cpython frame objects have the following attributes
 
@@ -77,62 +76,85 @@ public:
     // * = unsupported in Pyston
     // ** = getter supported, but setter unsupported
 
-    static void simpleDestructor(Box* b) {
-        auto f = static_cast<BoxedFrame*>(b);
-
-        f->it.~PythonFrameIterator();
-    }
-
     static Box* code(Box* obj, void*) {
         auto f = static_cast<BoxedFrame*>(obj);
+
+        if (!f->_code)
+            f->_code = (Box*)f->frame_info->md->getCode();
         return f->_code;
     }
 
     static Box* locals(Box* obj, void*) {
         auto f = static_cast<BoxedFrame*>(obj);
-        f->update();
-        return f->it.fastLocalsToBoxedLocals();
+
+        if (f->hasExited())
+            return f->_locals;
+        return f->frame_info->updateBoxedLocals();
     }
 
     static Box* globals(Box* obj, void*) {
         auto f = static_cast<BoxedFrame*>(obj);
+
+        if (!f->_globals) {
+            f->_globals = f->frame_info->globals;
+            if (f->_globals && PyModule_Check(f->_globals))
+                f->_globals = f->_globals->getAttrWrapper();
+        }
         return f->_globals;
     }
 
     static Box* back(Box* obj, void*) {
         auto f = static_cast<BoxedFrame*>(obj);
-        f->update();
 
-        PythonFrameIterator it = f->it.back();
-        if (!it.exists())
-            return None;
-        return BoxedFrame::boxFrame(std::move(it));
+        if (!f->_back) {
+            if (!f->frame_info->back)
+                f->_back = None;
+            else
+                f->_back = BoxedFrame::boxFrame(f->frame_info->back);
+        }
+        return f->_back;
     }
 
     static Box* lineno(Box* obj, void*) {
         auto f = static_cast<BoxedFrame*>(obj);
-        f->update();
-        AST_stmt* stmt = f->it.getCurrentStatement();
+
+        if (f->hasExited())
+            return boxInt(f->_stmt->lineno);
+
+        AST_stmt* stmt = f->frame_info->stmt;
         return boxInt(stmt->lineno);
     }
 
-    DEFAULT_CLASS(frame_cls);
+    void handleFrameExit() {
+        if (hasExited())
+            return;
 
-    static Box* boxFrame(PythonFrameIterator it) {
-        FrameInfo* fi = it.getFrameInfo();
-        if (fi->frame_obj == NULL) {
-            auto md = it.getMD();
-            Box* globals = it.getGlobalsDict();
-            BoxedFrame* f = fi->frame_obj = new BoxedFrame(std::move(it));
-            f->_globals = globals;
-            f->_code = (Box*)md->getCode();
-        }
+        _back = back(this, NULL);
+        _code = code(this, NULL);
+        _globals = globals(this, NULL);
+        _locals = locals(this, NULL);
+        _stmt = frame_info->stmt;
+
+        frame_info = NULL; // this means exited == true
+        assert(hasExited());
+    }
+
+    DEFAULT_CLASS(frame_cls);
 
+    static Box* boxFrame(FrameInfo* fi) {
+        if (fi->frame_obj == NULL)
+            fi->frame_obj = new BoxedFrame(fi);
+        assert(fi->frame_obj->cls == frame_cls);
         return fi->frame_obj;
     }
 
     static void dealloc(Box* b) noexcept {
         Py_FatalError("unimplemented");
+
+        //Py_DECREF(f->_back);
+        //Py_DECREF(f->_code);
+        //Py_DECREF(f->_globals);
+        //Py_DECREF(f->_locals);
     }
     static int traverse(Box* self, visitproc visit, void *arg) noexcept {
         Py_FatalError("unimplemented");
@@ -146,25 +168,37 @@ extern "C" int PyFrame_ClearFreeList(void) noexcept {
     return 0; // number of entries cleared
 }
 
+Box* getFrame(FrameInfo* frame_info) {
+    return BoxedFrame::boxFrame(frame_info);
+}
+
 Box* getFrame(int depth) {
-    auto it = getPythonFrame(depth);
-    if (!it.exists())
+    FrameInfo* frame_info = getPythonFrameInfo(depth);
+    if (!frame_info)
         return NULL;
+    return BoxedFrame::boxFrame(frame_info);
+}
+
+void frameInvalidateBack(BoxedFrame* frame) {
+    RELEASE_ASSERT(!frame->hasExited(), "should not happen");
+    frame->_back = NULL;
+}
+
+extern "C" void initFrame(FrameInfo* frame_info) {
+    frame_info->back = (FrameInfo*)(cur_thread_state.frame_info);
+    cur_thread_state.frame_info = frame_info;
+}
 
-    return BoxedFrame::boxFrame(std::move(it));
+extern "C" void deinitFrame(FrameInfo* frame_info) {
+    cur_thread_state.frame_info = frame_info->back;
+    BoxedFrame* frame = frame_info->frame_obj;
+    if (frame)
+        frame->handleFrameExit();
 }
 
 extern "C" int PyFrame_GetLineNumber(PyFrameObject* _f) noexcept {
-    // TODO remove this when we are able to introspect exited frames:
-    // We check if the frame exited and only return the correct line number when it is still available.
-    // Because of a limitation in out current frame introspection we can also not inspect OSRed frames.
-    BoxedFrame* f = (BoxedFrame*)_f;
-    PythonFrameIterator new_it = f->it.getCurrentVersion();
-    if (new_it.exists() && new_it.getFrameInfo()->frame_obj == f) {
-        BoxedInt* lineno = (BoxedInt*)BoxedFrame::lineno((Box*)f, NULL);
-        return lineno->n;
-    }
-    return -1;
+    BoxedInt* lineno = (BoxedInt*)BoxedFrame::lineno((Box*)_f, NULL);
+    return lineno->n;
 }
 
 extern "C" PyObject* PyFrame_GetGlobals(PyFrameObject* f) noexcept {
@@ -179,8 +213,6 @@ void setupFrame() {
     frame_cls = BoxedClass::create(type_cls, object_cls, 0, 0, sizeof(BoxedFrame), false, "frame", false,
                                    (destructor)BoxedFrame::dealloc, NULL, true, (traverseproc)BoxedFrame::traverse,
                                    (inquiry)BoxedFrame::clear);
-    frame_cls->tp_dealloc = BoxedFrame::simpleDestructor;
-    frame_cls->has_safe_tp_dealloc = true;
 
     frame_cls->giveAttrDescriptor("f_code", BoxedFrame::code, NULL);
     frame_cls->giveAttrDescriptor("f_locals", BoxedFrame::locals, NULL);

src/runtime/generator.cpp

@@ -93,6 +93,8 @@ void generatorEntry(BoxedGenerator* g) {
         try {
             RegisterHelper context_registerer(g, __builtin_frame_address(0));
 
+            g->top_caller_frame_info = (FrameInfo*)cur_thread_state.frame_info;
+
             // call body of the generator
             BoxedFunctionBase* func = g->function;
 
@@ -109,6 +111,7 @@ void generatorEntry(BoxedGenerator* g) {
         g->entryExited = true;
         threading::popGenerator();
     }
+    assert(g->top_caller_frame_info == cur_thread_state.frame_info);
     swapContext(&g->context, g->returnContext, 0);
 }
 
@@ -155,8 +158,11 @@ template <ExceptionStyle S> static bool generatorSendInternal(BoxedGenerator* se
     else
         self->prev_stack = StatTimer::swapStack(self->prev_stack);
 #endif
-
+    auto* top_caller_frame_info = (FrameInfo*)cur_thread_state.frame_info;
     swapContext(&self->returnContext, self->context, (intptr_t)self);
+    assert(cur_thread_state.frame_info == top_caller_frame_info
+           && "the generator should reset the frame info before the swapContext");
+
 
 #if STAT_TIMERS
     self->prev_stack = StatTimer::swapStack(self->prev_stack);
@@ -314,7 +320,28 @@ extern "C" Box* yield(BoxedGenerator* obj, Box* value) {
     self->returnValue = value;
 
     threading::popGenerator();
+
+    FrameInfo* generator_frame_info = (FrameInfo*)cur_thread_state.frame_info;
+    // a generator will only switch back (yield/unhandled exception) to its caller when it is one frame away from the
+    // caller
+    assert(self->top_caller_frame_info == generator_frame_info->back);
+
+    // reset current frame to the caller tops frame --> removes the frame the generator added
+    cur_thread_state.frame_info = self->top_caller_frame_info;
     swapContext(&self->context, self->returnContext, 0);
+    FrameInfo* top_new_caller_frame_info = (FrameInfo*)cur_thread_state.frame_info;
+
+    // the caller of the generator can change between yield statements that means we can't just restore the top of the
+    // frame to the point before the yield instead we have to update it.
+    if (top_new_caller_frame_info != self->top_caller_frame_info) {
+        // caller changed
+        self->top_caller_frame_info = top_new_caller_frame_info;
+        generator_frame_info->back = top_new_caller_frame_info;
+        if (generator_frame_info->frame_obj)
+            frameInvalidateBack(generator_frame_info->frame_obj);
+    }
+    cur_thread_state.frame_info = generator_frame_info;
+
     threading::pushGenerator(obj, obj->stack_begin, obj->returnContext);
 
     // if the generator receives a exception from the caller we have to throw it
@@ -347,7 +374,8 @@ extern "C" BoxedGenerator::BoxedGenerator(BoxedFunctionBase* function, Box* arg1
       returnValue(nullptr),
       exception(nullptr, nullptr, nullptr),
       context(nullptr),
-      returnContext(nullptr)
+      returnContext(nullptr),
+      top_caller_frame_info(nullptr)
 #if STAT_TIMERS
       ,
       prev_stack(NULL),

src/runtime/inline/link_forcer.cpp

@@ -72,6 +72,8 @@ void force() {
     FORCE(createPureImaginary);
     FORCE(createSet);
     FORCE(decodeUTF8StringPtr);
+    FORCE(initFrame);
+    FORCE(deinitFrame);
 
     FORCE(getattr);
     FORCE(getattr_capi);

src/runtime/traceback.cpp

@@ -82,38 +82,15 @@ void printTraceback(Box* b) {
     }
 }
 
-Box* BoxedTraceback::getLines(Box* b) {
-    assert(b->cls == traceback_cls);
-
-    BoxedTraceback* tb = static_cast<BoxedTraceback*>(b);
-
-    if (!tb->py_lines) {
-        BoxedList* lines = new BoxedList();
-        for (BoxedTraceback* wtb = tb; wtb && wtb != None; wtb = static_cast<BoxedTraceback*>(wtb->tb_next)) {
-            auto& line = wtb->line;
-            auto l = BoxedTuple::create({ line.file, line.func, boxInt(line.line) });
-            listAppendInternal(lines, l);
-        }
-        tb->py_lines = lines;
-    }
-
-    return tb->py_lines;
-}
-
-void BoxedTraceback::here(LineInfo lineInfo, Box** tb) {
-    Box* old_tb = *tb;
-    *tb = new BoxedTraceback(std::move(lineInfo), *tb);
-    Py_DECREF(old_tb);
-}
-
 void BoxedTraceback::dealloc(Box* b) noexcept {
     BoxedTraceback* self = static_cast<BoxedTraceback*>(b);
 
     Py_DECREF(self->tb_next);
-    Py_XDECREF(self->py_lines);
     Py_DECREF(self->line.file);
     Py_DECREF(self->line.func);
 
+    Py_DECREF(self->tb_frame);
+
     PyObject_GC_Del(b);
 }
 
@@ -121,7 +98,7 @@ int BoxedTraceback::traverse(Box* self, visitproc visit, void *arg) noexcept {
     BoxedTraceback* tb = static_cast<BoxedTraceback*>(self);
 
     Py_VISIT(tb->tb_next);
-    Py_VISIT(tb->py_lines);
+    Py_VISIT(tb->tb_frame);
     Py_VISIT(tb->line.file);
     Py_VISIT(tb->line.func);
 
@@ -132,11 +109,10 @@ int BoxedTraceback::clear(Box* self) noexcept {
     abort();
 }
 
-static Box* traceback_tb_next(Box* self, void*) {
-    assert(self->cls == traceback_cls);
-
-    BoxedTraceback* traceback = static_cast<BoxedTraceback*>(self);
-    return traceback->tb_next;
+void BoxedTraceback::here(LineInfo lineInfo, Box** tb, Box* frame) {
+    Box* old_tb = *tb;
+    *tb = new BoxedTraceback(std::move(lineInfo), *tb, frame);
+    Py_DECREF(old_tb);
 }
 
 extern "C" int _Py_DisplaySourceLine(PyObject* f, const char* filename, int lineno, int indent) noexcept {
@@ -148,17 +124,15 @@ void setupTraceback() {
                                        (destructor)BoxedTraceback::dealloc, NULL, true,
                                        (traverseproc)BoxedTraceback::traverse, (inquiry)BoxedTraceback::clear);
 
-    traceback_cls->giveAttr("getLines",
-                            new BoxedFunction(FunctionMetadata::create((void*)BoxedTraceback::getLines, UNKNOWN, 1)));
-
     /*
      * Currently not supported.
-    traceback_cls->giveAttr("tb_frame", new (pyston_getset_cls) BoxedGetsetDescriptor(traceback_tb_frame, NULL, NULL));
     traceback_cls->giveAttr("tb_lasti", new (pyston_getset_cls) BoxedGetsetDescriptor(traceback_tb_lasti, NULL, NULL));
-    traceback_cls->giveAttr("tb_lineno", new (pyston_getset_cls) BoxedGetsetDescriptor(traceback_tb_lineno, NULL,
-    NULL));
     */
-    traceback_cls->giveAttrDescriptor("tb_next", traceback_tb_next, NULL);
+
+    traceback_cls->giveAttr(
+        "tb_frame", new BoxedMemberDescriptor(BoxedMemberDescriptor::OBJECT, offsetof(BoxedTraceback, tb_frame)));
+    traceback_cls->giveAttr(
+        "tb_next", new BoxedMemberDescriptor(BoxedMemberDescriptor::OBJECT, offsetof(BoxedTraceback, tb_next)));
     traceback_cls->giveAttr("tb_lineno",
                             new BoxedMemberDescriptor(BoxedMemberDescriptor::INT,
                                                       offsetof(BoxedTraceback, line) + offsetof(LineInfo, line)));

src/runtime/traceback.h

@@ -27,24 +27,30 @@ class GCVisitor;
 extern "C" BoxedClass* traceback_cls;
 class BoxedTraceback : public Box {
 public:
-    Box* tb_next;
     LineInfo line;
-    Box* py_lines;
+    Box* tb_next;
+    Box* tb_frame;
 
-    BoxedTraceback(LineInfo line, Box* tb_next) : tb_next(tb_next), line(std::move(line)), py_lines(NULL) {
+    BoxedTraceback(LineInfo line, Box* tb_next, Box* tb_frame)
+        : line(std::move(line)), tb_next(tb_next), tb_frame(tb_frame) {
         Py_INCREF(tb_next);
         Py_INCREF(line.file);
         Py_INCREF(line.func);
+        if (!tb_frame)
+            this->tb_frame = None;
+        else
+            assert(tb_frame->cls == frame_cls);
+        Py_INCREF(this->tb_frame);
     }
 
     DEFAULT_CLASS(traceback_cls);
 
-    static Box* getLines(Box* b);
+    static Box* lineno(Box* obj, void*);
 
     static void gcHandler(gc::GCVisitor* v, Box* b);
 
     // somewhat equivalent to PyTraceBack_Here
-    static void here(LineInfo lineInfo, Box** tb);
+    static void here(LineInfo lineInfo, Box** tb, Box* frame);
 
     static void dealloc(Box* b) noexcept;
     static int traverse(Box* self, visitproc visit, void *arg) noexcept;

src/runtime/types.h

@@ -1169,6 +1169,7 @@ public:
 
     struct Context* context, *returnContext;
     void* stack_begin;
+    FrameInfo* top_caller_frame_info;
 
 #if STAT_TIMERS
     StatTimer* prev_stack;
@@ -1272,7 +1273,11 @@ Box* codeForFunction(BoxedFunction*);
 Box* codeForFunctionMetadata(FunctionMetadata*);
 FunctionMetadata* metadataFromCode(Box* code);
 
+Box* getFrame(FrameInfo* frame_info);
 Box* getFrame(int depth);
+void frameInvalidateBack(BoxedFrame* frame);
+extern "C" void deinitFrame(FrameInfo* frame_info);
+extern "C" void initFrame(FrameInfo* frame_info);
 
 inline BoxedString* boxString(llvm::StringRef s) {
     if (s.size() <= 1) {

test/tests/generator_threads2.py

+import threading
+import traceback, sys
+
+def exc():
+    1/0
+
+def G():
+    traceback.print_stack(limit=2)
+    yield 1
+    traceback.print_stack(limit=2)
+    yield 2
+    exc()
+
+def f1(x):
+    print x.next()
+def f2(x):
+    print x.next()
+def f3(x):
+    try:
+        print x.next()
+    except:
+        print "exc"
+        traceback.print_tb(sys.exc_info()[2])
+def run(nthreads=4):
+    g = G()
+    def t(f):
+        return threading.Thread(target=f, args=(g,))
+    for t in [t(f1), t(f2), t(f3)]:
+        t.start()
+        t.join()
+run()

test/tests/sys_getframe_exited.py

-# expected: fail
-# - we don't (yet?) support looking at frame objects after
-#   their frame has exited
 import sys
 def f():
+    var = 42
     return sys._getframe(0)
 
 fr = f()
 print fr.f_locals
+
+
+def f():
+    var = 0
+    fr = sys._getframe(0)
+    var += 1
+    return fr
+fr = f()
+print fr.f_locals["var"]