move unwind_state to ThreadStateInternal. move the class decl to threading.h...

move unwind_state to ThreadStateInternal.  move the class decl to threading.h and ensure the small/hot accessors are inlined

from_cpython/Include/pystate.h

@@ -110,15 +110,6 @@ typedef struct _ts {
 
 } PyThreadState;
 #endif
-// somewhat similar to CPython's WHY_* enum
-// UNWIND_STATE_NORMAL : == WHY_EXCEPTION.  we call it "NORMAL" since we often unwind due to things other than exceptions (getGlobals, getLocals, etc)
-// UNWIND_STATE_RERAISE: same as NORMAL, except we are supposed to skip the first frame.
-// UNWIND_STATE_OSR    : The previous frame was an osr replacement for the next one, so we should skip it
-enum UnwindState {
-    UNWIND_STATE_NORMAL = 0,
-    UNWIND_STATE_RERAISE,
-    UNWIND_STATE_OSR
-};
 typedef struct _ts {
     int recursion_depth;
 
@@ -128,9 +119,6 @@ typedef struct _ts {
 
     PyObject *dict;  /* Stores per-thread state */
 
-    // one of the UNWIND_STATE_* above
-    int unwind_state;
-
     // Pyston note: additions in here need to be mirrored in ThreadStateInternal::accept
 } PyThreadState;
 

src/codegen/ast_interpreter.cpp

@@ -318,7 +318,8 @@ void ASTInterpreter::initArguments(int nargs, BoxedClosure* _closure, BoxedGener
 static std::unordered_map<void*, ASTInterpreter*> s_interpreterMap;
 static_assert(THREADING_USE_GIL, "have to make the interpreter map thread safe!");
 
-RegisterHelper::RegisterHelper() : frame_addr(NULL), interpreter(NULL) { }
+RegisterHelper::RegisterHelper() : frame_addr(NULL), interpreter(NULL) {
+}
 
 RegisterHelper::~RegisterHelper() {
     assert(interpreter);
@@ -335,12 +336,12 @@ void RegisterHelper::doRegister(void* frame_addr, ASTInterpreter* interpreter) {
     interpreter->frame_addr = frame_addr;
     s_interpreterMap[frame_addr] = interpreter;
 }
-    
-    void RegisterHelper::deregister(void* frame_addr) {
-        assert(frame_addr);
-        assert(s_interpreterMap.count(frame_addr));
-        s_interpreterMap.erase(frame_addr);
-    }
+
+void RegisterHelper::deregister(void* frame_addr) {
+    assert(frame_addr);
+    assert(s_interpreterMap.count(frame_addr));
+    s_interpreterMap.erase(frame_addr);
+}
 
 Value ASTInterpreter::executeInner(ASTInterpreter& interpreter, CFGBlock* start_block, AST_stmt* start_at,
                                    RegisterHelper* reg) {
@@ -631,7 +632,7 @@ Value ASTInterpreter::visit_invoke(AST_Invoke* node) {
         next_block = node->normal_dest;
     } catch (ExcInfo e) {
 
-        if (cur_thread_state.unwind_state == UNWIND_STATE_NORMAL) {
+        if (threading::ThreadStateInternal::getUnwindState() == UNWIND_STATE_NORMAL) {
             // when generating the traceback incrementally we only
             // include an interpreter frame if we unwind through
             // ASTInterpreter::execute_inner.  this will keep a toplevel
@@ -643,7 +644,7 @@ Value ASTInterpreter::visit_invoke(AST_Invoke* node) {
                                  reinterpret_cast<BoxedTraceback**>(&e.traceback));
         }
 
-        cur_thread_state.unwind_state = UNWIND_STATE_NORMAL;
+        threading::ThreadStateInternal::setUnwindState(UNWIND_STATE_NORMAL);
 
         next_block = node->exc_dest;
         last_exception = e;

src/codegen/unwinding.cpp

@@ -473,13 +473,13 @@ bool unwindProcessFrame(unw_word_t ip, unw_word_t bp, unw_cursor_t* cursor, Fram
         }
     }
 
-    if (cur_thread_state.unwind_state == UNWIND_STATE_NORMAL) {
+    if (threading::ThreadStateInternal::getUnwindState() == UNWIND_STATE_NORMAL) {
         bool stop = func(&info);
         if (stop)
             return true;
     }
 
-    cur_thread_state.unwind_state = (bool)cf->entry_descriptor ? UNWIND_STATE_OSR : UNWIND_STATE_NORMAL;
+    threading::ThreadStateInternal::setUnwindState((bool)cf->entry_descriptor ? UNWIND_STATE_OSR : UNWIND_STATE_NORMAL);
     return false;
 }
 
@@ -488,7 +488,8 @@ bool unwindProcessFrame(unw_word_t ip, unw_word_t bp, unw_cursor_t* cursor, Fram
 // C++11 range loops, for example).
 // Return true from the handler to stop iteration at that frame.
 template <typename Func> void unwindPythonStack(Func func) {
-    cur_thread_state.unwind_state = UNWIND_STATE_NORMAL; // ensure we won't be skipping any python frames at the start
+    threading::ThreadStateInternal::setUnwindState(
+        UNWIND_STATE_NORMAL); // ensure we won't be skipping any python frames at the start
     unw_context_t ctx;
     unw_cursor_t cursor;
     unw_getcontext(&ctx);

src/core/threading.cpp

@@ -37,9 +37,8 @@ namespace pyston {
 namespace threading {
 
 extern "C" {
-__thread PyThreadState cur_thread_state = {
-    0, NULL, NULL, NULL, NULL, UNWIND_STATE_NORMAL
-}; // not sure if we need to explicitly request zero-initialization
+__thread PyThreadState cur_thread_state
+    = { 0, NULL, NULL, NULL, NULL }; // not sure if we need to explicitly request zero-initialization
 }
 
 PthreadFastMutex threading_lock;
@@ -52,113 +51,38 @@ PthreadFastMutex threading_lock;
 // be checked while the threading_lock is held; might not be worth it.
 int num_starting_threads(0);
 
-class ThreadStateInternal {
-private:
-    bool saved;
-    ucontext_t ucontext;
-
-public:
-    void* stack_start;
-
-    struct StackInfo {
-        BoxedGenerator* next_generator;
-        void* stack_start;
-        void* stack_limit;
-
-        StackInfo(BoxedGenerator* next_generator, void* stack_start, void* stack_limit)
-            : next_generator(next_generator), stack_start(stack_start), stack_limit(stack_limit) {
-#if STACK_GROWS_DOWN
-            assert(stack_start > stack_limit);
-            assert((char*)stack_start - (char*)stack_limit < (1L << 30));
-#else
-            assert(stack_start < stack_limit);
-            assert((char*)stack_limit - (char*)stack_start < (1L << 30));
-#endif
-        }
-    };
-
-    std::vector<StackInfo> previous_stacks;
-    pthread_t pthread_id;
-
-    ExcInfo exc_info;
-
-    PyThreadState* public_thread_state;
-
-    ThreadStateInternal(void* stack_start, pthread_t pthread_id, PyThreadState* public_thread_state)
-        : saved(false),
-          stack_start(stack_start),
-          pthread_id(pthread_id),
-          exc_info(NULL, NULL, NULL),
-          public_thread_state(public_thread_state) {}
-
-    void saveCurrent() {
-        assert(!saved);
-        getcontext(&ucontext);
-        saved = true;
-    }
-
-    void popCurrent() {
-        assert(saved);
-        saved = false;
-    }
-
-    bool isValid() { return saved; }
-
-    ucontext_t* getContext() { return &ucontext; }
-
-    void pushGenerator(BoxedGenerator* g, void* new_stack_start, void* old_stack_limit) {
-        previous_stacks.emplace_back(g, this->stack_start, old_stack_limit);
-        this->stack_start = new_stack_start;
-    }
-
-    void popGenerator() {
-        assert(previous_stacks.size());
-        StackInfo& stack = previous_stacks.back();
-        stack_start = stack.stack_start;
-        previous_stacks.pop_back();
-    }
-
-    void assertNoGenerators() { assert(previous_stacks.size() == 0); }
+ThreadStateInternal::ThreadStateInternal(void* stack_start, pthread_t pthread_id, PyThreadState* public_thread_state)
+    : saved(false),
+      stack_start(stack_start),
+      pthread_id(pthread_id),
+      unwind_state(UNWIND_STATE_NORMAL),
+      exc_info(NULL, NULL, NULL),
+      public_thread_state(public_thread_state) {
+}
 
-    void accept(gc::GCVisitor* v) {
-        auto pub_state = public_thread_state;
-        v->visitIf(pub_state->curexc_type);
-        v->visitIf(pub_state->curexc_value);
-        v->visitIf(pub_state->curexc_traceback);
-        v->visitIf(pub_state->dict);
+void ThreadStateInternal::accept(gc::GCVisitor* v) {
+    auto pub_state = public_thread_state;
+    v->visitIf(pub_state->curexc_type);
+    v->visitIf(pub_state->curexc_value);
+    v->visitIf(pub_state->curexc_traceback);
+    v->visitIf(pub_state->dict);
 
-        v->visitIf(exc_info.type);
-        v->visitIf(exc_info.value);
-        v->visitIf(exc_info.traceback);
+    v->visitIf(exc_info.type);
+    v->visitIf(exc_info.value);
+    v->visitIf(exc_info.traceback);
 
-        for (auto& stack_info : previous_stacks) {
-            v->visit(stack_info.next_generator);
+    for (auto& stack_info : previous_stacks) {
+        v->visit(stack_info.next_generator);
 #if STACK_GROWS_DOWN
-            v->visitPotentialRange((void**)stack_info.stack_limit, (void**)stack_info.stack_start);
+        v->visitPotentialRange((void**)stack_info.stack_limit, (void**)stack_info.stack_start);
 #else
-            v->visitPotentialRange((void**)stack_info.stack_start, (void**)stack_info.stack_limit);
+        v->visitPotentialRange((void**)stack_info.stack_start, (void**)stack_info.stack_limit);
 #endif
-        }
     }
-};
-static std::unordered_map<pthread_t, ThreadStateInternal*> current_threads;
-static __thread ThreadStateInternal* current_internal_thread_state = 0;
-
-void pushGenerator(BoxedGenerator* g, void* new_stack_start, void* old_stack_limit) {
-    assert(new_stack_start);
-    assert(old_stack_limit);
-    assert(current_internal_thread_state);
-    current_internal_thread_state->pushGenerator(g, new_stack_start, old_stack_limit);
 }
 
-void popGenerator() {
-    assert(current_internal_thread_state);
-    current_internal_thread_state->popGenerator();
-}
-
-ExcInfo* getExceptionFerry() {
-    return &current_internal_thread_state->exc_info;
-}
+static std::unordered_map<pthread_t, ThreadStateInternal*> current_threads;
+__thread ThreadStateInternal* ThreadStateInternal::current = 0;
 
 // These are guarded by threading_lock
 static int signals_waiting(0);
@@ -196,10 +120,10 @@ static void visitLocalStack(gc::GCVisitor* v) {
     assert(sizeof(registers) % 8 == 0);
     v->visitPotentialRange((void**)&registers, (void**)((&registers) + 1));
 
-    assert(current_internal_thread_state);
+    assert(ThreadStateInternal::current);
 #if STACK_GROWS_DOWN
     void* stack_low = getCurrentStackLimit();
-    void* stack_high = current_internal_thread_state->stack_start;
+    void* stack_high = ThreadStateInternal::current->stack_start;
 #else
     void* stack_low = current_thread_state->stack_start;
     void* stack_high = getCurrentStackLimit();
@@ -208,7 +132,7 @@ static void visitLocalStack(gc::GCVisitor* v) {
     assert(stack_low < stack_high);
     v->visitPotentialRange((void**)stack_low, (void**)stack_high);
 
-    current_internal_thread_state->accept(v);
+    ThreadStateInternal::current->accept(v);
 }
 
 void visitAllStacks(gc::GCVisitor* v) {
@@ -283,7 +207,7 @@ static void _thread_context_dump(int signum, siginfo_t* info, void* _context) {
         printf("old rip: 0x%lx\n", (intptr_t)context->uc_mcontext.gregs[REG_RIP]);
     }
 
-    assert(current_internal_thread_state == current_threads[tid]);
+    assert(ThreadStateInternal::current == current_threads[tid]);
     pushThreadState(current_threads[tid], context);
     signals_waiting--;
 }
@@ -323,8 +247,8 @@ static void* _thread_start(void* _arg) {
 #else
         void* stack_bottom = stack_start;
 #endif
-        current_internal_thread_state = new ThreadStateInternal(stack_bottom, current_thread, &cur_thread_state);
-        current_threads[current_thread] = current_internal_thread_state;
+        ThreadStateInternal::current = new ThreadStateInternal(stack_bottom, current_thread, &cur_thread_state);
+        current_threads[current_thread] = ThreadStateInternal::current;
 
         num_starting_threads--;
 
@@ -336,7 +260,7 @@ static void* _thread_start(void* _arg) {
     assert(!PyErr_Occurred());
 
     void* rtn = start_func(arg1, arg2, arg3);
-    current_internal_thread_state->assertNoGenerators();
+    ThreadStateInternal::current->assertNoGenerators();
 
     {
         LOCK_REGION(&threading_lock);
@@ -345,7 +269,7 @@ static void* _thread_start(void* _arg) {
         if (VERBOSITY() >= 2)
             printf("thread tid=%ld exited\n", current_thread);
     }
-    current_internal_thread_state = 0;
+    ThreadStateInternal::current = 0;
 
     return rtn;
 }
@@ -423,9 +347,9 @@ static void* find_stack() {
 void registerMainThread() {
     LOCK_REGION(&threading_lock);
 
-    assert(!current_internal_thread_state);
-    current_internal_thread_state = new ThreadStateInternal(find_stack(), pthread_self(), &cur_thread_state);
-    current_threads[pthread_self()] = current_internal_thread_state;
+    assert(!ThreadStateInternal::current);
+    ThreadStateInternal::current = new ThreadStateInternal(find_stack(), pthread_self(), &cur_thread_state);
+    current_threads[pthread_self()] = ThreadStateInternal::current;
 
     struct sigaction act;
     memset(&act, 0, sizeof(act));
@@ -441,8 +365,8 @@ void registerMainThread() {
 }
 
 void finishMainThread() {
-    assert(current_internal_thread_state);
-    current_internal_thread_state->assertNoGenerators();
+    assert(ThreadStateInternal::current);
+    ThreadStateInternal::current->assertNoGenerators();
 
     // TODO maybe this is the place to wait for non-daemon threads?
 }
@@ -462,8 +386,8 @@ extern "C" void beginAllowThreads() noexcept {
     {
         LOCK_REGION(&threading_lock);
 
-        assert(current_internal_thread_state);
-        current_internal_thread_state->saveCurrent();
+        assert(ThreadStateInternal::current);
+        ThreadStateInternal::current->saveCurrent();
     }
 }
 
@@ -471,8 +395,8 @@ extern "C" void endAllowThreads() noexcept {
     {
         LOCK_REGION(&threading_lock);
 
-        assert(current_internal_thread_state);
-        current_internal_thread_state->popCurrent();
+        assert(ThreadStateInternal::current);
+        ThreadStateInternal::current->popCurrent();
     }
 
 

src/core/threading.h

@@ -32,8 +32,108 @@ namespace gc {
 class GCVisitor;
 }
 
+// somewhat similar to CPython's WHY_* enum
+// UNWIND_STATE_NORMAL : == WHY_EXCEPTION.  we call it "NORMAL" since we often unwind due to things other than
+// exceptions (getGlobals, getLocals, etc)
+// UNWIND_STATE_RERAISE: same as NORMAL, except we are supposed to skip the first frame.
+// UNWIND_STATE_OSR    : The previous frame was an osr replacement for the next one, so we should skip it
+enum UnwindState { UNWIND_STATE_NORMAL = 0, UNWIND_STATE_RERAISE, UNWIND_STATE_OSR };
+
 namespace threading {
 
+class ThreadStateInternal {
+private:
+    bool saved;
+    ucontext_t ucontext;
+
+    void _popGenerator() {
+        assert(previous_stacks.size());
+        StackInfo& stack = previous_stacks.back();
+        stack_start = stack.stack_start;
+        previous_stacks.pop_back();
+    }
+    void _pushGenerator(BoxedGenerator* g, void* new_stack_start, void* old_stack_limit) {
+        previous_stacks.emplace_back(g, this->stack_start, old_stack_limit);
+        this->stack_start = new_stack_start;
+    }
+
+public:
+    static __thread ThreadStateInternal* current;
+
+    void* stack_start;
+
+    struct StackInfo {
+        BoxedGenerator* next_generator;
+        void* stack_start;
+        void* stack_limit;
+
+        StackInfo(BoxedGenerator* next_generator, void* stack_start, void* stack_limit)
+            : next_generator(next_generator), stack_start(stack_start), stack_limit(stack_limit) {
+#if STACK_GROWS_DOWN
+            assert(stack_start > stack_limit);
+            assert((char*)stack_start - (char*)stack_limit < (1L << 30));
+#else
+            assert(stack_start < stack_limit);
+            assert((char*)stack_limit - (char*)stack_start < (1L << 30));
+#endif
+        }
+    };
+
+    std::vector<StackInfo> previous_stacks;
+    pthread_t pthread_id;
+
+    UnwindState unwind_state;
+    ExcInfo exc_info;
+
+    PyThreadState* public_thread_state;
+
+    ThreadStateInternal(void* stack_start, pthread_t pthread_id, PyThreadState* public_thread_state);
+    void saveCurrent() {
+        assert(!saved);
+        getcontext(&ucontext);
+        saved = true;
+    }
+    void popCurrent() {
+        assert(saved);
+        saved = false;
+    }
+    bool isValid() const { return saved; }
+    ucontext_t* getContext() { return &ucontext; }
+    void assertNoGenerators() { assert(previous_stacks.size() == 0); }
+    void accept(gc::GCVisitor* v);
+
+    // Some hooks to keep track of the list of stacks that this thread has been using.
+    // Every time we switch to a new generator, we need to pass a reference to the generator
+    // itself (so we can access the registers it is saving), the location of the new stack, and
+    // where we stopped executing on the old stack.
+    inline static void pushGenerator(BoxedGenerator* g, void* new_stack_start, void* old_stack_limit) {
+        assert(new_stack_start);
+        assert(old_stack_limit);
+        assert(ThreadStateInternal::current);
+        current->_pushGenerator(g, new_stack_start, old_stack_limit);
+    }
+
+    inline static void popGenerator() {
+        assert(ThreadStateInternal::current);
+        current->_popGenerator();
+    }
+
+    inline static void setUnwindState(UnwindState state) {
+        assert(ThreadStateInternal::current);
+        current->unwind_state = state;
+    }
+
+    inline static UnwindState getUnwindState() {
+        assert(ThreadStateInternal::current);
+        return current->unwind_state;
+    }
+
+    static ExcInfo* getExceptionFerry() {
+        assert(ThreadStateInternal::current);
+        return &current->exc_info;
+    }
+};
+
 // Whether or not a second thread was ever started:
 bool threadWasStarted();
 
@@ -48,15 +148,6 @@ void finishMainThread();
 // stacks and thread-local PyThreadState objects
 void visitAllStacks(gc::GCVisitor* v);
 
-// Some hooks to keep track of the list of stacks that this thread has been using.
-// Every time we switch to a new generator, we need to pass a reference to the generator
-// itself (so we can access the registers it is saving), the location of the new stack, and
-// where we stopped executing on the old stack.
-void pushGenerator(BoxedGenerator* g, void* new_stack_start, void* old_stack_limit);
-void popGenerator();
-
-ExcInfo* getExceptionFerry();
-
 #ifndef THREADING_USE_GIL
 #define THREADING_USE_GIL 1
 #define THREADING_USE_GRWL 0

src/runtime/cxx_unwind.cpp

@@ -24,12 +24,12 @@
 #include "codegen/ast_interpreter.h" // interpreter_instr_addr
 #include "codegen/unwinding.h"       // getCFForAddress
 #include "core/ast.h"
-#include "core/stats.h"              // StatCounter
-#include "core/threading.h"          // for getExceptionFerry
-#include "core/types.h"              // for ExcInfo
-#include "core/util.h"               // Timer
-#include "runtime/generator.h"       // generatorEntry
-#include "runtime/traceback.h"       // BoxedTraceback::addLine
+#include "core/stats.h"        // StatCounter
+#include "core/threading.h"    // for getExceptionFerry
+#include "core/types.h"        // for ExcInfo
+#include "core/util.h"         // Timer
+#include "runtime/generator.h" // generatorEntry
+#include "runtime/traceback.h" // BoxedTraceback::addLine
 
 #define UNW_LOCAL_ONLY
 #include <libunwind.h>
@@ -493,7 +493,8 @@ static inline void unwind_loop(ExcInfo* exc_data) {
     unw_getcontext(&uc);
     unw_init_local(&cursor, &uc);
 
-    BoxedTraceback** tb_loc = reinterpret_cast<BoxedTraceback**>(&threading::getExceptionFerry()->traceback);
+    BoxedTraceback** tb_loc
+        = reinterpret_cast<BoxedTraceback**>(&threading::ThreadStateInternal::getExceptionFerry()->traceback);
 
     while (unw_step(&cursor) > 0) {
         unw_proc_info_t pip;
@@ -633,7 +634,7 @@ extern "C" void* __cxa_allocate_exception(size_t size) noexcept {
     // our exception info in curexc_*, and then unset these in __cxa_end_catch, then we'll wipe our exception info
     // during unwinding!
 
-    return pyston::threading::getExceptionFerry();
+    return pyston::threading::ThreadStateInternal::getExceptionFerry();
 }
 
 // Takes the value that resume() sent us in RAX, and returns a pointer to the exception object actually thrown. In our

src/runtime/generator.cpp

@@ -90,7 +90,7 @@ void generatorEntry(BoxedGenerator* g) {
         assert(g->cls == generator_cls);
         assert(g->function->cls == function_cls);
 
-        threading::pushGenerator(g, g->stack_begin, g->returnContext);
+        threading::ThreadStateInternal::pushGenerator(g, g->stack_begin, g->returnContext);
         try {
             RegisterHelper context_registerer(g, __builtin_frame_address(0));
 
@@ -107,7 +107,7 @@ void generatorEntry(BoxedGenerator* g) {
 
         // we returned from the body of the generator. next/send/throw will notify the caller
         g->entryExited = true;
-        threading::popGenerator();
+        threading::ThreadStateInternal::popGenerator();
     }
     swapContext(&g->context, g->returnContext, 0);
 }
@@ -264,9 +264,9 @@ extern "C" Box* yield(BoxedGenerator* obj, Box* value) {
     BoxedGenerator* self = static_cast<BoxedGenerator*>(obj);
     self->returnValue = value;
 
-    threading::popGenerator();
+    threading::ThreadStateInternal::popGenerator();
     swapContext(&self->context, self->returnContext, 0);
-    threading::pushGenerator(obj, obj->stack_begin, obj->returnContext);
+    threading::ThreadStateInternal::pushGenerator(obj, obj->stack_begin, obj->returnContext);
 
     // if the generator receives a exception from the caller we have to throw it
     if (self->exception.type) {

src/runtime/stacktrace.cpp

@@ -204,7 +204,8 @@ extern "C" void raise0() {
     if (exc_info->type == None)
         raiseExcHelper(TypeError, "exceptions must be old-style classes or derived from BaseException, not NoneType");
 
-    cur_thread_state.unwind_state = UNWIND_STATE_RERAISE;
+
+    threading::ThreadStateInternal::setUnwindState(UNWIND_STATE_RERAISE);
     raiseRaw(*exc_info);
 }
 
@@ -283,7 +284,8 @@ ExcInfo excInfoForRaise(Box* type, Box* value, Box* tb) {
 extern "C" void raise3(Box* arg0, Box* arg1, Box* arg2) {
     bool reraise = arg2 != NULL && arg2 != None;
     auto exc_info = excInfoForRaise(arg0, arg1, arg2);
-    cur_thread_state.unwind_state = reraise ? UNWIND_STATE_RERAISE : UNWIND_STATE_NORMAL;
+
+    threading::ThreadStateInternal::setUnwindState(reraise ? UNWIND_STATE_RERAISE : UNWIND_STATE_NORMAL);
     raiseRaw(exc_info);
 }