While investigating fork() behavior, found another threading bug

We weren't able to exit if we had a thread running in the background,
since we released the GIL at the end of the C level main() and the
background thread would start running again after all the data structures
had been torn down.

Now, just keep holding the GIL as we exit.

src/asm_writing/rewriter.h

@@ -20,6 +20,7 @@
 
 #include "asm_writing/assembler.h"
 #include "asm_writing/icinfo.h"
+#include "core/threading.h"
 
 namespace pyston {
 
@@ -432,7 +433,8 @@ public:
 
         // This check isn't thread safe and should be fine to remove if it causes
         // issues (along with the nvars/start_vars accounting)
-        ASSERT(RewriterVar::nvars == start_vars, "%d %d", RewriterVar::nvars, start_vars);
+        ASSERT(threading::threadWasStarted() || RewriterVar::nvars == start_vars, "%d %d", RewriterVar::nvars,
+               start_vars);
     }
 
     Location getReturnDestination();

src/core/threading.cpp

@@ -277,7 +277,14 @@ static void* _thread_start(void* _arg) {
     return rtn;
 }
 
+static bool thread_was_started = false;
+bool threadWasStarted() {
+    return thread_was_started;
+}
+
 intptr_t start_thread(void* (*start_func)(Box*, Box*, Box*), Box* arg1, Box* arg2, Box* arg3) {
+    thread_was_started = true;
+
     {
         LOCK_REGION(&threading_lock);
         num_starting_threads++;

src/core/threading.h

@@ -28,6 +28,8 @@ class Box;
 
 namespace threading {
 
+bool threadWasStarted();
+
 // returns a thread id (currently, the pthread_t id)
 intptr_t start_thread(void* (*start_func)(Box*, Box*, Box*), Box* arg1, Box* arg2, Box* arg3);
 

src/jit.cpp

@@ -93,7 +93,7 @@ int main(int argc, char** argv) {
     const char* fn = NULL;
 
     threading::registerMainThread();
-    threading::GLReadRegion _glock;
+    threading::acquireGLRead();
 
     {
         Timer _t("for initCodegen");
@@ -222,6 +222,11 @@ int main(int argc, char** argv) {
 
     threading::finishMainThread();
 
+    // Acquire the GIL to make sure we stop the other threads, since we will tear down
+    // data structures they are potentially running on.
+    // Note: we will purposefully not release the GIL on exiting.
+    threading::promoteGL();
+
     _t.split("joinRuntime");
 
     int rtncode = joinRuntime();

src/runtime/capi.cpp

@@ -915,7 +915,14 @@ Done:
 }
 
 extern "C" void PyOS_AfterFork(void) {
-    Py_FatalError("unimplemented");
+    // TODO CPython does a number of things after a fork:
+    // - clears pending signals
+    // - updates the cached "main_pid"
+    // - reinitialize and reacquire the GIL
+    // - reinitialize the import lock
+    // - change the definition of the main thread to the current thread
+    // - call threading._after_fork
+    // Also see PyEval_ReInitThreads
 }
 
 extern "C" {

test/tests/dangling_thread.py

+# Make sure that we can exit with threads running in the background.
+
+from thread import start_new_thread
+import time
+
+counter = 0
+def daemon_thread():
+    global counter
+    while True:
+        for i in xrange(100):
+            counter += 1
+        time.sleep(0.0)
+
+start_new_thread(daemon_thread, ())
+while counter < 100:
+    time.sleep(0.01)
+
+print "exiting"
+# exit without stopping the daemon thread