Tell Python includes we have threads and fix up the resulting issues

I was surprised that Py_BEGIN_ALLOW_THREADS was working, but it turns
out that we weren't enabling threading so it wasn't doing anything.

Makefile

@@ -245,6 +245,7 @@ ifneq ($(USE_CLANG),0)
 endif
 
 ifeq ($(USE_CCACHE),1)
+	CC := ccache $(CC)
 	CXX := ccache $(CXX)
 	CXX_PROFILE := ccache $(CXX_PROFILE)
 	CLANG_CXX := ccache $(CLANG_CXX)
@@ -308,7 +309,7 @@ NONSTDLIB_SRCS := $(MAIN_SRCS) $(OPTIONAL_SRCS) $(TOOL_SRCS) $(UNITTEST_SRCS)
 all: pyston_dbg pyston_release pyston_prof ext_python ext_pyston unittests
 
 ALL_HEADERS := $(wildcard src/*/*.h) $(wildcard src/*/*/*.h) $(wildcard ./include/*.h)
-tags: $(SRCS) $(OPTIONAL_SRCS) $(ALL_HEADERS)
+tags: $(SRCS) $(OPTIONAL_SRCS) $(FROM_CPYTHON_SRCS) $(ALL_HEADERS)
 	$(ECHO) Calculating tags...
 	$(VERB) ctags $^
 

include/ceval.h

@@ -129,12 +129,18 @@ PyAPI_FUNC(void) PyEval_AcquireThread(PyThreadState *tstate);
 PyAPI_FUNC(void) PyEval_ReleaseThread(PyThreadState *tstate);
 PyAPI_FUNC(void) PyEval_ReInitThreads(void);
 
+// Pyston change: add our internal API here that doesn't make reference to PyThreadState.
+// If anyone goes out of their way to use the PyThreadState* APIs directly, we should
+// fail instead of assuming that they didn't care about the PyThreadState.
+PyAPI_FUNC(void) beginAllowThreads(void);
+PyAPI_FUNC(void) endAllowThreads(void);
+
+// Pyston change: switch these to use our internal API
 #define Py_BEGIN_ALLOW_THREADS { \
-                        PyThreadState *_save; \
-                        _save = PyEval_SaveThread();
-#define Py_BLOCK_THREADS        PyEval_RestoreThread(_save);
-#define Py_UNBLOCK_THREADS      _save = PyEval_SaveThread();
-#define Py_END_ALLOW_THREADS    PyEval_RestoreThread(_save); \
+                        beginAllowThreads();
+#define Py_BLOCK_THREADS        endAllowThreads();
+#define Py_UNBLOCK_THREADS      beginAllowThreads();
+#define Py_END_ALLOW_THREADS    endAllowThreads(); \
                  }
 
 #else /* !WITH_THREAD */

include/pyconfig.h

@@ -37,6 +37,8 @@
 #define Py_USING_UNICODE 1
 #define Py_UNICODE_SIZE 4
 
+#define WITH_THREAD
+
 // Copied from a CPython ./configure run on a Linux machine:
 // TODO copy these more systematically
 #define HAVE_WAIT3 1

src/core/threading.cpp

@@ -21,6 +21,8 @@
 #include <sys/syscall.h>
 #include <unistd.h>
 
+#include "Python.h"
+
 #include "core/common.h"
 #include "core/options.h"
 #include "core/stats.h"
@@ -375,7 +377,7 @@ void finishMainThread() {
 // It adds some perf overhead I suppose, though I haven't measured it.
 // It also means that you're not allowed to do that much inside an AllowThreads region...
 // TODO maybe we should let the client decide which way to handle it
-GLAllowThreadsReadRegion::GLAllowThreadsReadRegion() {
+extern "C" void beginAllowThreads() {
     // I don't think it matters whether the GL release happens before or after the state
     // saving; do it before, then, to reduce the amount we hold the GL:
     releaseGLRead();
@@ -388,7 +390,7 @@ GLAllowThreadsReadRegion::GLAllowThreadsReadRegion() {
     }
 }
 
-GLAllowThreadsReadRegion::~GLAllowThreadsReadRegion() {
+extern "C" void endAllowThreads() {
     {
         LOCK_REGION(&threading_lock);
         current_threads[gettid()]->popCurrent();
@@ -398,7 +400,6 @@ GLAllowThreadsReadRegion::~GLAllowThreadsReadRegion() {
     acquireGLRead();
 }
 
-
 #if THREADING_USE_GIL
 #if THREADING_USE_GRWL
 #error "Can't turn on both the GIL and the GRWL!"

src/core/threading.h

@@ -114,10 +114,13 @@ MAKE_REGION(GLPromoteRegion, promoteGL, demoteGL);
 // MAKE_REGION(GLWriteReleaseRegion, releaseGLWrite, acquireGLWrite);
 #undef MAKE_REGION
 
+extern "C" void beginAllowThreads();
+extern "C" void endAllowThreads();
+
 class GLAllowThreadsReadRegion {
 public:
-    GLAllowThreadsReadRegion();
-    ~GLAllowThreadsReadRegion();
+    GLAllowThreadsReadRegion() { beginAllowThreads(); }
+    ~GLAllowThreadsReadRegion() { endAllowThreads(); }
 };
 
 

src/runtime/import.cpp

@@ -192,6 +192,15 @@ static Box* import(const std::string* name, bool return_first) {
     return return_first ? first_module : last_module;
 }
 
+extern "C" void _PyImport_AcquireLock() {
+    // TODO: currently no import lock!
+}
+
+extern "C" int _PyImport_ReleaseLock() {
+    // TODO: currently no import lock!
+    return 1;
+}
+
 extern "C" PyObject* PyImport_ImportModuleNoBlock(const char* name) {
     Py_FatalError("unimplemented");
 }