Merge branch 'zlib'

Makefile

@@ -290,7 +290,7 @@ SRCS := $(MAIN_SRCS) $(STDLIB_SRCS)
 STDLIB_OBJS := stdlib.bc.o stdlib.stripped.bc.o
 STDLIB_RELEASE_OBJS := stdlib.release.bc.o
 
-STDMODULE_SRCS := errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c $(EXTRA_STDMODULE_SRCS)
+STDMODULE_SRCS := errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c zlibmodule.c $(EXTRA_STDMODULE_SRCS)
 STDOBJECT_SRCS := structseq.c capsule.c stringobject.c $(EXTRA_STDOBJECT_SRCS)
 STDPYTHON_SRCS := pyctype.c getargs.c formatter_string.c pystrtod.c dtoa.c $(EXTRA_STDPYTHON_SRCS)
 FROM_CPYTHON_SRCS := $(addprefix from_cpython/Modules/,$(STDMODULE_SRCS)) $(addprefix from_cpython/Objects/,$(STDOBJECT_SRCS)) $(addprefix from_cpython/Python/,$(STDPYTHON_SRCS))

from_cpython/CMakeLists.txt

@@ -15,7 +15,7 @@ endforeach(STDLIB_FILE)
 add_custom_target(copy_stdlib ALL DEPENDS ${STDLIB_TARGETS})
 
 # compile specified files in from_cpython/Modules
-file(GLOB_RECURSE STDMODULE_SRCS Modules errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c)
+file(GLOB_RECURSE STDMODULE_SRCS Modules errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c zlibmodule.c)
 
 # compile specified files in from_cpython/Objects
 file(GLOB_RECURSE STDOBJECT_SRCS Objects structseq.c capsule.c stringobject.c)

from_cpython/Include/pyconfig.h

@@ -27,6 +27,7 @@
 #define SIZEOF_INT 4
 #define SIZEOF_LONG 8
 #define SIZEOF_LONG_LONG 8
+#define SIZEOF_PTHREAD_T 8
 #define HAVE_COPYSIGN 1
 #define HAVE_ROUND 1
 #define HAVE_HYPOT 1

src/capi/modsupport.cpp

@@ -283,6 +283,16 @@ extern "C" int PyModule_AddObject(PyObject* _m, const char* name, PyObject* valu
     return 0;
 }
 
+extern "C" int PyModule_AddStringConstant(PyObject* m, const char* name, const char* value) noexcept {
+    PyObject* o = PyString_FromString(value);
+    if (!o)
+        return -1;
+    if (PyModule_AddObject(m, name, o) == 0)
+        return 0;
+    Py_DECREF(o);
+    return -1;
+}
+
 extern "C" int PyModule_AddIntConstant(PyObject* _m, const char* name, long value) noexcept {
     return PyModule_AddObject(_m, name, boxInt(value));
 }

src/core/threading.cpp

@@ -613,10 +613,6 @@ void allowGLReadPreemption() {
 }
 #endif
 
-extern "C" long PyThread_get_thread_ident(void) noexcept {
-    return pthread_self();
-}
-
 // We don't support CPython's TLS (yet?)
 extern "C" void PyThread_ReInitTLS(void) noexcept {
     // don't have to do anything since we don't support TLS

src/runtime/builtin_modules/thread.cpp

@@ -12,8 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <pthread.h>
 #include <stddef.h>
 
+#include "Python.h"
+#include "pythread.h"
+
 #include "core/threading.h"
 #include "core/types.h"
 #include "runtime/objmodel.h"
@@ -21,6 +25,34 @@
 
 using namespace pyston::threading;
 
+
+static int initialized;
+static void PyThread__init_thread(void); /* Forward */
+
+extern "C" void PyThread_init_thread(void) noexcept {
+#ifdef Py_DEBUG
+    char* p = Py_GETENV("PYTHONTHREADDEBUG");
+
+    if (p) {
+        if (*p)
+            thread_debug = atoi(p);
+        else
+            thread_debug = 1;
+    }
+#endif /* Py_DEBUG */
+    if (initialized)
+        return;
+    initialized = 1;
+    PyThread__init_thread();
+}
+
+/* Support for runtime thread stack size tuning.
+   A value of 0 means using the platform's default stack size
+   or the size specified by the THREAD_STACK_SIZE macro. */
+static size_t _pythread_stacksize = 0;
+
+#include "thread_pthread.h"
+
 namespace pyston {
 
 BoxedModule* thread_module;

src/runtime/builtin_modules/thread_pthread.h

+// This file is originally from CPython 2.7, with modifications for Pyston
+
+/* Posix threads interface */
+
+#include <stdlib.h>
+#include <string.h>
+#if defined(__APPLE__) || defined(HAVE_PTHREAD_DESTRUCTOR)
+#define destructor xxdestructor
+#endif
+#include <pthread.h>
+#if defined(__APPLE__) || defined(HAVE_PTHREAD_DESTRUCTOR)
+#undef destructor
+#endif
+#include <signal.h>
+
+/* The POSIX spec requires that use of pthread_attr_setstacksize
+   be conditional on _POSIX_THREAD_ATTR_STACKSIZE being defined. */
+#ifdef _POSIX_THREAD_ATTR_STACKSIZE
+#ifndef THREAD_STACK_SIZE
+#define THREAD_STACK_SIZE 0 /* use default stack size */
+#endif
+
+#if (defined(__APPLE__) || defined(__FreeBSD__)) && defined(THREAD_STACK_SIZE) && THREAD_STACK_SIZE == 0
+/* The default stack size for new threads on OSX is small enough that
+ * we'll get hard crashes instead of 'maximum recursion depth exceeded'
+ * exceptions.
+ *
+ * The default stack size below is the minimal stack size where a
+ * simple recursive function doesn't cause a hard crash.
+ */
+#undef THREAD_STACK_SIZE
+#define THREAD_STACK_SIZE 0x400000
+#endif
+/* for safety, ensure a viable minimum stacksize */
+#define THREAD_STACK_MIN 0x8000 /* 32kB */
+#else                           /* !_POSIX_THREAD_ATTR_STACKSIZE */
+#ifdef THREAD_STACK_SIZE
+#error "THREAD_STACK_SIZE defined but _POSIX_THREAD_ATTR_STACKSIZE undefined"
+#endif
+#endif
+
+/* The POSIX spec says that implementations supporting the sem_*
+   family of functions must indicate this by defining
+   _POSIX_SEMAPHORES. */
+#ifdef _POSIX_SEMAPHORES
+/* On FreeBSD 4.x, _POSIX_SEMAPHORES is defined empty, so
+   we need to add 0 to make it work there as well. */
+#if (_POSIX_SEMAPHORES + 0) == -1
+#define HAVE_BROKEN_POSIX_SEMAPHORES
+#else
+#include <semaphore.h>
+#include <errno.h>
+#endif
+#endif
+
+/* Before FreeBSD 5.4, system scope threads was very limited resource
+   in default setting.  So the process scope is preferred to get
+   enough number of threads to work. */
+#ifdef __FreeBSD__
+#include <osreldate.h>
+#if __FreeBSD_version >= 500000 && __FreeBSD_version < 504101
+#undef PTHREAD_SYSTEM_SCHED_SUPPORTED
+#endif
+#endif
+
+#if !defined(pthread_attr_default)
+#define pthread_attr_default ((pthread_attr_t*)NULL)
+#endif
+#if !defined(pthread_mutexattr_default)
+#define pthread_mutexattr_default ((pthread_mutexattr_t*)NULL)
+#endif
+#if !defined(pthread_condattr_default)
+#define pthread_condattr_default ((pthread_condattr_t*)NULL)
+#endif
+
+
+/* Whether or not to use semaphores directly rather than emulating them with
+ * mutexes and condition variables:
+ */
+#if defined(_POSIX_SEMAPHORES) && !defined(HAVE_BROKEN_POSIX_SEMAPHORES)
+#define USE_SEMAPHORES
+#else
+#undef USE_SEMAPHORES
+#endif
+
+
+/* On platforms that don't use standard POSIX threads pthread_sigmask()
+ * isn't present.  DEC threads uses sigprocmask() instead as do most
+ * other UNIX International compliant systems that don't have the full
+ * pthread implementation.
+ */
+#if defined(HAVE_PTHREAD_SIGMASK) && !defined(HAVE_BROKEN_PTHREAD_SIGMASK)
+#define SET_THREAD_SIGMASK pthread_sigmask
+#else
+#define SET_THREAD_SIGMASK sigprocmask
+#endif
+
+
+/* A pthread mutex isn't sufficient to model the Python lock type
+ * because, according to Draft 5 of the docs (P1003.4a/D5), both of the
+ * following are undefined:
+ *  -> a thread tries to lock a mutex it already has locked
+ *  -> a thread tries to unlock a mutex locked by a different thread
+ * pthread mutexes are designed for serializing threads over short pieces
+ * of code anyway, so wouldn't be an appropriate implementation of
+ * Python's locks regardless.
+ *
+ * The pthread_lock struct implements a Python lock as a "locked?" bit
+ * and a <condition, mutex> pair.  In general, if the bit can be acquired
+ * instantly, it is, else the pair is used to block the thread until the
+ * bit is cleared.     9 May 1994 tim@ksr.com
+ */
+
+typedef struct {
+    char locked; /* 0=unlocked, 1=locked */
+    /* a <cond, mutex> pair to handle an acquire of a locked lock */
+    pthread_cond_t lock_released;
+    pthread_mutex_t mut;
+} pthread_lock;
+
+#define CHECK_STATUS(name)                                                                                             \
+    if (status != 0) {                                                                                                 \
+        perror(name);                                                                                                  \
+        error = 1;                                                                                                     \
+    }
+
+/*
+ * Initialization.
+ */
+
+#ifdef _HAVE_BSDI
+static void _noop(void) {
+}
+
+static void PyThread__init_thread(void) {
+    /* DO AN INIT BY STARTING THE THREAD */
+    static int dummy = 0;
+    pthread_t thread1;
+    pthread_create(&thread1, NULL, (void*)_noop, &dummy);
+    pthread_join(thread1, NULL);
+}
+
+#else /* !_HAVE_BSDI */
+
+static void PyThread__init_thread(void) {
+#if defined(_AIX) && defined(__GNUC__)
+    extern void pthread_init(void);
+    pthread_init();
+#endif
+}
+
+#endif /* !_HAVE_BSDI */
+
+/*
+ * Thread support.
+ */
+
+
+long PyThread_start_new_thread(void (*func)(void*), void* arg) noexcept {
+    pthread_t th;
+    int status;
+#if defined(THREAD_STACK_SIZE) || defined(PTHREAD_SYSTEM_SCHED_SUPPORTED)
+    pthread_attr_t attrs;
+#endif
+#if defined(THREAD_STACK_SIZE)
+    size_t tss;
+#endif
+
+    // dprintf(("PyThread_start_new_thread called\n"));
+    if (!initialized)
+        PyThread_init_thread();
+
+#if defined(THREAD_STACK_SIZE) || defined(PTHREAD_SYSTEM_SCHED_SUPPORTED)
+    if (pthread_attr_init(&attrs) != 0)
+        return -1;
+#endif
+#if defined(THREAD_STACK_SIZE)
+    tss = (_pythread_stacksize != 0) ? _pythread_stacksize : THREAD_STACK_SIZE;
+    if (tss != 0) {
+        if (pthread_attr_setstacksize(&attrs, tss) != 0) {
+            pthread_attr_destroy(&attrs);
+            return -1;
+        }
+    }
+#endif
+#if defined(PTHREAD_SYSTEM_SCHED_SUPPORTED)
+    pthread_attr_setscope(&attrs, PTHREAD_SCOPE_SYSTEM);
+#endif
+
+    status = pthread_create(&th,
+#if defined(THREAD_STACK_SIZE) || defined(PTHREAD_SYSTEM_SCHED_SUPPORTED)
+                            &attrs,
+#else
+                            (pthread_attr_t*)NULL,
+#endif
+                            (void* (*)(void*))func, (void*)arg);
+
+#if defined(THREAD_STACK_SIZE) || defined(PTHREAD_SYSTEM_SCHED_SUPPORTED)
+    pthread_attr_destroy(&attrs);
+#endif
+    if (status != 0)
+        return -1;
+
+    pthread_detach(th);
+
+#if SIZEOF_PTHREAD_T <= SIZEOF_LONG
+    return (long)th;
+#else
+    return (long)*(long*)&th;
+#endif
+}
+
+/* XXX This implementation is considered (to quote Tim Peters) "inherently
+   hosed" because:
+     - It does not guarantee the promise that a non-zero integer is returned.
+     - The cast to long is inherently unsafe.
+     - It is not clear that the 'volatile' (for AIX?) and ugly casting in the
+       latter return statement (for Alpha OSF/1) are any longer necessary.
+*/
+long PyThread_get_thread_ident(void) noexcept {
+    volatile pthread_t threadid;
+    if (!initialized)
+        PyThread_init_thread();
+    /* Jump through some hoops for Alpha OSF/1 */
+    threadid = pthread_self();
+#if SIZEOF_PTHREAD_T <= SIZEOF_LONG
+    return (long)threadid;
+#else
+    return (long)*(long*)&threadid;
+#endif
+}
+
+void PyThread_exit_thread(void) noexcept {
+    // dprintf(("PyThread_exit_thread called\n"));
+    if (!initialized) {
+        exit(0);
+    }
+}
+
+#ifdef USE_SEMAPHORES
+
+/*
+ * Lock support.
+ */
+
+PyThread_type_lock PyThread_allocate_lock(void) noexcept {
+    sem_t* lock;
+    int status, error = 0;
+
+    // dprintf(("PyThread_allocate_lock called\n"));
+    if (!initialized)
+        PyThread_init_thread();
+
+    lock = (sem_t*)malloc(sizeof(sem_t));
+
+    if (lock) {
+        status = sem_init(lock, 0, 1);
+        CHECK_STATUS("sem_init");
+
+        if (error) {
+            free((void*)lock);
+            lock = NULL;
+        }
+    }
+
+    // dprintf(("PyThread_allocate_lock() -> %p\n", lock));
+    return (PyThread_type_lock)lock;
+}
+
+void PyThread_free_lock(PyThread_type_lock lock) noexcept {
+    sem_t* thelock = (sem_t*)lock;
+    int status, error = 0;
+
+    (void)error; /* silence unused-but-set-variable warning */
+    // dprintf(("PyThread_free_lock(%p) called\n", lock));
+
+    if (!thelock)
+        return;
+
+    status = sem_destroy(thelock);
+    CHECK_STATUS("sem_destroy");
+
+    free((void*)thelock);
+}
+
+/*
+ * As of February 2002, Cygwin thread implementations mistakenly report error
+ * codes in the return value of the sem_ calls (like the pthread_ functions).
+ * Correct implementations return -1 and put the code in errno. This supports
+ * either.
+ */
+static int fix_status(int status) {
+    return (status == -1) ? errno : status;
+}
+
+int PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) noexcept {
+    int success;
+    sem_t* thelock = (sem_t*)lock;
+    int status, error = 0;
+
+    (void)error; /* silence unused-but-set-variable warning */
+    // dprintf(("PyThread_acquire_lock(%p, %d) called\n", lock, waitflag));
+
+    do {
+        if (waitflag)
+            status = fix_status(sem_wait(thelock));
+        else
+            status = fix_status(sem_trywait(thelock));
+    } while (status == EINTR); /* Retry if interrupted by a signal */
+
+    if (waitflag) {
+        CHECK_STATUS("sem_wait");
+    } else if (status != EAGAIN) {
+        CHECK_STATUS("sem_trywait");
+    }
+
+    success = (status == 0) ? 1 : 0;
+
+    // dprintf(("PyThread_acquire_lock(%p, %d) -> %d\n", lock, waitflag, success));
+    return success;
+}
+
+void PyThread_release_lock(PyThread_type_lock lock) noexcept {
+    sem_t* thelock = (sem_t*)lock;
+    int status, error = 0;
+
+    (void)error; /* silence unused-but-set-variable warning */
+    // dprintf(("PyThread_release_lock(%p) called\n", lock));
+
+    status = sem_post(thelock);
+    CHECK_STATUS("sem_post");
+}
+
+#else /* USE_SEMAPHORES */
+
+/*
+ * Lock support.
+ */
+PyThread_type_lock PyThread_allocate_lock(void) noexcept {
+    pthread_lock* lock;
+    int status, error = 0;
+
+    // dprintf(("PyThread_allocate_lock called\n"));
+    if (!initialized)
+        PyThread_init_thread();
+
+    lock = (pthread_lock*)malloc(sizeof(pthread_lock));
+    if (lock) {
+        memset((void*)lock, '\0', sizeof(pthread_lock));
+        lock->locked = 0;
+
+        status = pthread_mutex_init(&lock->mut, pthread_mutexattr_default);
+        CHECK_STATUS("pthread_mutex_init");
+
+        status = pthread_cond_init(&lock->lock_released, pthread_condattr_default);
+        CHECK_STATUS("pthread_cond_init");
+
+        if (error) {
+            free((void*)lock);
+            lock = 0;
+        }
+    }
+
+    // dprintf(("PyThread_allocate_lock() -> %p\n", lock));
+    return (PyThread_type_lock)lock;
+}
+
+void PyThread_free_lock(PyThread_type_lock lock) noexcept {
+    pthread_lock* thelock = (pthread_lock*)lock;
+    int status, error = 0;
+
+    (void)error; /* silence unused-but-set-variable warning */
+    // dprintf(("PyThread_free_lock(%p) called\n", lock));
+
+    status = pthread_mutex_destroy(&thelock->mut);
+    CHECK_STATUS("pthread_mutex_destroy");
+
+    status = pthread_cond_destroy(&thelock->lock_released);
+    CHECK_STATUS("pthread_cond_destroy");
+
+    free((void*)thelock);
+}
+
+int PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) noexcept {
+    int success;
+    pthread_lock* thelock = (pthread_lock*)lock;
+    int status, error = 0;
+
+    // dprintf(("PyThread_acquire_lock(%p, %d) called\n", lock, waitflag));
+
+    status = pthread_mutex_lock(&thelock->mut);
+    CHECK_STATUS("pthread_mutex_lock[1]");
+    success = thelock->locked == 0;
+
+    if (!success && waitflag) {
+        /* continue trying until we get the lock */
+
+        /* mut must be locked by me -- part of the condition
+         * protocol */
+        while (thelock->locked) {
+            status = pthread_cond_wait(&thelock->lock_released, &thelock->mut);
+            CHECK_STATUS("pthread_cond_wait");
+        }
+        success = 1;
+    }
+    if (success)
+        thelock->locked = 1;
+    status = pthread_mutex_unlock(&thelock->mut);
+    CHECK_STATUS("pthread_mutex_unlock[1]");
+
+    if (error)
+        success = 0;
+    // dprintf(("PyThread_acquire_lock(%p, %d) -> %d\n", lock, waitflag, success));
+    return success;
+}
+
+void PyThread_release_lock(PyThread_type_lock lock) noexcept {
+    pthread_lock* thelock = (pthread_lock*)lock;
+    int status, error = 0;
+
+    (void)error; /* silence unused-but-set-variable warning */
+    // dprintf(("PyThread_release_lock(%p) called\n", lock));
+
+    status = pthread_mutex_lock(&thelock->mut);
+    CHECK_STATUS("pthread_mutex_lock[3]");
+
+    thelock->locked = 0;
+
+    status = pthread_mutex_unlock(&thelock->mut);
+    CHECK_STATUS("pthread_mutex_unlock[3]");
+
+    /* wake up someone (anyone, if any) waiting on the lock */
+    status = pthread_cond_signal(&thelock->lock_released);
+    CHECK_STATUS("pthread_cond_signal");
+}
+
+#endif /* USE_SEMAPHORES */
+
+/* set the thread stack size.
+ * Return 0 if size is valid, -1 if size is invalid,
+ * -2 if setting stack size is not supported.
+ */
+static int _pythread_pthread_set_stacksize(size_t size) {
+#if defined(THREAD_STACK_SIZE)
+    pthread_attr_t attrs;
+    size_t tss_min;
+    int rc = 0;
+#endif
+
+    /* set to default */
+    if (size == 0) {
+        _pythread_stacksize = 0;
+        return 0;
+    }
+
+#if defined(THREAD_STACK_SIZE)
+#if defined(PTHREAD_STACK_MIN)
+    tss_min = PTHREAD_STACK_MIN > THREAD_STACK_MIN ? PTHREAD_STACK_MIN : THREAD_STACK_MIN;
+#else
+    tss_min = THREAD_STACK_MIN;
+#endif
+    if (size >= tss_min) {
+        /* validate stack size by setting thread attribute */
+        if (pthread_attr_init(&attrs) == 0) {
+            rc = pthread_attr_setstacksize(&attrs, size);
+            pthread_attr_destroy(&attrs);
+            if (rc == 0) {
+                _pythread_stacksize = size;
+                return 0;
+            }
+        }
+    }
+    return -1;
+#else
+    return -2;
+#endif
+}
+
+#define THREAD_SET_STACKSIZE(x) _pythread_pthread_set_stacksize(x)

src/runtime/types.cpp

@@ -61,6 +61,7 @@ extern "C" void initselect();
 extern "C" void initfcntl();
 extern "C" void inittime();
 extern "C" void initarray();
+extern "C" void initzlib();
 
 namespace pyston {
 
@@ -1321,6 +1322,7 @@ void setupRuntime() {
     initfcntl();
     inittime();
     initarray();
+    initzlib();
 
     setupSysEnd();
 

test/tests/zlib_test.py

+import zlib
+
+s = "hello world!"
+s2 = zlib.compress(s)
+print repr(s2)
+s3 = zlib.decompress(s2)
+print repr(s3)

tools/lint.py

@@ -2,7 +2,13 @@ import os
 import sys
 
 def file_is_from_cpython(fn):
-    return 'from_cpython' in fn
+    if 'from_cpython' in fn:
+        return True
+
+    if fn.endswith("/thread_pthread.h"):
+        return True
+
+    return False
 
 def verify_include_guard(_, dir, files):
     for bn in files: