Merge pull request #1120 from undingen/threading_fixes

Threading and multiprocessing related fixes

from_cpython/Include/abstract.h

@@ -1359,7 +1359,7 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
        */
 #define PyMapping_Items(O) PyObject_CallMethod(O,"items",NULL)
 
-     PyAPI_FUNC(PyObject *) PyMapping_GetItemString(PyObject *o, char *key) PYSTON_NOEXCEPT;
+     PyAPI_FUNC(PyObject *) PyMapping_GetItemString(PyObject *o, const char *key) PYSTON_NOEXCEPT;
 
        /*
      Return element of o corresponding to the object, key, or NULL
@@ -1367,7 +1367,7 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
      o[key].
        */
 
-     PyAPI_FUNC(int) PyMapping_SetItemString(PyObject *o, char *key,
+     PyAPI_FUNC(int) PyMapping_SetItemString(PyObject *o, const char *key,
                                             PyObject *value) PYSTON_NOEXCEPT;
 
        /*

from_cpython/Include/pyerrors.h

@@ -249,7 +249,7 @@ PyAPI_FUNC(void) _PyErr_BadInternalCall(const char *filename, int lineno) PYSTON
 
 /* Function to create a new exception */
 PyAPI_FUNC(PyObject *) PyErr_NewException(
-    char *name, PyObject *base, PyObject *dict) PYSTON_NOEXCEPT;
+    const char *name, PyObject *base, PyObject *dict) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) PyErr_NewExceptionWithDoc(
     char *name, char *doc, PyObject *base, PyObject *dict) PYSTON_NOEXCEPT;
 PyAPI_FUNC(void) PyErr_WriteUnraisable(PyObject *) PYSTON_NOEXCEPT;

from_cpython/Python/errors.c

@@ -551,7 +551,7 @@ PyErr_Format(PyObject *exception, const char *format, ...)
 
 
 PyObject *
-PyErr_NewException(char *name, PyObject *base, PyObject *dict)
+PyErr_NewException(const char *name, PyObject *base, PyObject *dict)
 {
     char *dot;
     PyObject *modulename = NULL;

src/capi/abstract.cpp

@@ -1804,7 +1804,7 @@ extern "C" int PyMapping_HasKey(PyObject* o, PyObject* key) noexcept {
     return 0;
 }
 
-extern "C" PyObject* PyMapping_GetItemString(PyObject* o, char* key) noexcept {
+extern "C" PyObject* PyMapping_GetItemString(PyObject* o, const char* key) noexcept {
     PyObject* okey, *r;
 
     if (key == NULL)
@@ -1818,7 +1818,7 @@ extern "C" PyObject* PyMapping_GetItemString(PyObject* o, char* key) noexcept {
     return r;
 }
 
-extern "C" int PyMapping_SetItemString(PyObject* o, char* key, PyObject* value) noexcept {
+extern "C" int PyMapping_SetItemString(PyObject* o, const char* key, PyObject* value) noexcept {
     PyObject* okey;
     int r;
 

src/capi/types.h

@@ -41,7 +41,10 @@ public:
 
     static BoxedString* __repr__(BoxedCApiFunction* self) {
         assert(self->cls == capifunc_cls);
-        return boxString(self->method_def->ml_name);
+        if (self->passthrough == NULL)
+            return (BoxedString*)PyString_FromFormat("<built-in function %s>", self->method_def->ml_name);
+        return (BoxedString*)PyString_FromFormat("<built-in method %s of %s object at %p>", self->method_def->ml_name,
+                                                 self->passthrough->cls->tp_name, self->passthrough);
     }
 
     static Box* __call__(BoxedCApiFunction* self, BoxedTuple* varargs, BoxedDict* kwargs);

src/core/threading.cpp

@@ -522,11 +522,34 @@ void registerMainThread() {
     assert(!PyErr_Occurred());
 }
 
+/* Wait until threading._shutdown completes, provided
+   the threading module was imported in the first place.
+   The shutdown routine will wait until all non-daemon
+   "threading" threads have completed. */
+static void wait_for_thread_shutdown(void) noexcept {
+#ifdef WITH_THREAD
+    PyObject* result;
+    PyThreadState* tstate = PyThreadState_GET();
+    PyObject* threading = PyMapping_GetItemString(getSysModulesDict(), "threading");
+    if (threading == NULL) {
+        /* threading not imported */
+        PyErr_Clear();
+        return;
+    }
+    result = PyObject_CallMethod(threading, "_shutdown", "");
+    if (result == NULL)
+        PyErr_WriteUnraisable(threading);
+    else
+        Py_DECREF(result);
+    Py_DECREF(threading);
+#endif
+}
+
 void finishMainThread() {
     assert(current_internal_thread_state);
     current_internal_thread_state->assertNoGenerators();
 
-    // TODO maybe this is the place to wait for non-daemon threads?
+    wait_for_thread_shutdown();
 }
 
 bool isMainThread() {
@@ -601,6 +624,22 @@ extern "C" void PyEval_ReInitThreads() noexcept {
     threads_waiting_on_gil = 0;
 
     PerThreadSetBase::runAllForkHandlers();
+
+    /* Update the threading module with the new state.
+     */
+    Box* threading = PyMapping_GetItemString(getSysModulesDict(), "threading");
+    if (threading == NULL) {
+        /* threading not imported */
+        PyErr_Clear();
+        return;
+    }
+    Box* result = PyObject_CallMethod(threading, "_after_fork", NULL);
+    if (result == NULL)
+        PyErr_WriteUnraisable(threading);
+    else
+        Py_DECREF(result);
+    Py_DECREF(threading);
+
 }
 
 void acquireGLWrite() {

src/runtime/builtin_modules/builtins.cpp

@@ -2540,7 +2540,7 @@ void setupBuiltins() {
         { "reload", builtin_reload, METH_O, reload_doc },
     };
     for (auto& md : builtin_methods) {
-        builtins_module->giveAttr(md.ml_name, new BoxedCApiFunction(&md, builtins_module, boxString("__builtin__")));
+        builtins_module->giveAttr(md.ml_name, new BoxedCApiFunction(&md, NULL, boxString("__builtin__")));
     }
 }
 }

src/runtime/builtin_modules/sys.cpp

@@ -67,9 +67,6 @@ Box* sysExcClear() {
 static Box* sysExit(Box* arg) {
     assert(arg);
     Box* exc = runtimeCall(SystemExit, ArgPassSpec(1), arg, NULL, NULL, NULL, NULL);
-    // TODO this should be handled by the SystemExit constructor
-    exc->giveAttr("code", arg);
-
     raiseExc(exc);
 }
 
@@ -728,7 +725,7 @@ void setupSys() {
     sys_flags_cls->freeze();
 
     for (auto& md : sys_methods) {
-        sys_module->giveAttr(md.ml_name, new BoxedCApiFunction(&md, sys_module, boxString("sys")));
+        sys_module->giveAttr(md.ml_name, new BoxedCApiFunction(&md, NULL, boxString("sys")));
     }
 
     sys_module->giveAttr("__displayhook__", sys_module->getattr(internStringMortal("displayhook")));

src/runtime/builtin_modules/thread.cpp

@@ -90,7 +90,7 @@ static void* thread_start(Box* target, Box* varargs, Box* kwargs) {
 // TODO this should take kwargs, which defaults to empty
 Box* startNewThread(Box* target, Box* args, Box* kw) {
     intptr_t thread_id = start_thread(&thread_start, target, args, kw);
-    return boxInt(thread_id ^ 0x12345678901L);
+    return boxInt(thread_id);
 }
 
 #define CHECK_STATUS(name)                                                                                             \
@@ -244,11 +244,7 @@ void setupThread() {
     thread_lock_cls->giveAttr("locked_lock", thread_lock_cls->getattr(internStringMortal("locked")));
     thread_lock_cls->freeze();
 
-    ThreadError = BoxedClass::create(type_cls, Exception, NULL, Exception->attrs_offset, Exception->tp_weaklistoffset,
-                                     Exception->tp_basicsize, false, "error");
-    ThreadError->giveAttr("__module__", boxString("thread"));
-    ThreadError->freeze();
-
+    ThreadError = (BoxedClass*)PyErr_NewException("thread.error", NULL, NULL);
     thread_module->giveAttr("error", ThreadError);
 }
 }

test/tests/cPickle_test.py

@@ -46,3 +46,6 @@ for obj in [(1, 2), "hello world", u"hola world", 1.0, 1j, 1L, 2, {1:2}, set([3]
 import cStringIO
 StringIO = cPickle.loads(cPickle.dumps(cStringIO.StringIO))
 print type(StringIO())
+
+import thread
+print cPickle.loads(cPickle.dumps(thread.error))

test/tests/compile_eval.py

+print compile
 c = compile("a", "test.py", "eval")
 print type(c), c.co_filename, c.co_name
 

test/tests/dict_setting.py

@@ -51,6 +51,19 @@ print hasattr(c, "attr")
 print hasattr(c, "foo")
 print c.__dict__ is d1
 
+c = C()
+a = c.__dict__
+c.__dict__ = c.__dict__
+c.__dict__[u"uni"] = "u"
+c.__dict__[u"\u20ac"] = "u"
+c.__dict__[(1, 2, 3)] = "t"
+print sorted(c.__dict__.keys()), sorted(a.keys())
+print "all non str attributes:", sorted([item for item in dir(c) if not isinstance(item, str)])
+print "can we retrieve unicode attributes by there ascii value?", c.uni, c.__dict__["uni"]
+print "attrwrapper:", a["uni"], a[(1, 2, 3)]
+setattr(c, "uni", 1)
+print "test setattr()", c.uni, c.__dict__["uni"], a["uni"], len(c.__dict__), len(a)
+
 dictproxy = C.__dict__
 print type(dictproxy)
 print "foo" in dictproxy

test/tests/dict_setting_old_dict.py

-# expected: fail
-# - haven't bothered to implement this yet
-
 # Funny case: if we get the attrwrapper of an object, then change it to be dict-backed,
 # the original attrwrapper should remain valid but no longer connected to that object.
 
@@ -13,3 +10,10 @@ c1.a = 1
 print aw.items()
 c1.__dict__ = d = {}
 print aw.items()
+
+c2 = C()
+olddict = c2.__dict__
+c2.__dict__ = { "should not be there" : 1 }
+print olddict.has_key("should not be there")
+
+

test/tests/sys_test.py

 import sys
 import os.path
 
+print sys.excepthook, sys.displayhook
 print sys.version[:3]
 print os.path.exists(sys.executable)
 print sys.copyright[-200:]