add the cPickle module

the module is much faster than the python implementation
But I had to change the code a little bit because of our GC and more importantly because some of our types have different names etc...
I also noticed that we failed to to set capifunc.__module__ in a lot of cases which made pickle error

from_cpython/CMakeLists.txt

@@ -177,6 +177,7 @@ add_custom_command(OUTPUT ${STDMODULES}
 		       Modules/_cursesmodule.c
 		       Modules/mmapmodule.c
 		       Modules/_localemodule.c
+		       Modules/cPickle.c
                    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
 add_custom_target(sharedmods ALL DEPENDS ${CMAKE_BINARY_DIR}/from_cpython/Lib/_multiprocessing.pyston.so)
 

from_cpython/Include/classobject.h

@@ -58,6 +58,10 @@ PyAPI_DATA(PyTypeObject*) instancemethod_cls;
 #define PyMethod_Check(op) (Py_TYPE(op) == &PyMethod_Type)
 
 PyAPI_FUNC(PyObject *) PyClass_New(PyObject *, PyObject *, PyObject *) PYSTON_NOEXCEPT;
+
+// Pyston change: pyston addition returns PyClassObject->cl_name
+PyAPI_FUNC(PyObject *) PyClass_Name(PyObject *) PYSTON_NOEXCEPT;
+
 PyAPI_FUNC(PyObject *) PyInstance_New(PyObject *, PyObject *,
                                             PyObject *) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) PyInstance_NewRaw(PyObject *, PyObject *) PYSTON_NOEXCEPT;

from_cpython/Include/dictobject.h

@@ -110,6 +110,8 @@ PyAPI_DATA(PyTypeObject) PyDictValues_Type;
 
 PyAPI_DATA(PyTypeObject*) dict_cls;
 #define PyDict_Type (*dict_cls)
+PyAPI_DATA(PyTypeObject*) attrwrapper_cls;
+#define PyAttrWrapper_Type (*attrwrapper_cls)
 PyAPI_DATA(PyTypeObject*) dictiterkey_cls;
 #define PyDictIterKey_Type (*dictiterkey_cls)
 PyAPI_DATA(PyTypeObject*) dictitervalue_cls;

from_cpython/Include/methodobject.h

@@ -19,6 +19,10 @@ PyAPI_DATA(PyTypeObject*) capifunc_cls;
 
 #define PyCFunction_Check(op) (Py_TYPE(op) == &PyCFunction_Type)
 
+// Pyston change: expose our builtin_function_or_method type
+PyAPI_DATA(PyTypeObject*) builtin_function_or_method_cls;
+#define PyBuiltinFunction_Type (*builtin_function_or_method_cls)
+
 typedef PyObject *(*PyCFunction)(PyObject *, PyObject *);
 typedef PyObject *(*PyCFunctionWithKeywords)(PyObject *, PyObject *,
 					     PyObject *);

from_cpython/Lib/test/pickletester.py

@@ -1123,6 +1123,8 @@ class AbstractPickleModuleTests(unittest.TestCase):
         s = StringIO.StringIO("X''.")
         self.assertRaises(EOFError, self.module.load, s)
 
+    # Pyston change:
+    @unittest.skip("pyston does not support this currently")
     def test_restricted(self):
         # issue7128: cPickle failed in restricted mode
         builtins = {self.module.__name__: self.module,

from_cpython/Lib/test/test_cookie.py

-# expected: fail
 # Simple test suite for Cookie.py
 
 from test.test_support import run_unittest, run_doctest, check_warnings

from_cpython/Lib/test/test_cpickle.py

-# expected: fail
 import cPickle
 import cStringIO
 import io

from_cpython/Lib/test/test_pickle.py

-# expected: fail
 import pickle
 from cStringIO import StringIO
 

from_cpython/Lib/test/test_pickletools.py

-# expected: fail
 import pickle
 import pickletools
 from test import test_support

from_cpython/Modules/cPickle.c

@@ -144,6 +144,8 @@ typedef struct {
     PyObject **data;
 } Pdata;
 
+// Pyston change: use GC allocs instead of malloc
+#if 0
 static void
 Pdata_dealloc(Pdata *self)
 {
@@ -157,10 +159,13 @@ Pdata_dealloc(Pdata *self)
         free(self->data);
     PyObject_Del(self);
 }
+#endif
 
 static PyTypeObject PdataType = {
     PyVarObject_HEAD_INIT(NULL, 0) "cPickle.Pdata", sizeof(Pdata), 0,
-    (destructor)Pdata_dealloc,
+    // Pyston change: use GC allocs instead of malloc
+    // (destructor)Pdata_dealloc,
+    (destructor)0,
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0L,0L,0L,0L, ""
 };
 
@@ -175,7 +180,11 @@ Pdata_New(void)
         return NULL;
     self->size = 8;
     self->length = 0;
-    self->data = malloc(self->size * sizeof(PyObject*));
+
+    // Pyston change: use GC allocs instead of malloc
+    // self->data = malloc(self->size * sizeof(PyObject*));
+    self->data = PyMem_Malloc(self->size * sizeof(PyObject*));
+
     if (self->data)
         return (PyObject*)self;
     Py_DECREF(self);
@@ -225,7 +234,11 @@ Pdata_grow(Pdata *self)
     if (bigger > (PY_SSIZE_T_MAX / sizeof(PyObject *)))
         goto nomemory;
     nbytes = bigger * sizeof(PyObject *);
-    tmp = realloc(self->data, nbytes);
+
+    // Pyston change: use GC allocs instead of malloc
+    // tmp = realloc(self->data, nbytes);
+    tmp = PyMem_Realloc(self->data, nbytes);
+
     if (tmp == NULL)
         goto nomemory;
     self->data = tmp;
@@ -322,11 +335,17 @@ Pdata_popList(Pdata *self, Py_ssize_t start)
   }                                                 \
 }
 
+// Pyston change:
+/*
 #define FREE_ARG_TUP(self) {                        \
     if (Py_REFCNT(self->arg) > 1) {                 \
       Py_CLEAR(self->arg);                          \
     }                                               \
   }
+*/
+#define FREE_ARG_TUP(self) {                        \
+    Py_CLEAR(self->arg);                            \
+  }
 
 typedef struct Picklerobject {
     PyObject_HEAD
@@ -531,7 +550,9 @@ read_file(Unpicklerobject *self, char **s, Py_ssize_t n)
         Py_ssize_t size;
 
         size = ((n < 32) ? 32 : n);
-        if (!( self->buf = (char *)malloc(size))) {
+        // Pyston change: use GC allocs instead of malloc
+        // if (!( self->buf = (char *)malloc(size))) {
+        if (!( self->buf = (char *)PyMem_Malloc(size))) {
             PyErr_NoMemory();
             return -1;
         }
@@ -539,7 +560,9 @@ read_file(Unpicklerobject *self, char **s, Py_ssize_t n)
         self->buf_size = size;
     }
     else if (n > self->buf_size) {
-        char *newbuf = (char *)realloc(self->buf, n);
+        // Pyston change: use GC allocs instead of malloc
+        // char *newbuf = (char *)realloc(self->buf, n);
+        char *newbuf = (char *)PyMem_Realloc(self->buf, n);
         if (!newbuf)  {
             PyErr_NoMemory();
             return -1;
@@ -575,7 +598,9 @@ readline_file(Unpicklerobject *self, char **s)
     Py_ssize_t i;
 
     if (self->buf_size == 0) {
-        if (!( self->buf = (char *)malloc(40))) {
+        // Pyston change: use GC allocs instead of malloc
+        // if (!( self->buf = (char *)malloc(40))) {
+        if (!( self->buf = (char *)PyMem_Malloc(40))) {
             PyErr_NoMemory();
             return -1;
         }
@@ -599,7 +624,9 @@ readline_file(Unpicklerobject *self, char **s)
             return -1;
         }
         bigger = self->buf_size << 1;
-        newbuf = (char *)realloc(self->buf, bigger);
+        // Pyston change: use GC allocs instead of malloc
+        // newbuf = (char *)realloc(self->buf, bigger);
+        newbuf = (char *)PyMem_Realloc(self->buf, bigger);
         if (newbuf == NULL)  {
             PyErr_NoMemory();
             return -1;
@@ -800,7 +827,9 @@ get(Picklerobject *self, PyObject *id)
 static int
 put(Picklerobject *self, PyObject *ob)
 {
-    if (Py_REFCNT(ob) < 2 || self->fast)
+    // Pyston change:
+    // if (Py_REFCNT(ob) < 2 || self->fast)
+    if (/*Py_REFCNT(ob) < 2 ||*/ self->fast)
         return 0;
 
     return put2(self, ob);
@@ -2087,7 +2116,9 @@ save_inst(Picklerobject *self, PyObject *args)
     }
 
     if (!self->bin) {
-        if (!( name = ((PyClassObject *)class)->cl_name ))  {
+        // Pyston change:
+        // if (!( name = ((PyClassObject *)class)->cl_name ))  {
+        if (!( name = PyClass_Name(class)))  {
             PyErr_SetString(PicklingError, "class has no name");
             goto finally;
         }
@@ -2634,7 +2665,9 @@ save(Picklerobject *self, PyObject *args, int pers_save)
 #endif
     }
 
-    if (Py_REFCNT(args) > 1) {
+    // Pyston change:
+    // if (Py_REFCNT(args) > 1) {
+    if (/*Py_REFCNT(args) >*/ 1) {
         if (!( py_ob_id = PyLong_FromVoidPtr(args)))
             goto finally;
 
@@ -2689,6 +2722,14 @@ save(Picklerobject *self, PyObject *args, int pers_save)
         }
         break;
 
+    // Pyston change: add support for attrwrapper
+    case 'a':
+        if (type == &PyAttrWrapper_Type) {
+            res = save_dict(self, args);
+            goto finally;
+        }
+        break;
+
     case 'i':
         if (type == &PyInstance_Type) {
             res = save_inst(self, args);
@@ -2701,6 +2742,12 @@ save(Picklerobject *self, PyObject *args, int pers_save)
             res = save_global(self, args, NULL);
             goto finally;
         }
+
+        // Pyston change: moved because we call this type 'capifunc'
+        if (type == &PyCFunction_Type) {
+            res = save_global(self, args, NULL);
+            goto finally;
+        }
         break;
 
     case 'f':
@@ -2716,10 +2763,18 @@ save(Picklerobject *self, PyObject *args, int pers_save)
         break;
 
     case 'b':
+        // Pyston change: moved because we call this type 'capifunc'
+        /*
         if (type == &PyCFunction_Type) {
             res = save_global(self, args, NULL);
             goto finally;
         }
+        */
+        // Pyston change: added handler for 'builtin_function_or_method'
+        if (type == &PyBuiltinFunction_Type) {
+            res = save_global(self, args, NULL);
+            goto finally;
+        }
     }
 
     if (!pers_save && self->inst_pers_func) {
@@ -3144,7 +3199,8 @@ get_Pickler(PyObject *self, PyObject *args, PyObject *kwds)
     return (PyObject *)newPicklerobject(file, proto);
 }
 
-
+// Pyston change: use GC allocs instead of malloc
+#if 0
 static void
 Pickler_dealloc(Picklerobject *self)
 {
@@ -3160,6 +3216,7 @@ Pickler_dealloc(Picklerobject *self)
     PyMem_Free(self->write_buf);
     Py_TYPE(self)->tp_free((PyObject *)self);
 }
+#endif
 
 static int
 Pickler_traverse(Picklerobject *self, visitproc visit, void *arg)
@@ -3286,7 +3343,9 @@ static PyTypeObject Picklertype = {
     "cPickle.Pickler",            /*tp_name*/
     sizeof(Picklerobject),              /*tp_basicsize*/
     0,
-    (destructor)Pickler_dealloc,        /* tp_dealloc */
+    // Pyston change: use GC allocs instead of malloc
+    // (destructor)Pickler_dealloc,     /* tp_dealloc */
+    0,                                  /* tp_dealloc */
     0,                                  /* tp_print */
     0,                                  /* tp_getattr */
     0,                                  /* tp_setattr */
@@ -4643,10 +4702,16 @@ load_mark(Unpicklerobject *self)
         s=self->marks_size+20;
         if (s <= self->num_marks) s=self->num_marks + 1;
         if (self->marks == NULL)
-            marks=(Py_ssize_t *)malloc(s * sizeof(Py_ssize_t));
+            // Pyston change: use GC allocs instead of malloc
+            // marks=(Py_ssize_t *)malloc(s * sizeof(Py_ssize_t));
+            marks=(Py_ssize_t *)PyMem_Malloc(s * sizeof(Py_ssize_t));
         else
-            marks=(Py_ssize_t *)realloc(self->marks,
-                                        s * sizeof(Py_ssize_t));
+            // Pyston change: use GC allocs instead of malloc
+            // marks=(Py_ssize_t *)realloc(self->marks,
+            //                             s * sizeof(Py_ssize_t));
+            marks=(Py_ssize_t *)PyMem_Realloc(self->marks,
+                                              s * sizeof(Py_ssize_t));
+
         if (!marks) {
             PyErr_NoMemory();
             return -1;
@@ -5529,7 +5594,8 @@ get_Unpickler(PyObject *self, PyObject *file)
     return (PyObject *)newUnpicklerobject(file);
 }
 
-
+// Pyston change: use GC allocs instead of malloc
+#if 0
 static void
 Unpickler_dealloc(Unpicklerobject *self)
 {
@@ -5554,6 +5620,7 @@ Unpickler_dealloc(Unpicklerobject *self)
 
     Py_TYPE(self)->tp_free((PyObject *)self);
 }
+#endif
 
 static int
 Unpickler_traverse(Unpicklerobject *self, visitproc visit, void *arg)
@@ -5785,7 +5852,9 @@ static PyTypeObject Unpicklertype = {
     "cPickle.Unpickler",                 /*tp_name*/
     sizeof(Unpicklerobject),             /*tp_basicsize*/
     0,
-    (destructor)Unpickler_dealloc,      /* tp_dealloc */
+    // Pyston change: use GC allocs instead of malloc
+    // (destructor)Unpickler_dealloc,   /* tp_dealloc */
+    (destructor)0,                      /* tp_dealloc */
     0,                                  /* tp_print */
     (getattrfunc)Unpickler_getattr,     /* tp_getattr */
     (setattrfunc)Unpickler_setattr,     /* tp_setattr */
@@ -5870,6 +5939,10 @@ init_stuff(PyObject *module_dict)
     if (PyType_Ready(&Picklertype) < 0)
         return -1;
 
+    // Pyston change:
+    if (PyType_Ready(&PdataType) < 0)
+        return -1;
+
     INIT_STR(__class__);
     INIT_STR(__getinitargs__);
     INIT_STR(__dict__);

from_cpython/setup.py

@@ -153,6 +153,12 @@ def locale_ext():
             "Modules/_localemodule.c",
             ]))
 
+@unique
+def cPickle_ext():
+    return Extension("cPickle", sources = map(relpath, [
+            "Modules/cPickle.c",
+            ]))
+
 ext_modules = [future_builtins_ext(),
                multiprocessing_ext(),
                pyexpat_ext(),
@@ -166,7 +172,8 @@ ext_modules = [future_builtins_ext(),
                readline_ext(),
                termios_ext(),
                mmap_ext(),
-               locale_ext()
+               locale_ext(),
+               cPickle_ext()
                ]
 
 

src/capi/typeobject.cpp

@@ -2078,7 +2078,7 @@ static void add_tp_new_wrapper(BoxedClass* type) noexcept {
     if (type->getattr(new_str))
         return;
 
-    type->giveAttr(new_str, new BoxedCApiFunction(tp_new_methoddef, type));
+    type->giveAttr(new_str, new BoxedCApiFunction(tp_new_methoddef, type, NULL /* module name */));
 }
 
 void add_operators(BoxedClass* cls) noexcept {

src/capi/types.h

@@ -30,8 +30,10 @@ public:
     Box* module;
 
 public:
-    BoxedCApiFunction(PyMethodDef* method_def, Box* passthrough, Box* module = NULL)
-        : method_def(method_def), passthrough(passthrough), module(module) {}
+    BoxedCApiFunction(PyMethodDef* method_def, Box* passthrough, Box* module_name)
+        : method_def(method_def), passthrough(passthrough), module(module_name) {
+        assert(!module || PyString_Check(module_name));
+    }
 
     DEFAULT_CLASS(capifunc_cls);
 

src/runtime/builtin_modules/builtins.cpp

@@ -2177,7 +2177,7 @@ void setupBuiltins() {
         { "print", (PyCFunction)builtin_print, METH_VARARGS | METH_KEYWORDS, print_doc },
     };
     for (auto& md : builtin_methods) {
-        builtins_module->giveAttr(md.ml_name, new BoxedCApiFunction(&md, builtins_module));
+        builtins_module->giveAttr(md.ml_name, new BoxedCApiFunction(&md, builtins_module, boxString("__builtin__")));
     }
 }
 }

src/runtime/builtin_modules/sys.cpp

@@ -733,7 +733,7 @@ void setupSys() {
     sys_flags_cls->freeze();
 
     for (auto& md : sys_methods) {
-        sys_module->giveAttr(md.ml_name, new BoxedCApiFunction(&md, sys_module));
+        sys_module->giveAttr(md.ml_name, new BoxedCApiFunction(&md, sys_module, boxString("sys")));
     }
 
     sys_module->giveAttr("__displayhook__", sys_module->getattr(internStringMortal("displayhook")));

src/runtime/classobj.cpp

@@ -1615,6 +1615,12 @@ extern "C" PyObject* PyClass_New(PyObject* bases, PyObject* dict, PyObject* name
     }
 }
 
+extern "C" PyObject* PyClass_Name(PyObject* _classobj) noexcept {
+    RELEASE_ASSERT(PyClass_Check(_classobj), "");
+    BoxedClassobj* classobj = (BoxedClassobj*)_classobj;
+    return classobj->name;
+}
+
 extern "C" PyObject* PyMethod_New(PyObject* func, PyObject* self, PyObject* klass) noexcept {
     try {
         return new BoxedInstanceMethod(self, func, klass);

src/runtime/dict.cpp

@@ -163,6 +163,9 @@ Box* dictLen(BoxedDict* self) {
 }
 
 extern "C" Py_ssize_t PyDict_Size(PyObject* op) noexcept {
+    if (op->cls == attrwrapper_cls)
+        return PyObject_Size(op);
+
     RELEASE_ASSERT(PyDict_Check(op), "");
     return static_cast<BoxedDict*>(op)->d.size();
 }

src/runtime/list.cpp

@@ -1244,11 +1244,8 @@ extern "C" int PyList_SetSlice(PyObject* a, Py_ssize_t ilow, Py_ssize_t ihigh, P
     ASSERT(PyList_Check(l), "%s", l->cls->tp_name);
 
     try {
-        BoxedSlice* slice = (BoxedSlice*)createSlice(boxInt(ilow), boxInt(ihigh), None);
-        if (v)
-            listSetitemSlice(l, slice, v);
-        else
-            listDelitemSlice(l, slice);
+        adjustNegativeIndicesOnObject(l, &ilow, &ihigh);
+        listSetitemSliceInt64(l, ilow, ihigh, 1, v);
         return 0;
     } catch (ExcInfo e) {
         setCAPIException(e);

src/runtime/long.cpp

@@ -20,6 +20,7 @@
 #include <mpfr.h>
 #include <sstream>
 
+#include "llvm/ADT/SmallString.h"
 #include "llvm/Support/raw_ostream.h"
 
 #include "capi/typeobject.h"
@@ -212,18 +213,19 @@ extern "C" PyObject* PyLong_FromString(const char* str, char** pend, int base) n
             || (base == 2 && (str[1] == 'b' || str[1] == 'B'))))
         str += 2;
 
+    llvm::StringRef str_ref = str;
+    llvm::StringRef str_ref_trimmed = str_ref.rtrim(base < 22 ? "Ll \t\n\v\f\r" : " \t\n\v\f\r");
+
     BoxedLong* rtn = new BoxedLong();
     int r = 0;
-    if ((str[strlen(str) - 1] == 'L' || str[strlen(str) - 1] == 'l') && base < 22) {
-        std::string without_l(str, strlen(str) - 1);
-        r = mpz_init_set_str(rtn->n, without_l.c_str(), base);
-    } else {
-        // if base great than 22, 'l' or 'L' should count as a digit.
+    if (str_ref_trimmed != str_ref)
+        r = mpz_init_set_str(rtn->n, str_ref_trimmed.str().c_str(), base);
+    else
         r = mpz_init_set_str(rtn->n, str, base);
-    }
 
-    if (pend)
-        *pend = const_cast<char*>(str) + strlen(str);
+    if (pend) {
+        *pend = const_cast<char*>(str) + str_ref.size();
+    }
     if (r != 0) {
         PyErr_Format(PyExc_ValueError, "invalid literal for long() with base %d: '%s'", base, str);
         return NULL;
@@ -537,6 +539,12 @@ extern "C" void* PyLong_AsVoidPtr(PyObject* vv) noexcept {
     return (void*)x;
 }
 
+
+extern "C" size_t _PyLong_NumBits(PyObject* vv) noexcept {
+    RELEASE_ASSERT(PyLong_Check(vv), "");
+    return mpz_sizeinbase(((BoxedLong*)vv)->n, 2);
+}
+
 extern "C" int _PyLong_AsByteArray(PyLongObject* v, unsigned char* bytes, size_t n, int little_endian,
                                    int is_signed) noexcept {
     const mpz_t* op = &((BoxedLong*)v)->n;

test/CPYTHON_TEST_NOTES.md

@@ -63,9 +63,7 @@ test_collections        assertion failed when doing vars(collections.namedtuple(
 test_compileall         [unknown]
 test_compiler           [unknown]
 test_compile            [unknown]
-test_cookie             [unknown]
 test_copy               Please debug this test in VM.
-test_cpickle            [unknown]
 test_cprofile           [unknown]
 test_crypt              [unknown]
 test_csv                [unknown]
@@ -148,8 +146,6 @@ test_parser             [unknown]
 test_pdb                [unknown]
 test_peepholer          [unknown]
 test_pep352             various unique bugs
-test_pickletools        [unknown]
-test_pickle             unknown
 test_pkg                unknown bug
 test_poplib             SSLError (but only on CI)
 test_pprint             [unknown]

test/tests/cPickle_test.py

+import cPickle
+
+l = [[], (123,)]
+l.append(l)
+s = cPickle.dumps(l)
+print repr(s)
+
+l2 = cPickle.loads(s)
+l3 = l2.pop()
+print l2, l3, l2 is l3
+
+print cPickle.loads(cPickle.dumps("hello world"))
+
+# Sqlalchemy wants this:
+import operator
+print repr(cPickle.dumps(len))
+print repr(cPickle.dumps(operator.and_))
+
+class C(object):
+    pass
+c = C()
+c.a = 1
+# print repr(cPickle.dumps(c)) # Our output is different to cpythons because we don't do some refcounting opts.
+print cPickle.loads(cPickle.dumps(c)).a
+
+
+for obj in [(1, 2), "hello world", u"hola world", 1.0, 1j, 1L, 2, {1:2}, set([3]), frozenset([4])]:
+    print
+    print "Testing pickling subclasses of %s..." % type(obj)
+
+    class MySubclass(type(obj)):
+        pass
+
+    o = MySubclass(obj)
+    print repr(o), type(o)
+
+    for protocol in (0, 1, 2):
+        print "Protocol", protocol
+
+        s = cPickle.dumps(o, protocol=protocol)
+        # print repr(s) # Our output is different to cpythons because we don't do some refcounting opts.
+
+        o2 = cPickle.loads(s)
+        print repr(o2), type(o2)
+
+import cStringIO
+StringIO = cPickle.loads(cPickle.dumps(cStringIO.StringIO))
+print type(StringIO())

test/tests/pickle_test.py

@@ -42,3 +42,7 @@ for obj in [(1, 2), "hello world", u"hola world", 1.0, 1j, 1L, 2, {1:2}, set([3]
 
         o2 = pickle.loads(s)
         print repr(o2), type(o2)
+
+import cStringIO
+StringIO = pickle.loads(pickle.dumps(cStringIO.StringIO))
+print type(StringIO())