Merge pull request #944 from undingen/workarounds

Remove unneeded pyston workarounds + fix misc cpython test errors 

from_cpython/Lib/StringIO.py

@@ -39,9 +39,7 @@ def _complain_ifclosed(closed):
     if closed:
         raise ValueError, "I/O operation on closed file"
 
-# Pyston change: make this a new style class, looks like we don't support iterating otherwise
-# class StringIO:
-class StringIO(object):
+class StringIO:
     """class StringIO([buffer])
 
     When a StringIO object is created, it can be initialized to an existing

from_cpython/Lib/UserDict.py

@@ -80,9 +80,8 @@ class IterableUserDict(UserDict):
     def __iter__(self):
         return iter(self.data)
 
-# Pyston change: disable using the _abcoll module for now.
-# import _abcoll
-# _abcoll.MutableMapping.register(IterableUserDict)
+import _abcoll
+_abcoll.MutableMapping.register(IterableUserDict)
 
 
 class DictMixin:

from_cpython/Lib/copy.py

@@ -257,6 +257,14 @@ def _deepcopy_dict(x, memo):
         y[deepcopy(key, memo)] = deepcopy(value, memo)
     return y
 d[dict] = _deepcopy_dict
+
+# Pyston change: teach copy how to handle attrwrappers
+def get_attrwrapper_type():
+    class C(object):
+        pass
+    return type(C().__dict__)
+d[get_attrwrapper_type()] = _deepcopy_dict
+
 if PyStringMap is not None:
     d[PyStringMap] = _deepcopy_dict
 

from_cpython/Lib/csv.py

@@ -69,9 +69,7 @@ class excel_tab(excel):
     delimiter = '\t'
 register_dialect("excel-tab", excel_tab)
 
-# Pyston change: make this a new style class, looks like we don't support iterating otherwise
-# class DictReader:
-class DictReader(object):
+class DictReader:
     def __init__(self, f, fieldnames=None, restkey=None, restval=None,
                  dialect="excel", *args, **kwds):
         self._fieldnames = fieldnames   # list of keys for the dict

from_cpython/Lib/email/errors.py

@@ -22,11 +22,8 @@ class BoundaryError(MessageParseError):
     """Couldn't find terminating boundary."""
 
 
-# Pyston change: we don't support multiple inheritance yet, so this error class is tricky.
-# We could make it so that it only inherits one of the base classes, but I'd rather that
-# anyone who tries to use this error gets a loud error message rather than different behavior.
-# class MultipartConversionError(MessageError, TypeError):
-    # """Conversion to a multipart is prohibited."""
+class MultipartConversionError(MessageError, TypeError):
+    """Conversion to a multipart is prohibited."""
 
 
 class CharsetError(MessageError):

from_cpython/Lib/encodings/__init__.py

@@ -43,9 +43,7 @@ _norm_encoding_map = ('                                              . '
                       '                ')
 _aliases = aliases.aliases
 
-# Pyston change: we don't support multiple inheritance yet
-#class CodecRegistryError(LookupError, SystemError):
-class CodecRegistryError(LookupError):
+class CodecRegistryError(LookupError, SystemError):
     pass
 
 def normalize_encoding(encoding):

from_cpython/Lib/tempfile.py

@@ -29,8 +29,7 @@ __all__ = [
 
 # Imports.
 
-# Pyston change: don't import io
-# import io as _io
+import io as _io
 import os as _os
 import errno as _errno
 from random import Random as _Random
@@ -198,10 +197,8 @@ def _get_default_tempdir():
                 fd = _os.open(filename, flags, 0o600)
                 try:
                     try:
-                        # Pyston change: simplify this so that it doesn't need the _io module:
-                        _os.write(fd, b'blat')
-                        # with _io.open(fd, 'wb', closefd=False) as fp:
-                            # fp.write(b'blat')
+                        with _io.open(fd, 'wb', closefd=False) as fp:
+                            fp.write(b'blat')
                     finally:
                         _os.close(fd)
                 finally:

from_cpython/Lib/test/test_array.py

-# expected: fail
 """Test the arraymodule.
    Roger E. Masse
 """
@@ -766,6 +765,8 @@ class BaseTest(unittest.TestCase):
             b = buffer(a)
         self.assertEqual(b[0], a.tostring()[0])
 
+    # Pyston change: disable this test because of our GC
+    @unittest.skip("Pyston" in sys.version)
     def test_weakref(self):
         s = array.array(self.typecode, self.example)
         p = proxy(s)

from_cpython/Lib/test/test_urllib2.py

-# expected: fail
 import unittest
 from test import test_support
 
@@ -9,11 +8,6 @@ import StringIO
 import urllib2
 from urllib2 import Request, OpenerDirector
 
-try:
-    import ssl
-except ImportError:
-    ssl = None
-
 # XXX
 # Request
 # CacheFTPHandler (hard to write)
@@ -26,7 +20,7 @@ class TrivialTests(unittest.TestCase):
         self.assertRaises(ValueError, urllib2.urlopen, 'bogus url')
 
         # XXX Name hacking to get this to work on Windows.
-        fname = os.path.abspath(urllib2.__file__).replace(os.sep, '/')
+        fname = os.path.abspath(urllib2.__file__).replace('\\', '/')
 
         # And more hacking to get it to work on MacOS. This assumes
         # urllib.pathname2url works, unfortunately...
@@ -53,14 +47,6 @@ class TrivialTests(unittest.TestCase):
         for string, list in tests:
             self.assertEqual(urllib2.parse_http_list(string), list)
 
-    @unittest.skipUnless(ssl, "ssl module required")
-    def test_cafile_and_context(self):
-        context = ssl.create_default_context()
-        with self.assertRaises(ValueError):
-            urllib2.urlopen(
-                "https://localhost", cafile="/nonexistent/path", context=context
-            )
-
 
 def test_request_headers_dict():
     """

from_cpython/Lib/test/test_urllib2_localnet.py

-# expected: fail
 import urlparse
 import urllib2
 import BaseHTTPServer

from_cpython/Lib/types.py

@@ -33,8 +33,7 @@ try:
 except NameError:
     StringTypes = (StringType,)
 
-# Pyston change: 'buffer' is not implemented yet
-# BufferType = buffer
+BufferType = buffer
 
 TupleType = tuple
 ListType = list

src/runtime/builtin_modules/thread.cpp

@@ -28,6 +28,7 @@ using namespace pyston::threading;
 
 extern "C" void initthread();
 
+std::atomic_long nb_threads;
 
 static int initialized;
 static void PyThread__init_thread(void); /* Forward */
@@ -69,6 +70,8 @@ static void* thread_start(Box* target, Box* varargs, Box* kwargs) {
     timer.pushTopLevel(getCPUTicks());
 #endif
 
+    ++nb_threads;
+
     try {
         runtimeCall(target, ArgPassSpec(0, 0, true, kwargs != NULL), varargs, kwargs, NULL, NULL, NULL);
     } catch (ExcInfo e) {
@@ -79,6 +82,8 @@ static void* thread_start(Box* target, Box* varargs, Box* kwargs) {
     timer.popTopLevel(getCPUTicks());
 #endif
 
+    --nb_threads;
+
     return NULL;
 }
 
@@ -190,6 +195,10 @@ Box* stackSize() {
     Py_FatalError("unimplemented");
 }
 
+Box* threadCount() {
+    return boxInt(nb_threads);
+}
+
 void setupThread() {
     // Hacky: we want to use some of CPython's implementation of the thread module (the threading local stuff),
     // and some of ours (thread handling).  Start off by calling a cut-down version of initthread, and then
@@ -209,6 +218,8 @@ void setupThread() {
         "get_ident", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)getIdent, BOXED_INT, 0), "get_ident"));
     thread_module->giveAttr(
         "stack_size", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)stackSize, BOXED_INT, 0), "stack_size"));
+    thread_module->giveAttr(
+        "_count", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)threadCount, BOXED_INT, 0), "_count"));
 
     thread_lock_cls = BoxedClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(BoxedThreadLock), false, "lock");
     thread_lock_cls->tp_dealloc = BoxedThreadLock::threadLockDestructor;

src/runtime/classobj.cpp

@@ -1599,7 +1599,7 @@ extern "C" PyObject* PyMethod_Class(PyObject* im) noexcept {
 
 void setupClassobj() {
     classobj_cls = BoxedClass::create(type_cls, object_cls, &BoxedClassobj::gcHandler, offsetof(BoxedClassobj, attrs),
-                                      0, sizeof(BoxedClassobj), false, "classobj");
+                                      offsetof(BoxedClassobj, weakreflist), sizeof(BoxedClassobj), false, "classobj");
     instance_cls = BoxedClass::create(type_cls, object_cls, &BoxedInstance::gcHandler, offsetof(BoxedInstance, attrs),
                                       offsetof(BoxedInstance, weakreflist), sizeof(BoxedInstance), false, "instance");
 

src/runtime/classobj.h

@@ -46,6 +46,8 @@ public:
     BoxedTuple* bases;
     BoxedString* name;
 
+    Box** weakreflist;
+
     BoxedClassobj(BoxedString* name, BoxedTuple* bases) : bases(bases), name(name) {}
 
     static void gcHandler(GCVisitor* v, Box* _o) {

src/runtime/list.cpp

@@ -462,9 +462,7 @@ int list_ass_ext_slice(BoxedList* self, PyObject* item, PyObject* value) {
 
         /* protect against a[::-1] = a */
         if (self == value) {
-            abort();
-            // seq = list_slice((PyListObject*)value, 0,
-            // PyList_GET_SIZE(value));
+            seq = list_slice(value, 0, PyList_GET_SIZE(value));
         } else {
             seq = PySequence_Fast(value, "must assign iterable "
                                          "to extended slice");
@@ -1021,13 +1019,16 @@ Box* listCount(BoxedList* self, Box* elt) {
     return boxInt(count);
 }
 
-Box* listIndex(BoxedList* self, Box* elt, BoxedInt* _start, Box** args) {
-    BoxedInt* _stop = (BoxedInt*)args[0];
-    RELEASE_ASSERT(!_start || _start->cls == int_cls, "");
-    RELEASE_ASSERT(!_stop || _stop->cls == int_cls, "");
+Box* listIndex(BoxedList* self, Box* elt, Box* _start, Box** args) {
+    Box* _stop = (BoxedInt*)args[0];
 
-    int64_t start = _start ? _start->n : 0;
-    int64_t stop = _stop ? _stop->n : self->size;
+    int64_t start = 0;
+    int64_t stop = self->size;
+
+    if (!_PyEval_SliceIndex(_start, &start))
+        throwCAPIException();
+    if (!_PyEval_SliceIndex(_stop, &stop))
+        throwCAPIException();
 
     if (start < 0) {
         start += self->size;

test/CPYTHON_TEST_NOTES.md

@@ -23,7 +23,6 @@ test_aifc               Unsupported subclassing from file?
 test_al                 No module named al
 test_applesingle        Not really a failure, but it tries to skip itself and we don't support that
 test_argparse           [unknown]
-test_array              [unknown]
 test_ascii_formattd     [unknown]
 test_ast                [unknown]
 test_asynchat           [unknown]
@@ -248,9 +247,7 @@ test_unicode_file       exit code 139, no error message
 test_unittest           serialize_ast assert
 test_univnewlines2k     [unknown]
 test_univnewlines       [unknown]
-test_urllib2_localnet   [unknown]
 test_urllib2net         [unknown]
-test_urllib2            segfault due to attrwrapper corruption
 test_urllibnet          [unknown]
 test_userdict           segfault: repr of recursive dict?
 test_userlist           slice(1L, 1L)

test/tests/copy_test.py

@@ -7,6 +7,7 @@ c.a = 1
 c.b = (["str", 1], 2L)
 print sorted(c.__dict__.items())
 cc = copy.deepcopy(c)
+copy_dict = copy.deepcopy(c.__dict__)
 del c.a
 print sorted(cc.__dict__.items())
-
+print sorted(copy_dict.items())

test/tests/list.py

@@ -61,7 +61,7 @@ list_index = [1, 2, 3, 4, 5]
 for i in xrange(1, 6):
     assert list_index.index(i) == i-1
     try:
-        print list_index.index(i, 3, 4)
+        print list_index.index(i, 3L, 4L)
     except ValueError as e:
         print e
     try:
@@ -167,6 +167,8 @@ l = range(5)
 l[2:4] = tuple(range(2))
 print l
 
+l[::-1] = l
+print l
 
 l = [None]*4
 try:

test/tests/thread_test.py

-from thread import start_new_thread, allocate_lock
+from thread import start_new_thread, allocate_lock, _count
 import time
 
 print type(allocate_lock())
@@ -9,11 +9,13 @@ done = 0
 def run(arg):
     global done
     with print_lock:
+        print "num threads:", _count()
         print "in other thread!", arg
     done = 1
 
 
 print "starting!"
+print "num threads:", _count()
 with print_lock:
     t = start_new_thread(run, (5,))
     print type(t)
@@ -22,6 +24,7 @@ while not done:
     time.sleep(0)
 
 print "done!"
+print "num threads:", _count()
 
 done = False
 with print_lock:
@@ -46,6 +49,7 @@ def run2():
     global state
 
     print lock.acquire(0)
+    print "num threads:", _count()
     state = 1
     print lock.acquire()
     lock.release()

test/tests/weakrefs.py

@@ -48,6 +48,9 @@ test_wr(frozenset())
 test_wr((i*i for i in range(1000000)))
 test_wr(set)
 test_wr(file("/etc/passwd"))
+class Old:
+    pass
+test_wr(Old)
 # missing: db cursor from the bsddb module
 # missing: sockets
 test_wr(array.array('d', [1.0, 2.0, 3.14]))