Merge pull request #348 from undingen/io

Add the io/_io Module and make import gzip work

Makefile

@@ -292,8 +292,8 @@ STDLIB_OBJS := stdlib.bc.o stdlib.stripped.bc.o
 STDLIB_RELEASE_OBJS := stdlib.release.bc.o
 ASM_SRCS := $(wildcard src/runtime/*.S)
 
-STDMODULE_SRCS := errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c zlibmodule.c _codecsmodule.c socketmodule.c unicodedata.c _weakref.c cStringIO.c $(EXTRA_STDMODULE_SRCS)
-STDOBJECT_SRCS := structseq.c capsule.c stringobject.c exceptions.c unicodeobject.c unicodectype.c bytearrayobject.c bytes_methods.c weakrefobject.c $(EXTRA_STDOBJECT_SRCS)
+STDMODULE_SRCS := errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c zlibmodule.c _codecsmodule.c socketmodule.c unicodedata.c _weakref.c cStringIO.c _io/bufferedio.c _io/bytesio.c _io/fileio.c _io/iobase.c _io/_iomodule.c _io/stringio.c _io/textio.c $(EXTRA_STDMODULE_SRCS)
+STDOBJECT_SRCS := structseq.c capsule.c stringobject.c exceptions.c unicodeobject.c unicodectype.c bytearrayobject.c bytes_methods.c weakrefobject.c memoryobject.c $(EXTRA_STDOBJECT_SRCS)
 STDPYTHON_SRCS := pyctype.c getargs.c formatter_string.c pystrtod.c dtoa.c formatter_unicode.c structmember.c $(EXTRA_STDPYTHON_SRCS)
 FROM_CPYTHON_SRCS := $(addprefix from_cpython/Modules/,$(STDMODULE_SRCS)) $(addprefix from_cpython/Objects/,$(STDOBJECT_SRCS)) $(addprefix from_cpython/Python/,$(STDPYTHON_SRCS))
 

from_cpython/CMakeLists.txt

@@ -15,10 +15,10 @@ endforeach(STDLIB_FILE)
 add_custom_target(copy_stdlib ALL DEPENDS ${STDLIB_TARGETS})
 
 # compile specified files in from_cpython/Modules
-file(GLOB_RECURSE STDMODULE_SRCS Modules errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c zlibmodule.c _codecsmodule.c socketmodule.c unicodedata.c _weakref.c cStringIO.c)
+file(GLOB_RECURSE STDMODULE_SRCS Modules errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c zlibmodule.c _codecsmodule.c socketmodule.c unicodedata.c _weakref.c cStringIO.c bufferedio.c bytesio.c fileio.c iobase.c _iomodule.c stringio.c textio.c)
 
 # compile specified files in from_cpython/Objects
-file(GLOB_RECURSE STDOBJECT_SRCS Objects structseq.c capsule.c stringobject.c exceptions.c unicodeobject.c unicodectype.c bytearrayobject.c bytes_methods.c weakrefobject.c)
+file(GLOB_RECURSE STDOBJECT_SRCS Objects structseq.c capsule.c stringobject.c exceptions.c unicodeobject.c unicodectype.c bytearrayobject.c bytes_methods.c weakrefobject.c memoryobject.c)
 
 # compile specified files in from_cpython/Python
 file(GLOB_RECURSE STDPYTHON_SRCS Python getargs.c pyctype.c formatter_string.c pystrtod.c dtoa.c formatter_unicode.c structmember.c)

from_cpython/Include/Python.h

@@ -57,6 +57,7 @@
 #ifndef WITHOUT_COMPLEX
 #include "complexobject.h"
 #endif
+#include "memoryobject.h"
 #include "stringobject.h"
 #include "bufferobject.h"
 #include "bytesobject.h"

from_cpython/Include/memoryobject.h

+// This file is originally from CPython 2.7, with modifications for Pyston
+
+/* Memory view object. In Python this is available as "memoryview". */
+
+#ifndef Py_MEMORYOBJECT_H
+#define Py_MEMORYOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+PyAPI_DATA(PyTypeObject) PyMemoryView_Type;
+
+#define PyMemoryView_Check(op) (Py_TYPE(op) == &PyMemoryView_Type)
+
+/* Get a pointer to the underlying Py_buffer of a memoryview object. */
+#define PyMemoryView_GET_BUFFER(op) (&((PyMemoryViewObject *)(op))->view)
+/* Get a pointer to the PyObject from which originates a memoryview object. */
+#define PyMemoryView_GET_BASE(op) (((PyMemoryViewObject *)(op))->view.obj)
+
+
+PyAPI_FUNC(PyObject *) PyMemoryView_GetContiguous(PyObject *base, 
+						  int buffertype, 
+                          char fort) PYSTON_NOEXCEPT;
+
+    /* Return a contiguous chunk of memory representing the buffer
+       from an object in a memory view object.  If a copy is made then the
+       base object for the memory view will be a *new* bytes object. 
+       
+       Otherwise, the base-object will be the object itself and no 
+       data-copying will be done. 
+
+       The buffertype argument can be PyBUF_READ, PyBUF_WRITE,
+       PyBUF_SHADOW to determine whether the returned buffer
+       should be READONLY, WRITABLE, or set to update the
+       original buffer if a copy must be made.  If buffertype is
+       PyBUF_WRITE and the buffer is not contiguous an error will
+       be raised.  In this circumstance, the user can use
+       PyBUF_SHADOW to ensure that a a writable temporary
+       contiguous buffer is returned.  The contents of this
+       contiguous buffer will be copied back into the original
+       object after the memoryview object is deleted as long as
+       the original object is writable and allows setting an
+       exclusive write lock. If this is not allowed by the
+       original object, then a BufferError is raised.
+       
+       If the object is multi-dimensional and if fortran is 'F',
+       the first dimension of the underlying array will vary the
+       fastest in the buffer.  If fortran is 'C', then the last
+       dimension will vary the fastest (C-style contiguous).  If
+       fortran is 'A', then it does not matter and you will get
+       whatever the object decides is more efficient.  
+
+       A new reference is returned that must be DECREF'd when finished.
+    */
+
+PyAPI_FUNC(PyObject *) PyMemoryView_FromObject(PyObject *base) PYSTON_NOEXCEPT;
+
+PyAPI_FUNC(PyObject *) PyMemoryView_FromBuffer(Py_buffer *info) PYSTON_NOEXCEPT;
+    /* create new if bufptr is NULL 
+        will be a new bytesobject in base */
+
+
+/* The struct is declared here so that macros can work, but it shouldn't
+   be considered public. Don't access those fields directly, use the macros
+   and functions instead! */
+typedef struct {
+    PyObject_HEAD
+    PyObject *base;
+    Py_buffer view;
+} PyMemoryViewObject;
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_MEMORYOBJECT_H */

from_cpython/Include/pyconfig.h

@@ -27,6 +27,7 @@
 #define SIZEOF_INT 4
 #define SIZEOF_LONG 8
 #define SIZEOF_LONG_LONG 8
+#define SIZEOF_OFF_T 8
 #define SIZEOF_PTHREAD_T 8
 #define HAVE_COPYSIGN 1
 #define HAVE_ROUND 1

from_cpython/Include/sliceobject.h

@@ -8,9 +8,11 @@ extern "C" {
 
 /* The unique ellipsis object "..." */
 
-PyAPI_DATA(PyObject) _Py_EllipsisObject; /* Don't use this directly */
-
-#define Py_Ellipsis (&_Py_EllipsisObject)
+// Pyston change
+// PyAPI_DATA(PyObject) _Py_EllipsisObject; /* Don't use this directly */
+// #define Py_Ellipsis (&_Py_EllipsisObject)
+PyAPI_DATA(PyObject) *Ellipsis; /* Don't use this directly */
+#define Py_Ellipsis Ellipsis
 
 /* Slice object interface */
 

from_cpython/Modules/_io/_iomodule.c

@@ -628,10 +628,13 @@ init_io(void)
     if (m == NULL)
         return;
 
+    // Pyston change: we don't support importing os during init, but it looks like it works without it...
+#if 0
     /* put os in the module state */
     _PyIO_os_module = PyImport_ImportModule("os");
     if (_PyIO_os_module == NULL)
         goto fail;
+#endif
 
 #define ADD_TYPE(type, name) \
     if (PyType_Ready(type) < 0) \
@@ -647,9 +650,13 @@ init_io(void)
         goto fail;
 
     /* UnsupportedOperation inherits from ValueError and IOError */
-    _PyIO_unsupported_operation = PyObject_CallFunction(
-        (PyObject *)&PyType_Type, "s(OO){}",
-        "UnsupportedOperation", PyExc_ValueError, PyExc_IOError);
+    // Pyston change: during init we have to supply the module name by ourself.
+    _PyIO_unsupported_operation = PyObject_CallFunction((PyObject *)&PyType_Type, "s(OO){s:s}",
+          "UnsupportedOperation", PyExc_ValueError, PyExc_IOError, "__module__", "io");
+    // _PyIO_unsupported_operation = PyObject_CallFunction(
+    //    (PyObject *)&PyType_Type, "s(OO){}",
+    //    "UnsupportedOperation", PyExc_ValueError, PyExc_IOError);
+
     if (_PyIO_unsupported_operation == NULL)
         goto fail;
     Py_INCREF(_PyIO_unsupported_operation);
@@ -760,6 +767,35 @@ init_io(void)
     if (!(_PyIO_zero = PyLong_FromLong(0L)))
         goto fail;
 
+    // Pyston change: register with gc
+    // TODO: automatically register static variables as roots
+    PyGC_AddRoot(_PyIO_str_close);
+    PyGC_AddRoot(_PyIO_str_closed);
+    PyGC_AddRoot(_PyIO_str_decode);
+    PyGC_AddRoot(_PyIO_str_encode);
+    PyGC_AddRoot(_PyIO_str_fileno);
+    PyGC_AddRoot(_PyIO_str_flush);
+    PyGC_AddRoot(_PyIO_str_getstate);
+    PyGC_AddRoot(_PyIO_str_isatty);
+    PyGC_AddRoot(_PyIO_str_newlines);
+    PyGC_AddRoot(_PyIO_str_nl);
+    PyGC_AddRoot(_PyIO_str_read);
+    PyGC_AddRoot(_PyIO_str_read1);
+    PyGC_AddRoot(_PyIO_str_readable);
+    PyGC_AddRoot(_PyIO_str_readinto);
+    PyGC_AddRoot(_PyIO_str_readline);
+    PyGC_AddRoot(_PyIO_str_reset);
+    PyGC_AddRoot(_PyIO_str_seek);
+    PyGC_AddRoot(_PyIO_str_seekable);
+    PyGC_AddRoot(_PyIO_str_setstate);
+    PyGC_AddRoot(_PyIO_str_tell);
+    PyGC_AddRoot(_PyIO_str_truncate);
+    PyGC_AddRoot(_PyIO_str_writable);
+    PyGC_AddRoot(_PyIO_str_write);
+    // PyGC_AddRoot(_PyIO_empty_str); // this is already registered as root
+    PyGC_AddRoot(_PyIO_empty_bytes);
+    PyGC_AddRoot(_PyIO_zero);
+
     return;
 
   fail:

from_cpython/Modules/_io/iobase.c

@@ -201,10 +201,12 @@ _PyIOBase_finalize(PyObject *self)
 
     /* If _PyIOBase_finalize() is called from a destructor, we need to
        resurrect the object as calling close() can invoke arbitrary code. */
-    is_zombie = (Py_REFCNT(self) == 0);
-    if (is_zombie) {
-        ++Py_REFCNT(self);
-    }
+    // Pyston change:
+    is_zombie = 0;
+    // is_zombie = (Py_REFCNT(self) == 0);
+    //if (is_zombie) {
+    //    ++Py_REFCNT(self);
+    //}
     PyErr_Fetch(&tp, &v, &tb);
     /* If `closed` doesn't exist or can't be evaluated as bool, then the
        object is probably in an unusable state, so ignore. */
@@ -230,6 +232,8 @@ _PyIOBase_finalize(PyObject *self)
     }
     PyErr_Restore(tp, v, tb);
     if (is_zombie) {
+// Pyston change:
+#if 0
         if (--Py_REFCNT(self) != 0) {
             /* The object lives again. The following code is taken from
                slot_tp_del in typeobject.c. */
@@ -245,12 +249,14 @@ _PyIOBase_finalize(PyObject *self)
              * _Py_NewReference bumped tp_allocs:  both of those need to be
              * undone.
              */
+
 #ifdef COUNT_ALLOCS
             --Py_TYPE(self)->tp_frees;
             --Py_TYPE(self)->tp_allocs;
 #endif
             return -1;
         }
+#endif
     }
     return 0;
 }

from_cpython/Modules/_io/textio.c

@@ -305,8 +305,9 @@ _PyIncrementalNewlineDecoder_decode(PyObject *_self,
     if (!final) {
         if (output_len > 0
             && PyUnicode_AS_UNICODE(output)[output_len - 1] == '\r') {
-
-            if (Py_REFCNT(output) == 1) {
+            // Pyston change:
+            // if (Py_REFCNT(output) == 1) {
+            if (0) {
                 if (PyUnicode_Resize(&output, output_len - 1) < 0)
                     goto error;
             }
@@ -405,7 +406,9 @@ _PyIncrementalNewlineDecoder_decode(PyObject *_self,
             PyObject *translated = NULL;
             Py_UNICODE *out_str;
             Py_UNICODE *in, *out, *end;
-            if (Py_REFCNT(output) != 1) {
+            // Pyston change
+            // if (Py_REFCNT(output) != 1) {
+            if (1) {
                 /* We could try to optimize this so that we only do a copy
                    when there is something to translate. On the other hand,
                    most decoders should only output non-shared strings, i.e.
@@ -413,7 +416,7 @@ _PyIncrementalNewlineDecoder_decode(PyObject *_self,
                 translated = PyUnicode_FromUnicode(NULL, len);
                 if (translated == NULL)
                     goto error;
-                assert(Py_REFCNT(translated) == 1);
+                // assert(Py_REFCNT(translated) == 1); Pyston change
                 memcpy(PyUnicode_AS_UNICODE(translated),
                        PyUnicode_AS_UNICODE(output),
                        len * sizeof(Py_UNICODE));
@@ -1798,7 +1801,9 @@ _textiowrapper_readline(textio *self, Py_ssize_t limit)
         /* Our line ends in the current buffer */
         self->decoded_chars_used = endpos - offset_to_buffer;
         if (start > 0 || endpos < PyUnicode_GET_SIZE(line)) {
-            if (start == 0 && Py_REFCNT(line) == 1) {
+            // Pyston change:
+            // if (start == 0 && Py_REFCNT(line) == 1) {
+            if (0) {
                 if (PyUnicode_Resize(&line, endpos) < 0)
                     goto error;
             }

from_cpython/Modules/signalmodule.c

@@ -892,6 +892,9 @@ finisignal(void)
 int
 PyErr_CheckSignals(void)
 {
+    if (!is_tripped)
+        return 0;
+
     // Pyston change:
     Py_FatalError("TODO");
 
@@ -899,9 +902,6 @@ PyErr_CheckSignals(void)
     int i;
     PyObject *f;
 
-    if (!is_tripped)
-        return 0;
-
 #ifdef WITH_THREAD
     if (PyThread_get_thread_ident() != main_thread)
         return 0;

from_cpython/Modules/zlibmodule.c

@@ -1055,6 +1055,11 @@ PyInit_zlib(void)
     PyObject *m, *ver;
     Py_TYPE(&Comptype) = &PyType_Type;
     Py_TYPE(&Decomptype) = &PyType_Type;
+
+    // Pyston change
+    PyType_Ready(&Comptype);
+    PyType_Ready(&Decomptype);
+
     m = Py_InitModule4("zlib", zlib_methods,
                        zlib_module_documentation,
                        (PyObject*)NULL,PYTHON_API_VERSION);

src/capi/modsupport.cpp

@@ -83,7 +83,7 @@ static int _ustrlen(Py_UNICODE* u) {
 
 static PyObject* do_mktuple(const char**, va_list*, int, int, int) noexcept;
 static PyObject* do_mklist(const char**, va_list*, int, int, int) noexcept;
-// static PyObject *do_mkdict(const char**, va_list *, int, int, int) noexcept;
+static PyObject* do_mkdict(const char**, va_list*, int, int, int) noexcept;
 static PyObject* do_mkvalue(const char**, va_list*, int) noexcept;
 
 static PyObject* do_mkvalue(const char** p_format, va_list* p_va, int flags) noexcept {
@@ -95,10 +95,8 @@ static PyObject* do_mkvalue(const char** p_format, va_list* p_va, int flags) noe
             case '[':
                 return do_mklist(p_format, p_va, ']', countformat(*p_format, ']'), flags);
 
-#if 0
             case '{':
                 return do_mkdict(p_format, p_va, '}', countformat(*p_format, '}'), flags);
-#endif
 
             case 'b':
             case 'B':
@@ -198,6 +196,13 @@ static PyObject* do_mkvalue(const char** p_format, va_list* p_va, int flags) noe
                 return v;
             }
 #endif
+
+            case ':':
+            case ',':
+            case ' ':
+            case '\t':
+                break;
+
             default:
                 RELEASE_ASSERT(0, "%c", *((*p_format) - 1));
         }
@@ -239,6 +244,48 @@ static PyObject* do_mktuple(const char** p_format, va_list* p_va, int endchar, i
     return v;
 }
 
+static PyObject* do_mkdict(const char** p_format, va_list* p_va, int endchar, int n, int flags) noexcept {
+    PyObject* d;
+    int i;
+    int itemfailed = 0;
+    if (n < 0)
+        return NULL;
+    if ((d = PyDict_New()) == NULL)
+        return NULL;
+    /* Note that we can't bail immediately on error as this will leak
+       refcounts on any 'N' arguments. */
+    for (i = 0; i < n; i += 2) {
+        PyObject* k, *v;
+        int err;
+        k = do_mkvalue(p_format, p_va, flags);
+        if (k == NULL) {
+            itemfailed = 1;
+            Py_INCREF(Py_None);
+            k = Py_None;
+        }
+        v = do_mkvalue(p_format, p_va, flags);
+        if (v == NULL) {
+            itemfailed = 1;
+            Py_INCREF(Py_None);
+            v = Py_None;
+        }
+        err = PyDict_SetItem(d, k, v);
+        Py_DECREF(k);
+        Py_DECREF(v);
+        if (err < 0 || itemfailed) {
+            Py_DECREF(d);
+            return NULL;
+        }
+    }
+    if (d != NULL && **p_format != endchar) {
+        Py_DECREF(d);
+        d = NULL;
+        PyErr_SetString(PyExc_SystemError, "Unmatched paren in format");
+    } else if (endchar)
+        ++*p_format;
+    return d;
+}
+
 static PyObject* do_mklist(const char** p_format, va_list* p_va, int endchar, int n, int flags) noexcept {
     PyObject* v;
     int i;

src/capi/object.cpp

@@ -151,6 +151,16 @@ extern "C" int PyObject_SetAttr(PyObject* v, PyObject* name, PyObject* value) no
     Py_FatalError("unimplemented");
 }
 
+extern "C" int PyObject_SetAttrString(PyObject* v, const char* name, PyObject* w) noexcept {
+    try {
+        setattr(v, name, w);
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return -1;
+    }
+    return 0;
+}
+
 extern "C" PyObject* PyObject_GetAttrString(PyObject* o, const char* attr) noexcept {
     // TODO do something like this?  not sure if this is safe; will people expect that calling into a known function
     // won't end up doing a GIL check?

src/capi/typeobject.cpp

@@ -2315,7 +2315,8 @@ extern "C" int PyType_Ready(PyTypeObject* cls) noexcept {
     RELEASE_ASSERT(cls->tp_getattro == NULL || cls->tp_getattro == PyObject_GenericGetAttr, "");
     RELEASE_ASSERT(cls->tp_setattro == NULL || cls->tp_setattro == PyObject_GenericSetAttr, "");
 
-    int ALLOWABLE_FLAGS = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_CHECKTYPES;
+    int ALLOWABLE_FLAGS = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_CHECKTYPES
+                          | Py_TPFLAGS_HAVE_NEWBUFFER;
     RELEASE_ASSERT((cls->tp_flags & ~ALLOWABLE_FLAGS) == 0, "");
     if (cls->tp_as_number) {
         RELEASE_ASSERT(cls->tp_flags & Py_TPFLAGS_CHECKTYPES, "Pyston doesn't yet support non-checktypes behavior");

src/runtime/builtin_modules/builtins.cpp

@@ -44,6 +44,8 @@
 namespace pyston {
 
 extern "C" {
+Box* Ellipsis = 0;
+
 // Copied from CPython:
 #if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
 const char* Py_FileSystemDefaultEncoding = "mbcs";
@@ -960,7 +962,7 @@ void setupBuiltins() {
 
     BoxedHeapClass* ellipsis_cls
         = BoxedHeapClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(Box), false, "ellipsis");
-    Box* Ellipsis = new (ellipsis_cls) Box();
+    Ellipsis = new (ellipsis_cls) Box();
     assert(Ellipsis->cls);
     gc::registerPermanentRoot(Ellipsis);
 
@@ -1139,6 +1141,12 @@ void setupBuiltins() {
     builtins_module->giveAttr("property", property_cls);
     builtins_module->giveAttr("staticmethod", staticmethod_cls);
     builtins_module->giveAttr("classmethod", classmethod_cls);
+
+    assert(memoryview_cls);
+    Py_TYPE(&PyMemoryView_Type) = &PyType_Type;
+    PyType_Ready(&PyMemoryView_Type);
+    builtins_module->giveAttr("memoryview", memoryview_cls);
+
     builtins_module->giveAttr(
         "eval", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)eval, UNKNOWN, 1, 0, false, false), "eval"));
 

src/runtime/import.cpp

@@ -294,7 +294,14 @@ extern "C" PyObject* PyImport_ImportModule(const char* name) noexcept {
     if (strcmp("__builtin__", name) == 0)
         return builtins_module;
 
-    Py_FatalError("unimplemented");
+    try {
+        // TODO: check if this has the same behaviour as the cpython implementation
+        std::string str = name;
+        return import(0, None, &str);
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return NULL;
+    }
 }
 
 extern "C" Box* import(int level, Box* from_imports, const std::string* module_name) {

src/runtime/int.cpp

@@ -33,6 +33,10 @@
 namespace pyston {
 
 extern "C" unsigned long PyInt_AsUnsignedLongMask(PyObject* op) noexcept {
+    if (op && PyInt_Check(op))
+        return PyInt_AS_LONG((PyIntObject*)op);
+    if (op && PyLong_Check(op))
+        return PyLong_AsUnsignedLongMask(op);
     Py_FatalError("unimplemented");
 }
 

src/runtime/long.cpp

@@ -78,7 +78,12 @@ extern "C" int _PyLong_AsInt(PyObject* obj) noexcept {
     return (int)result;
 }
 
-extern "C" unsigned long PyLong_AsUnsignedLongMask(PyObject* op) noexcept {
+extern "C" unsigned long PyLong_AsUnsignedLongMask(PyObject* vv) noexcept {
+    if (PyLong_Check(vv)) {
+        BoxedLong* l = static_cast<BoxedLong*>(vv);
+        return mpz_get_ui(l->n);
+    }
+
     Py_FatalError("unimplemented");
 }
 
@@ -136,10 +141,29 @@ extern "C" unsigned long PyLong_AsUnsignedLong(PyObject* vv) noexcept {
 }
 
 extern "C" long PyLong_AsLong(PyObject* vv) noexcept {
+    int overflow;
+    long result = PyLong_AsLongAndOverflow(vv, &overflow);
+    if (overflow) {
+        /* XXX: could be cute and give a different
+           message for overflow == -1 */
+        PyErr_SetString(PyExc_OverflowError, "Python int too large to convert to C long");
+    }
+    return result;
+}
+
+extern "C" Py_ssize_t PyLong_AsSsize_t(PyObject* vv) noexcept {
     RELEASE_ASSERT(PyLong_Check(vv), "");
-    BoxedLong* l = static_cast<BoxedLong*>(vv);
-    RELEASE_ASSERT(mpz_fits_slong_p(l->n), "");
-    return mpz_get_si(l->n);
+
+    if (PyLong_Check(vv)) {
+        BoxedLong* l = static_cast<BoxedLong*>(vv);
+        if (mpz_fits_slong_p(l->n)) {
+            return mpz_get_si(l->n);
+        } else {
+            PyErr_SetString(PyExc_OverflowError, "long int too large to convert to int");
+            return -1;
+        }
+    }
+    Py_FatalError("unimplemented");
 }
 
 extern "C" long PyLong_AsLongAndOverflow(Box* vv, int* overflow) noexcept {
@@ -155,18 +179,35 @@ extern "C" long PyLong_AsLongAndOverflow(Box* vv, int* overflow) noexcept {
     if (PyInt_Check(vv))
         return PyInt_AsLong(vv);
 
-    if (PyLong_Check(vv)) {
-        BoxedLong* l = static_cast<BoxedLong*>(vv);
-        if (mpz_fits_slong_p(l->n)) {
-            return mpz_get_si(l->n);
-        } else {
-            *overflow = mpz_sgn(l->n);
+    if (!PyLong_Check(vv)) {
+        PyNumberMethods* nb;
+        nb = vv->cls->tp_as_number;
+        if (nb == NULL || nb->nb_int == NULL) {
+            PyErr_SetString(PyExc_TypeError, "an integer is required");
             return -1;
         }
+
+        vv = (*nb->nb_int)(vv);
+        if (vv == NULL)
+            return -1;
+
+        if (PyInt_Check(vv))
+            return PyInt_AsLong(vv);
+
+        if (!PyLong_Check(vv)) {
+            PyErr_SetString(PyExc_TypeError, "nb_int should return int object");
+            return -1;
+        }
+        // fallthrough: this has to be a long
     }
 
-    // TODO CPython tries to go through tp_as_number
-    Py_FatalError("unsupported case");
+    BoxedLong* l = static_cast<BoxedLong*>(vv);
+    if (mpz_fits_slong_p(l->n)) {
+        return mpz_get_si(l->n);
+    } else {
+        *overflow = mpz_sgn(l->n);
+        return -1;
+    }
 }
 
 extern "C" double PyLong_AsDouble(PyObject* vv) noexcept {
@@ -288,11 +329,6 @@ extern "C" PyObject* _PyLong_FromByteArray(const unsigned char* bytes, size_t n,
     if (n == 0)
         return PyLong_FromLong(0);
 
-    if (is_signed) {
-        Py_FatalError("unimplemented");
-        return 0;
-    }
-
     if (!little_endian) {
         // TODO: check if the behaviour of mpz_import is right when big endian is specified.
         Py_FatalError("unimplemented");
@@ -302,6 +338,17 @@ extern "C" PyObject* _PyLong_FromByteArray(const unsigned char* bytes, size_t n,
     BoxedLong* rtn = new BoxedLong();
     mpz_init(rtn->n);
     mpz_import(rtn->n, 1, 1, n, little_endian ? -1 : 1, 0, &bytes[0]);
+
+
+    RELEASE_ASSERT(little_endian, "");
+    if (is_signed && bytes[n - 1] >= 0x80) { // todo add big endian support
+        mpz_t t;
+        mpz_init(t);
+        mpz_setbit(t, n * 8);
+        mpz_sub(rtn->n, rtn->n, t);
+        mpz_clear(t);
+    }
+
     return rtn;
 }
 
@@ -367,6 +414,8 @@ BoxedLong* _longNew(Box* val, Box* _base) {
             const std::string& s = static_cast<BoxedString*>(val)->s;
             int r = mpz_init_set_str(rtn->n, s.c_str(), 10);
             RELEASE_ASSERT(r == 0, "");
+        } else if (val->cls == float_cls) {
+            mpz_init_set_si(rtn->n, static_cast<BoxedFloat*>(val)->d);
         } else {
             static const std::string long_str("__long__");
             Box* r = callattr(val, &long_str, CallattrFlags({.cls_only = true, .null_on_nonexistent = true }),

src/runtime/str.cpp

@@ -307,6 +307,10 @@ extern "C" Box* strAdd(BoxedString* lhs, Box* _rhs) {
     return new BoxedString(lhs->s + rhs->s);
 }
 
+extern "C" PyObject* PyString_InternFromString(const char* s) noexcept {
+    return new BoxedString(s);
+}
+
 /* Format codes
  * F_LJUST      '-'
  * F_SIGN       '+'
@@ -1523,6 +1527,16 @@ Box* strJoin(BoxedString* self, Box* rhs) {
     return boxString(std::move(output_str));
 }
 
+extern "C" PyObject* _PyString_Join(PyObject* sep, PyObject* x) noexcept {
+    try {
+        RELEASE_ASSERT(sep->cls == str_cls, "");
+        return strJoin((BoxedString*)sep, x);
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return NULL;
+    }
+}
+
 Box* strReplace(Box* _self, Box* _old, Box* _new, Box** _args) {
     if (_self->cls != str_cls)
         raiseExcHelper(TypeError, "descriptor 'replace' requires a 'str' object but received a '%s'",
@@ -2193,6 +2207,14 @@ static Py_ssize_t string_buffer_getsegcount(PyObject* o, Py_ssize_t* lenp) noexc
     return 1;
 }
 
+static Py_ssize_t string_buffer_getcharbuf(PyStringObject* self, Py_ssize_t index, const char** ptr) {
+    if (index != 0) {
+        PyErr_SetString(PyExc_SystemError, "accessing non-existent string segment");
+        return -1;
+    }
+    return string_buffer_getreadbuf((PyObject*)self, index, (const void**)ptr);
+}
+
 static int string_buffer_getbuffer(BoxedString* self, Py_buffer* view, int flags) noexcept {
     assert(self->cls == str_cls);
     return PyBuffer_FillInfo(view, (PyObject*)self, &self->s[0], self->s.size(), 1, flags);
@@ -2202,7 +2224,7 @@ static PyBufferProcs string_as_buffer = {
     (readbufferproc)string_buffer_getreadbuf, // comments are the only way I've found of
     (writebufferproc)NULL,                    // forcing clang-format to break these onto multiple lines
     (segcountproc)string_buffer_getsegcount,  //
-    (charbufferproc)NULL,                     //
+    (charbufferproc)string_buffer_getcharbuf, //
     (getbufferproc)string_buffer_getbuffer,   //
     (releasebufferproc)NULL,
 };

src/runtime/types.cpp

@@ -69,6 +69,7 @@ extern "C" void _PyUnicode_Init();
 extern "C" void initunicodedata();
 extern "C" void init_weakref();
 extern "C" void initcStringIO();
+extern "C" void init_io();
 
 namespace pyston {
 
@@ -1542,6 +1543,7 @@ void setupRuntime() {
     initunicodedata();
     init_weakref();
     initcStringIO();
+    init_io();
 
     // some additional setup to ensure weakrefs participate in our GC
     BoxedClass* weakref_ref_cls = &_PyWeakref_RefType;

src/runtime/types.h

@@ -84,6 +84,7 @@ extern BoxedClass* object_cls, *type_cls, *bool_cls, *int_cls, *long_cls, *float
     *builtin_function_or_method_cls;
 }
 #define unicode_cls (&PyUnicode_Type)
+#define memoryview_cls (&PyMemoryView_Type)
 
 extern "C" {
 extern Box* None, *NotImplemented, *True, *False;

test/tests/gzip_test.py

+# allow-warning: import level 0 will be treated as -1
+import gzip
+import io
+
+text = "Hello World!"
+
+# generate zip in memory
+outdata = io.BytesIO()
+timestamp = 1
+f = gzip.GzipFile(None, "wb", 9, outdata, timestamp)
+print f.write(text)
+f.close()
+
+output = outdata.getvalue()
+print output.encode("base64")
+
+# decode zip from memory
+f = gzip.GzipFile(None, "rb", 9, io.BytesIO(output))
+print f.read()
+f.close()

test/tests/io_test.py

+# allow-warning: import level 0 will be treated as -1
+import io
+filename = "io_test_.txt"
+
+f = io.open(filename, "w")
+print type(f)
+print f.isatty()
+print f.write(unicode("Hello World"))
+print f.tell()
+f.close()
+
+f = io.StringIO()
+print type(f)
+f.write(unicode('First line.\n'))
+f.write(unicode('Second line.\n'))
+print f.getvalue()
+print f.isatty()
+print f.tell()
+f.close()
+
+f = io.open(filename)
+print type(f)
+print f.read()
+f.close()

test/tests/long.py

@@ -45,6 +45,9 @@ except ValueError, e:
 print long("100", 16)
 print long("100", 10)
 print long("100", 26)
+print long(-1.1)
+print long(1.9)
+print long(-1.9)
 
 print type(hash(1L))
 print hash(1L) == hash(2L)