Merge branch 'file'

Migrate to (a subset of) CPython's file implementation instead of our own.

src/gc/gc_alloc.cpp

@@ -42,7 +42,8 @@ extern "C" void* gc_compat_realloc(void* ptr, size_t sz) noexcept {
 }
 
 extern "C" void gc_compat_free(void* ptr) noexcept {
-    gc_free(ptr);
+    if (ptr)
+        gc_free(ptr);
 }
 
 // We may need to hook malloc as well.  For now, these definitions serve

src/runtime/builtin_modules/builtins.cpp

@@ -28,6 +28,7 @@
 #include "gc/collector.h"
 #include "runtime/capi.h"
 #include "runtime/classobj.h"
+#include "runtime/file.h"
 #include "runtime/ics.h"
 #include "runtime/import.h"
 #include "runtime/inline/xrange.h"
@@ -329,27 +330,8 @@ extern "C" Box* id(Box* arg) {
 
 Box* open(Box* arg1, Box* arg2) {
     assert(arg2);
-
-    if (arg1->cls != str_cls) {
-        fprintf(stderr, "TypeError: coercing to Unicode: need string of buffer, %s found\n", getTypeName(arg1));
-        raiseExcHelper(TypeError, "");
-    }
-    if (arg2->cls != str_cls) {
-        fprintf(stderr, "TypeError: coercing to Unicode: need string of buffer, %s found\n", getTypeName(arg2));
-        raiseExcHelper(TypeError, "");
-    }
-
-    const std::string& fn = static_cast<BoxedString*>(arg1)->s;
-    const std::string& mode = static_cast<BoxedString*>(arg2)->s;
-
-    FILE* f = fopen(fn.c_str(), mode.c_str());
-    if (!f) {
-        PyErr_SetFromErrnoWithFilename(IOError, fn.c_str());
-        checkAndThrowCAPIException();
-        abort(); // unreachable;
-    }
-
-    return new BoxedFile(f, fn, mode);
+    // This could be optimized quite a bit if it ends up being important:
+    return runtimeCall(file_cls, ArgPassSpec(2), arg1, arg2, NULL, NULL, NULL);
 }
 
 extern "C" Box* chr(Box* arg) {
@@ -872,6 +854,9 @@ public:
     static Box* __init__(BoxedEnvironmentError* self, Box* errno_, Box* strerror, Box** _args) {
         Box* filename = _args[0];
 
+        if (!errno_)
+            return None;
+
         RELEASE_ASSERT(isSubclass(self->cls, EnvironmentError), "");
 
         self->myerrno = errno_;
@@ -1038,8 +1023,8 @@ void setupBuiltins() {
 
     EnvironmentError->gc_visit = BoxedEnvironmentError::gcHandler;
     EnvironmentError->giveAttr(
-        "__init__",
-        new BoxedFunction(boxRTFunction((void*)BoxedEnvironmentError::__init__, NONE, 4, 1, false, false), { NULL }));
+        "__init__", new BoxedFunction(boxRTFunction((void*)BoxedEnvironmentError::__init__, NONE, 4, 3, false, false),
+                                      { NULL, NULL, NULL }));
     EnvironmentError->giveAttr(
         "errno", new BoxedMemberDescriptor(BoxedMemberDescriptor::OBJECT, offsetof(BoxedEnvironmentError, myerrno)));
     EnvironmentError->giveAttr("strerror", new BoxedMemberDescriptor(BoxedMemberDescriptor::OBJECT,

src/runtime/builtin_modules/sys.cpp

@@ -22,6 +22,7 @@
 #include "codegen/unwinding.h"
 #include "core/types.h"
 #include "gc/collector.h"
+#include "runtime/file.h"
 #include "runtime/inline/boxing.h"
 #include "runtime/int.h"
 #include "runtime/types.h"

src/runtime/capi.cpp

@@ -79,6 +79,10 @@ Box* BoxedWrapperDescriptor::__get__(BoxedWrapperDescriptor* self, Box* inst, Bo
     return new BoxedWrapperObject(self, inst);
 }
 
+extern "C" int PyObject_AsCharBuffer(PyObject* obj, const char** buffer, Py_ssize_t* buffer_len) noexcept {
+    Py_FatalError("unimplemented");
+}
+
 // copied from CPython's getargs.c:
 extern "C" int PyBuffer_FillInfo(Py_buffer* view, PyObject* obj, void* buf, Py_ssize_t len, int readonly,
                                  int flags) noexcept {

src/runtime/file.h

+// Copyright (c) 2014-2015 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef PYSTON_RUNTIME_FILE_H
+#define PYSTON_RUNTIME_FILE_H
+
+#include "core/types.h"
+#include "runtime/types.h"
+
+namespace pyston {
+
+class BoxedFile : public Box {
+public:
+    PyObject_HEAD FILE* f_fp;
+    PyObject* f_name;
+    PyObject* f_mode;
+    int (*f_close)(FILE*);
+    int f_softspace;    /* Flag used by 'print' command */
+    int f_binary;       /* Flag which indicates whether the file is
+                       open in binary (1) or text (0) mode */
+    char* f_buf;        /* Allocated readahead buffer */
+    char* f_bufend;     /* Points after last occupied position */
+    char* f_bufptr;     /* Current buffer position */
+    char* f_setbuf;     /* Buffer for setbuf(3) and setvbuf(3) */
+    int f_univ_newline; /* Handle any newline convention */
+    int f_newlinetypes; /* Types of newlines seen */
+    int f_skipnextlf;   /* Skip next \n */
+    PyObject* f_encoding;
+    PyObject* f_errors;
+#if 0
+    PyObject* weakreflist; /* List of weak references */
+#endif
+    int unlocked_count; /* Num. currently running sections of code
+                       using f_fp with the GIL released. */
+    int readable;
+    int writable;
+
+    BoxedFile(FILE* f, std::string fname, const char* fmode, int (*close)(FILE*) = fclose)
+        __attribute__((visibility("default")));
+
+    DEFAULT_CLASS(file_cls);
+};
+}
+
+#endif

src/runtime/objmodel.cpp

@@ -41,6 +41,7 @@
 #include "gc/heap.h"
 #include "runtime/capi.h"
 #include "runtime/classobj.h"
+#include "runtime/file.h"
 #include "runtime/float.h"
 #include "runtime/generator.h"
 #include "runtime/ics.h"
@@ -173,10 +174,11 @@ extern "C" bool softspace(Box* b, bool newval) {
     assert(b);
 
     if (isSubclass(b->cls, file_cls)) {
-        bool& ss = static_cast<BoxedFile*>(b)->softspace;
-        bool r = ss;
+        int& ss = static_cast<BoxedFile*>(b)->f_softspace;
+        int r = ss;
         ss = newval;
-        return r;
+        assert(r == 0 || r == 1);
+        return (bool)r;
     }
 
     bool r;

src/runtime/str.cpp

@@ -1833,6 +1833,23 @@ extern "C" int _PyString_Resize(PyObject** pv, Py_ssize_t newsize) noexcept {
     return 0;
 }
 
+extern "C" void PyString_Concat(register PyObject** pv, register PyObject* w) noexcept {
+    try {
+        if (*pv == NULL)
+            return;
+
+        if (w == NULL || !PyString_Check(*pv)) {
+            *pv = NULL;
+            return;
+        }
+
+        *pv = strAdd((BoxedString*)*pv, w);
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        *pv = NULL;
+    }
+}
+
 extern "C" void PyString_ConcatAndDel(register PyObject** pv, register PyObject* w) noexcept {
     Py_FatalError("unimplemented");
 }

src/runtime/types.cpp

@@ -27,6 +27,7 @@
 #include "gc/collector.h"
 #include "runtime/capi.h"
 #include "runtime/classobj.h"
+#include "runtime/file.h"
 #include "runtime/ics.h"
 #include "runtime/iterobject.h"
 #include "runtime/list.h"
@@ -1138,6 +1139,8 @@ void setupRuntime() {
 
     closure_cls->freeze();
 
+    setupCAPI();
+
     setupBool();
     setupInt();
     setupLong();
@@ -1205,8 +1208,6 @@ void setupRuntime() {
     setupThread();
     setupGC();
 
-    setupCAPI();
-
     PyType_Ready(&PyCapsule_Type);
 
     initerrno();

src/runtime/types.h

@@ -404,19 +404,6 @@ public:
 };
 extern "C" BoxedTuple* EmptyTuple;
 
-class BoxedFile : public Box {
-public:
-    FILE* f;
-    std::string fname;
-    std::string fmode;
-    bool closed;
-    bool softspace;
-    BoxedFile(FILE* f, std::string fname, std::string fmode) __attribute__((visibility("default")))
-    : f(f), fname(fname), fmode(fmode), closed(false), softspace(false) {}
-
-    DEFAULT_CLASS(file_cls);
-};
-
 struct PyHasher {
     size_t operator()(Box*) const;
 };

test/tests/file.py

@@ -39,6 +39,9 @@ with open('README.md') as f:
     print lines[:5]
     print lines[-5:]
 
+with open('README.md') as f:
+    print len(f.readlines())
+
 # Check that opening a non-existent file results in an IOError.
 try:
    f = open('this-should-definitely-not-exist.txt')
@@ -46,6 +49,6 @@ except IOError as e:
    print str(e)
 
 f = open("/dev/null", "w")
-f.write("hello world")
+print f.write("hello world")
 print f.flush()
-f.close()
+print f.close()

test/tests/thread_test.py

@@ -3,16 +3,20 @@ import time
 
 print type(allocate_lock())
 
+print_lock = allocate_lock()
+
 done = 0
 def run(arg):
     global done
-    print "in other thread!", arg
+    with print_lock:
+        print "in other thread!", arg
     done = 1
 
 
 print "starting!"
-t = start_new_thread(run, (5,))
-print type(t)
+with print_lock:
+    t = start_new_thread(run, (5,))
+    print type(t)
 
 while not done:
     time.sleep(0)