A number of improvements to CAPI compatibility

Separated out from some more involved changes that are
required to get the 'pwd' module working.

Makefile

@@ -276,7 +276,8 @@ STDLIB_OBJS := stdlib.bc.o stdlib.stripped.bc.o
 STDLIB_RELEASE_OBJS := stdlib.release.bc.o
 
 STDMODULE_SRCS := errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c $(EXTRA_STDMODULE_SRCS)
-FROM_CPYTHON_SRCS := $(addprefix lib_python/2.7_Modules/,$(STDMODULE_SRCS)) $(wildcard src/capi/*.c)
+STDOBJECT_SRCS := structseq.c $(EXTRA_STDOBJECT_SRCS)
+FROM_CPYTHON_SRCS := $(addprefix lib_python/2.7_Modules/,$(STDMODULE_SRCS)) $(addprefix lib_python/2.7_Objects/,$(STDOBJECT_SRCS)) $(wildcard src/capi/*.c)
 
 # The stdlib objects have slightly longer dependency chains,
 # so put them first in the list:

include/tupleobject.h

@@ -65,7 +65,7 @@ PyAPI_FUNC(void) _PyTuple_MaybeUntrack(PyObject *);
 //#define PyTuple_GET_ITEM(op, i) (((PyTupleObject *)(op))->ob_item[i])
 //#define PyTuple_GET_SIZE(op)    Py_SIZE(op)
 /* Macro, *only* to be used to fill in brand new tuples */
-#define PyTuple_SET_ITEM(op, i, v) PyTuple_SetItem(op, i, v)
+#define PyTuple_SET_ITEM(op, i, v) PyTuple_SetItem((PyObject*)op, i, v)
 //#define PyTuple_SET_ITEM(op, i, v) (((PyTupleObject *)(op))->ob_item[i] = v)
 
 PyAPI_FUNC(int) PyTuple_ClearFreeList(void);

src/capi/getargs.cpp

@@ -114,7 +114,17 @@ static int vgetargs1(PyObject* _tuple, const char* fmt, va_list* ap, int flags)
                         }
 
                         *p = arg;
+                    } else if (fmt && *fmt == '&') {
+                        // Copied from CPython:
+                        typedef int (*converter)(PyObject*, void*);
+                        converter convert = va_arg(*ap, converter);
+                        void* addr = va_arg(*ap, void*);
+                        fmt++;
+                        if (!(*convert)(arg, addr)) {
+                            Py_FatalError("unsupported error case");
+                        }
                     } else {
+                        RELEASE_ASSERT(*fmt != '?', "unsupported");
                         PyObject** p = (PyObject**)va_arg(*ap, PyObject**);
                         *p = arg;
                     }

src/capi/types.h

@@ -60,7 +60,7 @@ public:
         } else {
             RELEASE_ASSERT(0, "0x%x", self->ml_flags);
         }
-        assert(rtn);
+        RELEASE_ASSERT(rtn, "need to throw an exception");
         return rtn;
     }
 };

src/runtime/builtin_modules/builtins.cpp

@@ -451,7 +451,8 @@ extern "C" {
 BoxedClass* BaseException, *Exception, *StandardError, *AssertionError, *AttributeError, *GeneratorExit, *TypeError,
     *NameError, *KeyError, *IndexError, *IOError, *OSError, *ZeroDivisionError, *ValueError, *UnboundLocalError,
     *RuntimeError, *ImportError, *StopIteration, *Warning, *SyntaxError, *OverflowError, *DeprecationWarning,
-    *MemoryError, *LookupError, *EnvironmentError, *ArithmeticError, *BufferError, *KeyboardInterrupt, *SystemExit;
+    *MemoryError, *LookupError, *EnvironmentError, *ArithmeticError, *BufferError, *KeyboardInterrupt, *SystemExit,
+    *SystemError;
 }
 
 Box* exceptionNew1(BoxedClass* cls) {
@@ -744,6 +745,7 @@ void setupBuiltins() {
     /*NotImplementedError=*/makeBuiltinException(RuntimeError, "NotImplementedError");
     KeyboardInterrupt = makeBuiltinException(BaseException, "KeyboardInterrupt");
     SystemExit = makeBuiltinException(BaseException, "SystemExit");
+    SystemError = makeBuiltinException(StandardError, "SystemError");
 
     repr_obj = new BoxedFunction(boxRTFunction((void*)repr, UNKNOWN, 1));
     builtins_module->giveAttr("repr", repr_obj);

src/runtime/builtin_modules/posix.cpp

@@ -29,7 +29,7 @@ BoxedModule* posix_module;
 
 namespace posix {
 
-Box* urandom(Box* _n) {
+static Box* urandom(Box* _n) {
     RELEASE_ASSERT(_n->cls == int_cls, "");
 
     int64_t n = static_cast<BoxedInt*>(_n)->n;
@@ -52,12 +52,18 @@ Box* urandom(Box* _n) {
 
     return r;
 }
+
+static Box* posix_getuid() {
+    return boxInt(getuid());
 }
 
+} // namespace posix
+
 void setupPosix() {
     posix_module = createModule("posix", "__builtin__");
 
     posix_module->giveAttr("urandom", new BoxedFunction(boxRTFunction((void*)posix::urandom, STR, 1)));
+    posix_module->giveAttr("getuid", new BoxedFunction(boxRTFunction((void*)posix::posix_getuid, BOXED_INT, 0)));
 
     posix_module->giveAttr("error", OSError);
 }

src/runtime/capi.cpp

@@ -100,33 +100,6 @@ extern "C" {
 int Py_Py3kWarningFlag;
 }
 
-extern "C" PyObject* PyDict_New() {
-    return new BoxedDict();
-}
-
-extern "C" int PyDict_SetItem(PyObject* mp, PyObject* _key, PyObject* _item) {
-    Box* b = static_cast<Box*>(mp);
-    Box* key = static_cast<Box*>(_key);
-    Box* item = static_cast<Box*>(_item);
-
-    static std::string setitem_str("__setitem__");
-    Box* r;
-    try {
-        // TODO should demote GIL?
-        r = callattrInternal(b, &setitem_str, CLASS_ONLY, NULL, ArgPassSpec(2), key, item, NULL, NULL, NULL);
-    } catch (Box* b) {
-        fprintf(stderr, "Error: uncaught error would be propagated to C code!\n");
-        Py_FatalError("unimplemented");
-    }
-
-    RELEASE_ASSERT(r, "");
-    return 0;
-}
-
-extern "C" int PyDict_SetItemString(PyObject* mp, const char* key, PyObject* item) {
-    return PyDict_SetItem(mp, boxStrConstant(key), item);
-}
-
 BoxedClass* capifunc_cls;
 
 extern "C" void conservativeGCHandler(GCVisitor* v, Box* b) {
@@ -500,6 +473,10 @@ extern "C" PyObject* PyObject_GetIter(PyObject*) {
     Py_FatalError("unimplemented");
 }
 
+extern "C" PyObject* PyObject_Repr(PyObject*) {
+    Py_FatalError("unimplemented");
+}
+
 extern "C" PyObject* PyObject_GetAttr(PyObject* o, PyObject* attr_name) {
     if (!isSubclass(attr_name->cls, str_cls)) {
         PyErr_Format(PyExc_TypeError, "attribute name must be string, not '%.200s'", Py_TYPE(attr_name)->tp_name);

src/runtime/dict.cpp

@@ -118,6 +118,61 @@ Box* dictGetitem(BoxedDict* self, Box* k) {
     return pos;
 }
 
+extern "C" PyObject* PyDict_New() {
+    return new BoxedDict();
+}
+
+// We don't assume that dicts passed to PyDict are necessarily dicts, since there are a couple places
+// that we provide dict-like objects instead of proper dicts.
+// The performance should hopefully be comparable to the CPython fast case, since we can use
+// runtimeICs.
+extern "C" int PyDict_SetItem(PyObject* mp, PyObject* _key, PyObject* _item) {
+    ASSERT(mp->cls == dict_cls || mp->cls == attrwrapper_cls, "%s", getTypeName(mp)->c_str());
+
+    assert(mp);
+    Box* b = static_cast<Box*>(mp);
+    Box* key = static_cast<Box*>(_key);
+    Box* item = static_cast<Box*>(_item);
+
+    try {
+        // TODO should demote GIL?
+        setitem(b, key, item);
+    } catch (Box* b) {
+        abort();
+    }
+    return 0;
+}
+
+extern "C" int PyDict_SetItemString(PyObject* mp, const char* key, PyObject* item) {
+    Box* key_s;
+    try {
+        key_s = boxStrConstant(key);
+    } catch (Box* b) {
+        abort();
+    }
+
+    return PyDict_SetItem(mp, key_s, item);
+}
+
+extern "C" PyObject* PyDict_GetItem(PyObject* dict, PyObject* key) {
+    ASSERT(dict->cls == dict_cls || dict->cls == attrwrapper_cls, "%s", getTypeName(dict)->c_str());
+    try {
+        return getitem(dict, key);
+    } catch (Box* b) {
+        abort();
+    }
+}
+
+extern "C" PyObject* PyDict_GetItemString(PyObject* dict, const char* key) {
+    Box* key_s;
+    try {
+        key_s = boxStrConstant(key);
+    } catch (Box* b) {
+        abort();
+    }
+    return PyDict_GetItem(dict, key_s);
+}
+
 Box* dictSetitem(BoxedDict* self, Box* k, Box* v) {
     // printf("Starting setitem\n");
     Box*& pos = self->d[k];

src/runtime/long.cpp

@@ -68,8 +68,31 @@ extern "C" long PyLong_AsLong(PyObject* vv) {
     return mpz_get_si(l->n);
 }
 
-extern "C" long PyLong_AsLongAndOverflow(PyObject*, int*) {
-    Py_FatalError("unimplemented");
+extern "C" long PyLong_AsLongAndOverflow(Box* vv, int* overflow) {
+    // Ported from CPython; original comment:
+    /* This version by Tim Peters */
+
+    *overflow = 0;
+    if (vv == NULL) {
+        PyErr_BadInternalCall();
+        return -1;
+    }
+
+    if (PyInt_Check(vv))
+        return PyInt_AsLong(vv);
+
+    if (PyLong_Check(vv)) {
+        BoxedLong* l = static_cast<BoxedLong*>(vv);
+        if (mpz_fits_slong_p(l->n)) {
+            return mpz_get_si(l->n);
+        } else {
+            *overflow = mpz_sgn(l->n);
+            return -1;
+        }
+    }
+
+    // TODO CPython tries to go through tp_as_number
+    Py_FatalError("unsupported case");
 }
 
 extern "C" double PyLong_AsDouble(PyObject* vv) {

src/runtime/types.cpp

@@ -396,7 +396,7 @@ extern "C" {
 BoxedClass* object_cls, *type_cls, *none_cls, *bool_cls, *int_cls, *float_cls, *str_cls, *function_cls,
     *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls, *file_cls, *member_cls,
     *closure_cls, *generator_cls, *complex_cls, *basestring_cls, *unicode_cls, *property_cls, *staticmethod_cls,
-    *classmethod_cls;
+    *classmethod_cls, *attrwrapper_cls;
 
 
 BoxedTuple* EmptyTuple;
@@ -590,6 +590,20 @@ Box* sliceRepr(BoxedSlice* self) {
     return new BoxedString(std::move(s));
 }
 
+extern "C" bool PySlice_Check(PyObject*) {
+    Py_FatalError("unimplemented");
+}
+
+extern "C" int PySlice_GetIndices(PySliceObject* r, Py_ssize_t length, Py_ssize_t* start, Py_ssize_t* stop,
+                                  Py_ssize_t* step) {
+    Py_FatalError("unimplemented");
+}
+
+extern "C" int PySlice_GetIndicesEx(PySliceObject* r, Py_ssize_t length, Py_ssize_t* start, Py_ssize_t* stop,
+                                    Py_ssize_t* step, Py_ssize_t* slicelength) {
+    Py_FatalError("unimplemented");
+}
+
 Box* typeRepr(BoxedClass* self) {
     if (isUserDefined(self)) {
         std::ostringstream os;
@@ -647,7 +661,6 @@ CLFunction* unboxRTFunction(Box* b) {
 // A dictionary-like wrapper around the attributes array.
 // Not sure if this will be enough to satisfy users who expect __dict__
 // or PyModule_GetDict to return real dicts.
-BoxedClass* attrwrapper_cls;
 class AttrWrapper : public Box {
 private:
     Box* b;

src/runtime/types.h

@@ -81,7 +81,7 @@ extern "C" {
 extern BoxedClass* object_cls, *type_cls, *bool_cls, *int_cls, *long_cls, *float_cls, *str_cls, *function_cls,
     *none_cls, *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls, *file_cls, *xrange_cls,
     *member_cls, *method_cls, *closure_cls, *generator_cls, *complex_cls, *basestring_cls, *unicode_cls, *property_cls,
-    *staticmethod_cls, *classmethod_cls;
+    *staticmethod_cls, *classmethod_cls, *attrwrapper_cls;
 }
 extern "C" {
 extern Box* None, *NotImplemented, *True, *False;

test/tests/sort_small.py

 # run_args: -n
-# statcheck: stats.get('slowpath_getitem', 0) <= 20
-# statcheck: stats['slowpath_setitem'] <= 20
+# statcheck: noninit_count('slowpath_getitem') <= 20
+# statcheck: noninit_count('slowpath_setitem') <= 20
 
 def sort(l):
     n = len(l)

tools/tester.py

@@ -235,7 +235,7 @@ def run_test(fn, check_stats, run_memcheck):
 
     if check_stats:
         def noninit_count(s):
-            return stats[s] - stats.get("_init_" + s, 0)
+            return stats.get(s, 0) - stats.get("_init_" + s, 0)
 
         for l in statchecks:
             test = eval(l)