Merge pull request #783 from rudi-c/numpy_fix

Some work on the NumPy test

from_cpython/Include/Python.h

@@ -134,6 +134,13 @@ PyObject* PyGC_AddRoot(PyObject*) PYSTON_NOEXCEPT;
 //      PyGC_AddRoot(obj);
 // to reduce any chances of compiler reorderings or a GC somehow happening between the assignment
 // to the static slot and the call to PyGC_AddRoot.
+//
+
+// Similarly, some extension modules have static non-type objects that our GC thinks are errors
+// since they are not in the heap, unless we root them. This function should be used purely as
+// a temporary patching mechanism - we want to have a better way of dealing with such objects
+// in the future (even if it's an invalid use of CPython APIs).
+PyObject* PyGC_AddNonHeapRoot(PyObject* obj, int size) PYSTON_NOEXCEPT;
 
 // Pyston change : expose these type objects
 extern PyTypeObject Pattern_Type;

from_cpython/Include/pyconfig.h.in

@@ -23,6 +23,7 @@
 #define SIZEOF__BOOL 1
 #define SIZEOF_VOID_P 8
 #define SIZEOF_SIZE_T 8
+#define SIZEOF_SHORT 2
 #define SIZEOF_INT 4
 #define SIZEOF_LONG 8
 #define SIZEOF_LONG_LONG 8

src/analysis/scoping_analysis.cpp

@@ -524,6 +524,7 @@ public:
     // bool visit_classdef(AST_ClassDef *node) override { return false; }
     bool visit_continue(AST_Continue* node) override { return false; }
     bool visit_dict(AST_Dict* node) override { return false; }
+    bool visit_ellipsis(AST_Ellipsis* node) override { return false; }
     bool visit_excepthandler(AST_ExceptHandler* node) override { return false; }
     bool visit_expr(AST_Expr* node) override { return false; }
     bool visit_extslice(AST_ExtSlice* node) override { return false; }

src/capi/typeobject.cpp

@@ -3270,7 +3270,6 @@ extern "C" int PyType_Ready(PyTypeObject* cls) noexcept {
                        | Py_TPFLAGS_DICT_SUBCLASS | Py_TPFLAGS_BASE_EXC_SUBCLASS | Py_TPFLAGS_TYPE_SUBCLASS;
     RELEASE_ASSERT((cls->tp_flags & ~ALLOWABLE_FLAGS) == 0, "");
 
-    RELEASE_ASSERT(cls->tp_free == NULL || cls->tp_free == PyObject_Del || cls->tp_free == PyObject_GC_Del, "");
     RELEASE_ASSERT(cls->tp_is_gc == NULL, "");
     RELEASE_ASSERT(cls->tp_mro == NULL, "");
     RELEASE_ASSERT(cls->tp_cache == NULL, "");
@@ -3301,6 +3300,19 @@ extern "C" int PyType_Ready(PyTypeObject* cls) noexcept {
 
     assert(cls->tp_name);
 
+    // Inherit some special protocols. Normally methods are automatically inherited,
+    // when a Python class is declared, but that may not be the case with C extensions.
+    if (base != NULL) {
+        if (cls->tp_as_number == NULL)
+            cls->tp_as_number = base->tp_as_number;
+        if (cls->tp_as_sequence == NULL)
+            cls->tp_as_sequence = base->tp_as_sequence;
+        if (cls->tp_as_mapping == NULL)
+            cls->tp_as_mapping = base->tp_as_mapping;
+        if (cls->tp_as_buffer == NULL)
+            cls->tp_as_buffer = base->tp_as_buffer;
+    }
+
     if (cls->tp_call) {
         cls->tpp_call.capi_val = tppProxyToTpCall<CAPI>;
         cls->tpp_call.cxx_val = tppProxyToTpCall<CXX>;

src/codegen/parser.cpp

@@ -384,6 +384,12 @@ AST_DictComp* read_dictcomp(BufferedReader* reader) {
     return rtn;
 }
 
+AST_Ellipsis* read_ellipsis(BufferedReader* reader) {
+    AST_Ellipsis* rtn = new AST_Ellipsis();
+    rtn->col_offset = -1;
+    rtn->lineno = -1;
+    return rtn;
+}
 
 AST_ExceptHandler* read_excepthandler(BufferedReader* reader) {
     AST_ExceptHandler* rtn = new AST_ExceptHandler();
@@ -822,6 +828,8 @@ AST_expr* readASTExpr(BufferedReader* reader) {
             return read_dict(reader);
         case AST_TYPE::DictComp:
             return read_dictcomp(reader);
+        case AST_TYPE::Ellipsis:
+            return read_ellipsis(reader);
         case AST_TYPE::ExtSlice:
             return read_extslice(reader);
         case AST_TYPE::GeneratorExp:

src/codegen/serialize_ast.cpp

@@ -277,6 +277,7 @@ private:
         writeExpr(node->value);
         return true;
     }
+    virtual bool visit_ellipsis(AST_Ellipsis* node) { return true; }
     virtual bool visit_excepthandler(AST_ExceptHandler* node) {
         writeStmtVector(node->body);
         writeColOffset(node->col_offset);

src/core/ast.cpp

@@ -2034,6 +2034,10 @@ public:
         output->push_back(node);
         return false;
     }
+    virtual bool visit_ellipsis(AST_Ellipsis* node) {
+        output->push_back(node);
+        return false;
+    }
     virtual bool visit_excepthandler(AST_ExceptHandler* node) {
         output->push_back(node);
         return false;

src/core/cfg.cpp

@@ -855,6 +855,8 @@ private:
         return rtn;
     }
 
+    AST_expr* remapEllipsis(AST_Ellipsis* node) { return node; }
+
     AST_expr* remapExtSlice(AST_ExtSlice* node) {
         AST_ExtSlice* rtn = new AST_ExtSlice();
         rtn->lineno = node->lineno;
@@ -1183,6 +1185,9 @@ private:
             case AST_TYPE::DictComp:
                 rtn = remapScopedComprehension<AST_Dict>(ast_cast<AST_DictComp>(node));
                 break;
+            case AST_TYPE::Ellipsis:
+                rtn = remapEllipsis(ast_cast<AST_Ellipsis>(node));
+                break;
             case AST_TYPE::ExtSlice:
                 rtn = remapExtSlice(ast_cast<AST_ExtSlice>(node));
                 break;

src/gc/collector.cpp

@@ -243,6 +243,13 @@ extern "C" PyObject* PyGC_AddRoot(PyObject* obj) noexcept {
     return obj;
 }
 
+extern "C" PyObject* PyGC_AddNonHeapRoot(PyObject* obj, int size) noexcept {
+    if (obj) {
+        registerNonheapRootObject(obj, size);
+    }
+    return obj;
+}
+
 void registerNonheapRootObject(void* obj, int size) {
     // I suppose that things could work fine even if this were true, but why would it happen?
     assert(global_heap.getAllocationFromInteriorPointer(obj) == NULL);

src/runtime/complex.cpp

@@ -31,7 +31,19 @@ extern "C" Box* createPureImaginary(double i) {
 }
 
 extern "C" Py_complex PyComplex_AsCComplex(PyObject* op) noexcept {
+    // TODO: Incomplete v.s. CPython's implementation.
+    Py_complex cval;
+    if (PyComplex_Check(op)) {
+        cval.real = ((BoxedComplex*)op)->real;
+        cval.imag = ((BoxedComplex*)op)->imag;
+        return cval;
+    } else if (op->cls == int_cls) {
+        cval.real = ((BoxedInt*)op)->n;
+        cval.imag = 0.0;
+        return cval;
+    } else {
         Py_FatalError("unimplemented");
+    }
 }
 
 extern "C" double PyComplex_RealAsDouble(PyObject* op) noexcept {
@@ -255,6 +267,12 @@ Box* complexHash(BoxedComplex* self) {
     return boxInt(combined);
 }
 
+Box* complexAbs(BoxedComplex* self) {
+    assert(self->cls == complex_cls);
+    // TODO: CPython does a lot more safety checks.
+    return boxFloat(sqrt(self->real * self->real + self->imag * self->imag));
+}
+
 Box* complexStr(BoxedComplex* self) {
     assert(self->cls == complex_cls);
     return boxString(complexFmt(self->real, self->imag, 12, 'g'));
@@ -364,6 +382,7 @@ void setupComplex() {
 
     complex_cls->giveAttr("__pos__", new BoxedFunction(boxRTFunction((void*)complexPos, BOXED_COMPLEX, 1)));
     complex_cls->giveAttr("__hash__", new BoxedFunction(boxRTFunction((void*)complexHash, BOXED_INT, 1)));
+    complex_cls->giveAttr("__abs__", new BoxedFunction(boxRTFunction((void*)complexAbs, BOXED_FLOAT, 1)));
     complex_cls->giveAttr("__str__", new BoxedFunction(boxRTFunction((void*)complexStr, STR, 1)));
     complex_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)complexRepr, STR, 1)));
     complex_cls->giveAttr("real",

src/runtime/dict.cpp

@@ -174,6 +174,11 @@ Box* dictViewItems(BoxedDict* self) {
     return rtn;
 }
 
+// Analoguous to CPython's, used for sq_ slots.
+static Py_ssize_t dict_length(PyDictObject* mp) {
+    return ((BoxedDict*)mp)->d.size();
+}
+
 Box* dictLen(BoxedDict* self) {
     if (!isSubclass(self->cls, dict_cls))
         raiseExcHelper(TypeError, "descriptor '__len__' requires a 'dict' object but received a '%s'",
@@ -374,9 +379,7 @@ extern "C" PyObject* PyDict_GetItemString(PyObject* dict, const char* key) noexc
 }
 
 Box* dictSetitem(BoxedDict* self, Box* k, Box* v) {
-    // printf("Starting setitem\n");
     Box*& pos = self->d[k];
-    // printf("Got the pos\n");
 
     if (pos != NULL) {
         pos = v;
@@ -402,6 +405,22 @@ Box* dictDelitem(BoxedDict* self, Box* k) {
     return None;
 }
 
+// Analoguous to CPython's, used for sq_ slots.
+static int dict_ass_sub(PyDictObject* mp, PyObject* v, PyObject* w) noexcept {
+    try {
+        Box* res;
+        if (w == NULL) {
+            res = dictDelitem((BoxedDict*)mp, v);
+        } else {
+            res = dictSetitem((BoxedDict*)mp, v, w);
+        }
+        assert(res == None);
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+    }
+    return 0;
+}
+
 extern "C" int PyDict_DelItem(PyObject* op, PyObject* key) noexcept {
     ASSERT(isSubclass(op->cls, dict_cls) || op->cls == attrwrapper_cls, "%s", getTypeName(op));
     try {
@@ -820,7 +839,11 @@ void setupDict() {
     dict_cls->tp_init = dict_init;
     dict_cls->tp_repr = dict_repr;
 
+    dict_cls->tp_as_mapping->mp_length = (lenfunc)dict_length;
     dict_cls->tp_as_mapping->mp_subscript = (binaryfunc)dictGetitem<CAPI>;
+    dict_cls->tp_as_mapping->mp_ass_subscript = (objobjargproc)dict_ass_sub;
+
+    dict_cls->tp_as_sequence->sq_contains = (objobjproc)PyDict_Contains;
 
     dict_keys_cls->giveAttr(
         "__iter__", new BoxedFunction(boxRTFunction((void*)dictViewKeysIter, typeFromClass(dict_iterator_cls), 1)));

src/runtime/import.cpp

@@ -809,7 +809,7 @@ BoxedModule* importCExtension(const std::string& full_name, const std::string& l
     // Let the GC know about the static variables.
     uintptr_t bss_start = (uintptr_t)dlsym(handle, "__bss_start");
     uintptr_t bss_end = (uintptr_t)dlsym(handle, "_end");
-    RELEASE_ASSERT(bss_end - bss_start < 100000, "Large BSS section detected - there maybe something wrong");
+    RELEASE_ASSERT(bss_end - bss_start < 1000000, "Large BSS section detected - there maybe something wrong");
     // only track void* aligned memory
     bss_start = (bss_start + (sizeof(void*) - 1)) & ~(sizeof(void*) - 1);
     bss_end -= bss_end % sizeof(void*);

src/runtime/list.cpp

@@ -31,6 +31,8 @@
 
 namespace pyston {
 
+static int list_ass_slice(PyListObject* a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject* v);
+
 extern "C" int PyList_Append(PyObject* op, PyObject* newitem) noexcept {
     RELEASE_ASSERT(PyList_Check(op), "");
     try {
@@ -198,7 +200,7 @@ static Box* list_slice(Box* o, Py_ssize_t ilow, Py_ssize_t ihigh) noexcept {
     return (PyObject*)np;
 }
 
-extern "C" Box* listGetitemUnboxed(BoxedList* self, int64_t n) {
+static inline Box* listGetitemUnboxed(BoxedList* self, int64_t n) {
     assert(isSubclass(self->cls, list_cls));
     if (n < 0)
         n = self->size + n;
@@ -252,6 +254,17 @@ extern "C" Box* listGetslice(BoxedList* self, Box* boxedStart, Box* boxedStop) {
     return _listSlice(self, start, stop, 1, stop - start);
 }
 
+// Analoguous to CPython's, used for sq_ slots.
+static PyObject* list_item(PyListObject* a, Py_ssize_t i) noexcept {
+    try {
+        BoxedList* self = (BoxedList*)a;
+        return listGetitemUnboxed(self, i);
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return NULL;
+    }
+}
+
 template <ExceptionStyle S> Box* listGetitem(BoxedList* self, Box* slice) {
     if (S == CAPI) {
         try {
@@ -297,6 +310,22 @@ extern "C" Box* listSetitemInt(BoxedList* self, BoxedInt* slice, Box* v) {
     return listSetitemUnboxed(self, slice->n, v);
 }
 
+// Analoguous to CPython's, used for sq_ slots.
+static int list_ass_item(PyListObject* a, Py_ssize_t i, PyObject* v) {
+    PyObject* old_value;
+    if (i < 0 || i >= Py_SIZE(a)) {
+        PyErr_SetString(PyExc_IndexError, "list assignment index out of range");
+        return -1;
+    }
+    if (v == NULL)
+        return list_ass_slice(a, i, i + 1, v);
+    Py_INCREF(v);
+    old_value = a->ob_item[i];
+    a->ob_item[i] = v;
+    Py_DECREF(old_value);
+    return 0;
+}
+
 extern "C" int PyList_SetItem(PyObject* op, Py_ssize_t i, PyObject* newitem) noexcept {
     PyObject* olditem;
     PyObject** p;
@@ -449,7 +478,7 @@ int list_ass_ext_slice(BoxedList* self, PyObject* item, PyObject* value) {
     }
 }
 
-Box* listSetitemSliceInt64(BoxedList* self, i64 start, i64 stop, i64 step, Box* v) {
+static inline void listSetitemSliceInt64(BoxedList* self, i64 start, i64 stop, i64 step, Box* v) {
     RELEASE_ASSERT(step == 1, "step sizes must be 1 in this code path");
 
     boundSliceWithLength(&start, &stop, start, stop, self->size);
@@ -488,8 +517,6 @@ Box* listSetitemSliceInt64(BoxedList* self, i64 start, i64 stop, i64 step, Box*
     }
 
     self->size += delts;
-
-    return None;
 }
 
 extern "C" Box* listSetitemSlice(BoxedList* self, BoxedSlice* slice, Box* v) {
@@ -511,7 +538,14 @@ extern "C" Box* listSetitemSlice(BoxedList* self, BoxedSlice* slice, Box* v) {
         return None;
     }
 
-    return listSetitemSliceInt64(self, start, stop, step, v);
+    listSetitemSliceInt64(self, start, stop, step, v);
+    return None;
+}
+
+// Analoguous to CPython's, used for sq_ slots.
+static int list_ass_slice(PyListObject* a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject* v) {
+    listSetitemSliceInt64((BoxedList*)a, ilow, ihigh, 1, v);
+    return 0;
 }
 
 extern "C" Box* listSetslice(BoxedList* self, Box* boxedStart, Box* boxedStop, Box** args) {
@@ -522,7 +556,8 @@ extern "C" Box* listSetslice(BoxedList* self, Box* boxedStart, Box* boxedStop, B
     sliceIndex(boxedStart, &start);
     sliceIndex(boxedStop, &stop);
 
-    return listSetitemSliceInt64(self, start, stop, 1, value);
+    listSetitemSliceInt64(self, start, stop, 1, value);
+    return None;
 }
 
 extern "C" Box* listSetitem(BoxedList* self, Box* slice, Box* v) {
@@ -822,23 +857,33 @@ extern "C" Box* PyList_GetSlice(PyObject* a, Py_ssize_t ilow, Py_ssize_t ihigh)
     return listGetitemSlice<CAPI>(self, new BoxedSlice(boxInt(ilow), boxInt(ihigh), boxInt(1)));
 }
 
-Box* listContains(BoxedList* self, Box* elt) {
+static inline int list_contains_shared(BoxedList* self, Box* elt) {
+    assert(isSubclass(self->cls, list_cls));
+
     int size = self->size;
     for (int i = 0; i < size; i++) {
         Box* e = self->elts->elts[i];
 
         bool identity_eq = e == elt;
         if (identity_eq)
-            return True;
+            return true;
 
         int r = PyObject_RichCompareBool(e, elt, Py_EQ);
         if (r == -1)
             throwCAPIException();
 
         if (r)
-            return True;
+            return true;
     }
-    return False;
+    return false;
+}
+
+static int list_contains(PyListObject* a, PyObject* el) {
+    return list_contains_shared((BoxedList*)a, el);
+}
+
+Box* listContains(BoxedList* self, Box* elt) {
+    return boxBool(list_contains_shared(self, elt));
 }
 
 Box* listCount(BoxedList* self, Box* elt) {
@@ -1195,9 +1240,13 @@ void setupList() {
     list_cls->giveAttr("__hash__", None);
     list_cls->freeze();
 
-    list_cls->tp_as_sequence->sq_slice = list_slice;
-    list_cls->tp_as_sequence->sq_length = list_length;
     list_cls->tp_iter = listIter;
+    list_cls->tp_as_sequence->sq_length = list_length;
+    list_cls->tp_as_sequence->sq_item = (ssizeargfunc)list_item;
+    list_cls->tp_as_sequence->sq_slice = list_slice;
+    list_cls->tp_as_sequence->sq_ass_item = (ssizeobjargproc)list_ass_item;
+    list_cls->tp_as_sequence->sq_ass_slice = (ssizessizeobjargproc)list_ass_slice;
+    list_cls->tp_as_sequence->sq_contains = (objobjproc)list_contains;
 
     CLFunction* hasnext = boxRTFunction((void*)listiterHasnextUnboxed, BOOL, 1);
     addRTFunction(hasnext, (void*)listiterHasnext, BOXED_BOOL);

src/runtime/str.cpp

@@ -1660,6 +1660,7 @@ static Box* str_slice(Box* o, Py_ssize_t i, Py_ssize_t j) {
     return PyString_FromStringAndSize(a->data() + i, j - i);
 }
 
+// Analoguous to CPython's, used for sq_ slots.
 static Py_ssize_t str_length(Box* a) {
     return Py_SIZE(a);
 }
@@ -2062,14 +2063,14 @@ Box* strSwapcase(BoxedString* self) {
     return rtn;
 }
 
-Box* strContains(BoxedString* self, Box* elt) {
+static inline int string_contains_shared(BoxedString* self, Box* elt) {
     assert(PyString_Check(self));
 
     if (PyUnicode_Check(elt)) {
         int r = PyUnicode_Contains(self, elt);
         if (r < 0)
             throwCAPIException();
-        return boxBool(r);
+        return r;
     }
 
     if (!PyString_Check(elt))
@@ -2083,9 +2084,16 @@ Box* strContains(BoxedString* self, Box* elt) {
         found_idx = self->s().find(sub->s()[0]);
     else
         found_idx = self->s().find(sub->s());
-    if (found_idx == std::string::npos)
-        return False;
-    return True;
+    return (found_idx != std::string::npos);
+}
+
+// Analoguous to CPython's, used for sq_ slots.
+static int string_contains(PyObject* str_obj, PyObject* sub_obj) {
+    return string_contains_shared((BoxedString*)str_obj, sub_obj);
+}
+
+Box* strContains(BoxedString* self, Box* elt) {
+    return boxBool(string_contains_shared(self, elt));
 }
 
 // compares (a+a_pos, len) with (str)
@@ -2284,6 +2292,17 @@ Box* strEncode(BoxedString* self, Box* encoding, Box* error) {
     return result;
 }
 
+static PyObject* string_item(PyStringObject* self, register Py_ssize_t i) {
+    BoxedString* boxedString = (BoxedString*)self;
+
+    if (i < 0 || i >= boxedString->size()) {
+        raiseExcHelper(IndexError, "string index out of range");
+    }
+
+    char c = boxedString->s()[i];
+    return characters[c & UCHAR_MAX];
+}
+
 template <ExceptionStyle S> Box* strGetitem(BoxedString* self, Box* slice) {
     if (S == CAPI) {
         try {
@@ -2857,10 +2876,12 @@ void setupStr() {
     add_operators(str_cls);
     str_cls->freeze();
 
-    str_cls->tp_as_sequence->sq_slice = str_slice;
-    str_cls->tp_as_sequence->sq_length = str_length;
     str_cls->tp_iter = (decltype(str_cls->tp_iter))strIter;
     str_cls->tp_hash = (hashfunc)str_hash;
+    str_cls->tp_as_sequence->sq_length = str_length;
+    str_cls->tp_as_sequence->sq_item = (ssizeargfunc)string_item;
+    str_cls->tp_as_sequence->sq_slice = str_slice;
+    str_cls->tp_as_sequence->sq_contains = (objobjproc)string_contains;
 
     basestring_cls->giveAttr("__doc__",
                              boxString("Type basestring cannot be instantiated; it is the base for str and unicode."));

testsuite @ 1b28a3b4

-Subproject commit e1d16ce8922700ef6bde50589e84db3a5c095c2b
+Subproject commit 1b28a3b4c941c7b93fcc0c5174147641fdeadf30