Merge pull request #715 from tjhance/test_iter_cpython

Get test_iter.py to work

from_cpython/Include/iterobject.h

@@ -7,6 +7,15 @@
 extern "C" {
 #endif
 
+// Pyston change: moved this from iterobject.c
+typedef struct {
+    PyObject_HEAD
+    PyObject *it_callable; /* Set to NULL when iterator is exhausted */
+    PyObject *it_sentinel; /* Set to NULL when iterator is exhausted */
+    // Pyston changes:
+    PyObject *it_nextvalue; /* Set to non-null when iterator is advanced in __hasnext__ */
+} calliterobject;
+
 // Pyston change: this is no longer a static object
 //PyAPI_DATA(PyTypeObject) PySeqIter_Type;
 

from_cpython/Lib/test/test_iter.py

-# expected: fail
 # Test iterators.
 
 import unittest

from_cpython/Objects/iterobject.c

@@ -4,15 +4,6 @@
 
 #include "Python.h"
 
-extern PyTypeObject PyCallIter_Type;
-typedef struct {
-    PyObject_HEAD
-    PyObject *it_callable; /* Set to NULL when iterator is exhausted */
-    PyObject *it_sentinel; /* Set to NULL when iterator is exhausted */
-    // Pyston changes:
-    PyObject *it_nextvalue; /* Set to non-null when iterator is advanced in __hasnext__ */
-} calliterobject;
-
 PyObject *
 PyCallIter_New(PyObject *callable, PyObject *sentinel)
 {
@@ -52,7 +43,7 @@ calliter_traverse(calliterobject *it, visitproc visit, void *arg)
 
 // Pyston change: extract most of the body of calliter_iternext here
 // so we can use it from both calliter_iternext and calliter_hasnext
-static PyObject *
+PyObject *
 calliter_next(calliterobject *it)
 {
     if (it->it_callable != NULL) {
@@ -98,24 +89,6 @@ calliter_iternext(calliterobject *it)
     return calliter_next(it);
 }
 
-// Pyston addition: __hasnext__ based iteration
-static int
-calliter_hasnext(calliterobject *it)
-{
-    if (!it->it_nextvalue) {
-        it->it_nextvalue = calliter_next(it);
-    }
-    return it->it_nextvalue != NULL;
-}
-
-// Pyston change: give iter objects a __hasnext__ method
-void
-PyCallIter_AddHasNext()
-{
-    PyCallIter_Type._tpp_hasnext = calliter_hasnext;
-}
-
-
 PyTypeObject PyCallIter_Type = {
     // Pyston change:
     PyVarObject_HEAD_INIT(NULL /* &PyType_Type */, 0)

src/capi/abstract.cpp

@@ -1542,13 +1542,11 @@ extern "C" int PySequence_DelSlice(PyObject* o, Py_ssize_t i1, Py_ssize_t i2) no
 }
 
 extern "C" Py_ssize_t PySequence_Count(PyObject* o, PyObject* value) noexcept {
-    fatalOrError(PyExc_NotImplementedError, "unimplemented");
-    return -1;
+    return _PySequence_IterSearch(o, value, PY_ITERSEARCH_COUNT);
 }
 
 extern "C" Py_ssize_t PySequence_Index(PyObject* o, PyObject* value) noexcept {
-    fatalOrError(PyExc_NotImplementedError, "unimplemented");
-    return -1;
+    return _PySequence_IterSearch(o, value, PY_ITERSEARCH_INDEX);
 }
 
 extern "C" PyObject* PyObject_CallFunction(PyObject* callable, const char* format, ...) noexcept {

src/codegen/compvars.cpp

@@ -1086,7 +1086,10 @@ public:
     }
 
     CompilerVariable* contains(IREmitter& emitter, const OpInfo& info, VAR* var, CompilerVariable* lhs) override {
-        return makeBool(false);
+        llvm::CallSite call
+            = emitter.createCall(info.unw_info, g.funcs.raiseNotIterableError, embedConstantPtr("int", g.i8_ptr));
+        call.setDoesNotReturn();
+        return new ConcreteCompilerVariable(BOOL, llvm::UndefValue::get(BOOL->llvmType()), true);
     }
 
     ConcreteCompilerType* getBoxType() override { return BOXED_INT; }
@@ -1330,7 +1333,10 @@ public:
     }
 
     CompilerVariable* contains(IREmitter& emitter, const OpInfo& info, VAR* var, CompilerVariable* lhs) override {
-        return makeBool(false);
+        llvm::CallSite call
+            = emitter.createCall(info.unw_info, g.funcs.raiseNotIterableError, embedConstantPtr("float", g.i8_ptr));
+        call.setDoesNotReturn();
+        return new ConcreteCompilerVariable(BOOL, llvm::UndefValue::get(BOOL->llvmType()), true);
     }
 
     ConcreteCompilerType* getBoxType() override { return BOXED_FLOAT; }
@@ -2059,7 +2065,10 @@ public:
     }
 
     CompilerVariable* contains(IREmitter& emitter, const OpInfo& info, VAR* var, CompilerVariable* lhs) override {
-        return makeBool(false);
+        llvm::CallSite call
+            = emitter.createCall(info.unw_info, g.funcs.raiseNotIterableError, embedConstantPtr("bool", g.i8_ptr));
+        call.setDoesNotReturn();
+        return new ConcreteCompilerVariable(BOOL, llvm::UndefValue::get(BOOL->llvmType()), true);
     }
 
     ConcreteCompilerType* getBoxType() override { return BOXED_BOOL; }

src/runtime/builtin_modules/builtins.cpp

@@ -598,6 +598,192 @@ Box* reduce(Box* f, Box* container, Box* initial) {
     return current;
 }
 
+// from cpython, bltinmodule.c
+PyObject* filterstring(PyObject* func, BoxedString* strobj) {
+    PyObject* result;
+    Py_ssize_t i, j;
+    Py_ssize_t len = PyString_Size(strobj);
+    Py_ssize_t outlen = len;
+
+    if (func == Py_None) {
+        /* If it's a real string we can return the original,
+         * as no character is ever false and __getitem__
+         * does return this character. If it's a subclass
+         * we must go through the __getitem__ loop */
+        if (PyString_CheckExact(strobj)) {
+            Py_INCREF(strobj);
+            return strobj;
+        }
+    }
+    if ((result = PyString_FromStringAndSize(NULL, len)) == NULL)
+        return NULL;
+
+    for (i = j = 0; i < len; ++i) {
+        PyObject* item;
+        int ok;
+
+        item = (*strobj->cls->tp_as_sequence->sq_item)(strobj, i);
+        if (item == NULL)
+            goto Fail_1;
+        if (func == Py_None) {
+            ok = 1;
+        } else {
+            PyObject* arg, *good;
+            arg = PyTuple_Pack(1, item);
+            if (arg == NULL) {
+                Py_DECREF(item);
+                goto Fail_1;
+            }
+            good = PyEval_CallObject(func, arg);
+            Py_DECREF(arg);
+            if (good == NULL) {
+                Py_DECREF(item);
+                goto Fail_1;
+            }
+            ok = PyObject_IsTrue(good);
+            Py_DECREF(good);
+        }
+        if (ok > 0) {
+            Py_ssize_t reslen;
+            if (!PyString_Check(item)) {
+                PyErr_SetString(PyExc_TypeError, "can't filter str to str:"
+                                                 " __getitem__ returned different type");
+                Py_DECREF(item);
+                goto Fail_1;
+            }
+            reslen = PyString_GET_SIZE(item);
+            if (reslen == 1) {
+                PyString_AS_STRING(result)[j++] = PyString_AS_STRING(item)[0];
+            } else {
+                /* do we need more space? */
+                Py_ssize_t need = j;
+
+                /* calculate space requirements while checking for overflow */
+                if (need > PY_SSIZE_T_MAX - reslen) {
+                    Py_DECREF(item);
+                    goto Fail_1;
+                }
+
+                need += reslen;
+
+                if (need > PY_SSIZE_T_MAX - len) {
+                    Py_DECREF(item);
+                    goto Fail_1;
+                }
+
+                need += len;
+
+                if (need <= i) {
+                    Py_DECREF(item);
+                    goto Fail_1;
+                }
+
+                need = need - i - 1;
+
+                assert(need >= 0);
+                assert(outlen >= 0);
+
+                if (need > outlen) {
+                    /* overallocate, to avoid reallocations */
+                    if (outlen > PY_SSIZE_T_MAX / 2) {
+                        Py_DECREF(item);
+                        return NULL;
+                    }
+
+                    if (need < 2 * outlen) {
+                        need = 2 * outlen;
+                    }
+                    if (_PyString_Resize(&result, need)) {
+                        Py_DECREF(item);
+                        return NULL;
+                    }
+                    outlen = need;
+                }
+                memcpy(PyString_AS_STRING(result) + j, PyString_AS_STRING(item), reslen);
+                j += reslen;
+            }
+        }
+        Py_DECREF(item);
+        if (ok < 0)
+            goto Fail_1;
+    }
+
+    if (j < outlen)
+        _PyString_Resize(&result, j);
+
+    return result;
+
+Fail_1:
+    Py_DECREF(result);
+    return NULL;
+}
+
+static PyObject* filtertuple(PyObject* func, PyObject* tuple) {
+    PyObject* result;
+    Py_ssize_t i, j;
+    Py_ssize_t len = PyTuple_Size(tuple);
+
+    if (len == 0) {
+        if (PyTuple_CheckExact(tuple))
+            Py_INCREF(tuple);
+        else
+            tuple = PyTuple_New(0);
+        return tuple;
+    }
+
+    if ((result = PyTuple_New(len)) == NULL)
+        return NULL;
+
+    for (i = j = 0; i < len; ++i) {
+        PyObject* item, *good;
+        int ok;
+
+        if (tuple->cls->tp_as_sequence && tuple->cls->tp_as_sequence->sq_item) {
+            item = tuple->cls->tp_as_sequence->sq_item(tuple, i);
+            if (item == NULL)
+                goto Fail_1;
+        } else {
+            PyErr_SetString(PyExc_TypeError, "filter(): unsubscriptable tuple");
+            goto Fail_1;
+        }
+        if (func == Py_None) {
+            Py_INCREF(item);
+            good = item;
+        } else {
+            PyObject* arg = PyTuple_Pack(1, item);
+            if (arg == NULL) {
+                Py_DECREF(item);
+                goto Fail_1;
+            }
+            good = PyEval_CallObject(func, arg);
+            Py_DECREF(arg);
+            if (good == NULL) {
+                Py_DECREF(item);
+                goto Fail_1;
+            }
+        }
+        ok = PyObject_IsTrue(good);
+        Py_DECREF(good);
+        if (ok > 0) {
+            if (PyTuple_SetItem(result, j++, item) < 0)
+                goto Fail_1;
+        } else {
+            Py_DECREF(item);
+            if (ok < 0)
+                goto Fail_1;
+        }
+    }
+
+    if (_PyTuple_Resize(&result, j) < 0)
+        return NULL;
+
+    return result;
+
+Fail_1:
+    Py_DECREF(result);
+    return NULL;
+}
+
 Box* filter2(Box* f, Box* container) {
     // If the filter-function argument is None, filter() works by only returning
     // the elements that are truthy.  This is equivalent to using the bool() constructor.
@@ -607,6 +793,24 @@ Box* filter2(Box* f, Box* container) {
     if (f == None)
         f = bool_cls;
 
+    // Special cases depending on the type of container influences the return type
+    // TODO There are other special cases like this
+    if (PyTuple_Check(container)) {
+        Box* rtn = filtertuple(f, static_cast<BoxedTuple*>(container));
+        if (!rtn) {
+            throwCAPIException();
+        }
+        return rtn;
+    }
+
+    if (PyString_Check(container)) {
+        Box* rtn = filterstring(f, static_cast<BoxedString*>(container));
+        if (!rtn) {
+            throwCAPIException();
+        }
+        return rtn;
+    }
+
     Box* rtn = new BoxedList();
     for (Box* e : container->pyElements()) {
         Box* r = runtimeCall(f, ArgPassSpec(1), e, NULL, NULL, NULL, NULL);
@@ -977,6 +1181,10 @@ Box* builtinIter(Box* obj, Box* sentinel) {
     if (sentinel == NULL)
         return getiter(obj);
 
+    if (!PyCallable_Check(obj)) {
+        raiseExcHelper(TypeError, "iter(v, w): v must be callable");
+    }
+
     Box* r = PyCallIter_New(obj, sentinel);
     if (!r)
         throwCAPIException();
@@ -1326,7 +1534,7 @@ void setupBuiltins() {
         "reduce", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)reduce, UNKNOWN, 3, 1, false, false), "reduce",
                                                    { NULL }));
     builtins_module->giveAttr("filter",
-                              new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)filter2, LIST, 2), "filter"));
+                              new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)filter2, UNKNOWN, 2), "filter"));
     builtins_module->giveAttr(
         "zip", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)zip, LIST, 0, 0, true, false), "zip"));
     builtins_module->giveAttr(

src/runtime/inline/list.cpp

@@ -39,21 +39,42 @@ Box* listiterHasnext(Box* s) {
     assert(s->cls == list_iterator_cls);
     BoxedListIterator* self = static_cast<BoxedListIterator*>(s);
 
-    return boxBool(self->pos < self->l->size);
+    if (!self->l) {
+        return False;
+    }
+
+    bool ans = (self->pos < self->l->size);
+    if (!ans) {
+        self->l = NULL;
+    }
+    return boxBool(ans);
 }
 
 i1 listiterHasnextUnboxed(Box* s) {
     assert(s->cls == list_iterator_cls);
     BoxedListIterator* self = static_cast<BoxedListIterator*>(s);
 
-    return self->pos < self->l->size;
+    if (!self->l) {
+        return false;
+    }
+
+    bool ans = (self->pos < self->l->size);
+    if (!ans) {
+        self->l = NULL;
+    }
+    return ans;
 }
 
 Box* listiterNext(Box* s) {
     assert(s->cls == list_iterator_cls);
     BoxedListIterator* self = static_cast<BoxedListIterator*>(s);
 
+    if (!self->l) {
+        raiseExcHelper(StopIteration, "");
+    }
+
     if (!(self->pos >= 0 && self->pos < self->l->size)) {
+        self->l = NULL;
         raiseExcHelper(StopIteration, "");
     }
 

src/runtime/iterobject.cpp

@@ -28,6 +28,8 @@
 #include "runtime/types.h"
 #include "runtime/util.h"
 
+extern "C" PyObject* calliter_next(calliterobject* it);
+
 namespace pyston {
 
 BoxedClass* seqiter_cls;
@@ -43,11 +45,19 @@ bool seqiterHasnextUnboxed(Box* s) {
     RELEASE_ASSERT(s->cls == seqiter_cls || s->cls == seqreviter_cls, "");
     BoxedSeqIter* self = static_cast<BoxedSeqIter*>(s);
 
+    if (!self->b) {
+        return false;
+    }
+
     Box* next;
     try {
         next = getitem(self->b, boxInt(self->idx));
     } catch (ExcInfo e) {
-        return false;
+        if (e.matches(IndexError) || e.matches(StopIteration)) {
+            self->b = NULL;
+            return false;
+        } else
+            throw e;
     }
     self->idx++;
     self->next = next;
@@ -58,11 +68,19 @@ Box* seqiterHasnext(Box* s) {
     RELEASE_ASSERT(s->cls == seqiter_cls || s->cls == seqreviter_cls, "");
     BoxedSeqIter* self = static_cast<BoxedSeqIter*>(s);
 
+    if (!self->b) {
+        return False;
+    }
+
     Box* next;
     try {
         next = getitem(self->b, boxInt(self->idx));
     } catch (ExcInfo e) {
-        return False;
+        if (e.matches(IndexError) || e.matches(StopIteration)) {
+            self->b = NULL;
+            return False;
+        } else
+            throw e;
     }
     self->idx++;
     self->next = next;
@@ -73,13 +91,17 @@ Box* seqreviterHasnext(Box* s) {
     RELEASE_ASSERT(s->cls == seqiter_cls || s->cls == seqreviter_cls, "");
     BoxedSeqIter* self = static_cast<BoxedSeqIter*>(s);
 
-    if (self->idx == -1)
+    if (self->idx == -1 || !self->b)
         return False;
     Box* next;
     try {
         next = getitem(self->b, boxInt(self->idx));
     } catch (ExcInfo e) {
-        return False;
+        if (e.matches(IndexError) || e.matches(StopIteration)) {
+            self->b = NULL;
+            return False;
+        } else
+            throw e;
     }
     self->idx--;
     self->next = next;
@@ -165,6 +187,18 @@ extern "C" PyObject* PySeqIter_New(PyObject* seq) noexcept {
     }
 }
 
+bool calliter_hasnext(Box* b) {
+    calliterobject* it = (calliterobject*)b;
+    if (!it->it_nextvalue) {
+        it->it_nextvalue = calliter_next(it);
+        if (PyErr_Occurred()) {
+            throwCAPIException();
+        }
+    }
+    return it->it_nextvalue != NULL;
+}
+
+
 void setupIter() {
     seqiter_cls
         = BoxedHeapClass::create(type_cls, object_cls, seqiterGCVisit, 0, 0, sizeof(BoxedSeqIter), false, "iterator");

src/runtime/iterobject.h

@@ -51,6 +51,8 @@ public:
     DEFAULT_CLASS(iterwrapper_cls);
 };
 
+bool calliter_hasnext(Box* b);
+
 void setupIter();
 }
 

src/runtime/objmodel.cpp

@@ -4230,7 +4230,7 @@ Box* compareInternal(Box* lhs, Box* rhs, int op_type, CompareRewriteArgs* rewrit
             if (result < 0)
                 throwCAPIException();
             assert(result == 0 || result == 1);
-            return boxBool(result);
+            return boxBool(op_type == AST_TYPE::NotIn ? !result : result);
         }
 
         if (rewrite_args) {
@@ -4893,8 +4893,12 @@ Box* getiter(Box* o) {
     // TODO add rewriting to this?  probably want to try to avoid this path though
     static BoxedString* iter_str = internStringImmortal("__iter__");
     Box* r = callattrInternal0(o, iter_str, LookupScope::CLASS_ONLY, NULL, ArgPassSpec(0));
-    if (r)
+    if (r) {
+        if (!PyIter_Check(r)) {
+            raiseExcHelper(TypeError, "iter() returned non-iterator of type '%s'", r->cls->tp_name);
+        }
         return r;
+    }
     return getiterHelper(o);
 }
 

src/runtime/types.cpp

@@ -2793,8 +2793,6 @@ inline void initUserAttrs(Box* obj, BoxedClass* cls) {
     }
 }
 
-extern "C" void PyCallIter_AddHasNext();
-
 extern "C" PyVarObject* PyObject_InitVar(PyVarObject* op, PyTypeObject* tp, Py_ssize_t size) noexcept {
     assert(op);
     assert(tp);
@@ -3476,8 +3474,10 @@ void setupRuntime() {
 
     PyType_Ready(&PyByteArrayIter_Type);
     PyType_Ready(&PyCapsule_Type);
-    PyCallIter_AddHasNext();
+
+    PyCallIter_Type.tpp_hasnext = calliter_hasnext;
     PyType_Ready(&PyCallIter_Type);
+
     PyType_Ready(&PyCObject_Type);
     PyType_Ready(&PyDictProxy_Type);