Merge pull request #891 from undingen/iter_cleanup

Add missing tp_iter and tp_iternext functions

src/capi/typeobject.cpp

@@ -1125,13 +1125,7 @@ static PyObject* slot_tp_del(PyObject* self) noexcept {
 
 PyObject* slot_sq_item(PyObject* self, Py_ssize_t i) noexcept {
     STAT_TIMER(t0, "us_timer_slot_sqitem", SLOT_AVOIDABILITY(self));
-
-    try {
-        return getitem(self, boxInt(i));
-    } catch (ExcInfo e) {
-        setCAPIException(e);
-        return NULL;
-    }
+    return getitemInternal<CAPI>(self, boxInt(i), NULL);
 }
 
 /* Pyston change: static */ Py_ssize_t slot_sq_length(PyObject* self) noexcept {

src/runtime/builtin_modules/builtins.cpp

@@ -1636,7 +1636,7 @@ void setupBuiltins() {
     enumerate_cls->giveAttr("__hasnext__",
                             new BoxedFunction(boxRTFunction((void*)BoxedEnumerate::hasnext, BOXED_BOOL, 1)));
     enumerate_cls->freeze();
-    enumerate_cls->tp_iter = BoxedEnumerate::iter;
+    enumerate_cls->tp_iter = PyObject_SelfIter;
     builtins_module->giveAttr("enumerate", enumerate_cls);
 
 

src/runtime/dict.cpp

@@ -854,6 +854,7 @@ void setupDict() {
         "__iter__", new BoxedFunction(boxRTFunction((void*)dictIterIter, typeFromClass(dict_iterator_cls), 1)));
     dict_iterator_cls->giveAttr("next", new BoxedFunction(boxRTFunction((void*)dictIterNext, UNKNOWN, 1)));
     dict_iterator_cls->freeze();
+    dict_iterator_cls->tp_iter = PyObject_SelfIter;
     dict_iterator_cls->tp_iternext = dictiter_next;
 
     // Manually set some tp_* slots *after* calling freeze() -> fixup_slot_dispatchers().
@@ -869,6 +870,7 @@ void setupDict() {
     // subclass Python classes.
     dict_cls->tp_init = dict_init;
     dict_cls->tp_repr = dict_repr;
+    dict_cls->tp_iter = dict_iter;
 
     dict_cls->tp_as_mapping->mp_length = (lenfunc)dict_length;
     dict_cls->tp_as_mapping->mp_subscript = (binaryfunc)dictGetitem<CAPI>;

src/runtime/dict.h

@@ -43,6 +43,7 @@ public:
 
 Box* dictGetitem(BoxedDict* self, Box* k);
 
+Box* dict_iter(Box* s) noexcept;
 Box* dictIterKeys(Box* self);
 Box* dictIterValues(Box* self);
 Box* dictIterItems(Box* self);

src/runtime/inline/dict.cpp

@@ -23,12 +23,16 @@ BoxedDictIterator::BoxedDictIterator(BoxedDict* d, IteratorType type)
     : d(d), it(d->d.begin()), itEnd(d->d.end()), type(type) {
 }
 
-Box* dictIterKeys(Box* s) {
+Box* dict_iter(Box* s) noexcept {
     assert(PyDict_Check(s));
     BoxedDict* self = static_cast<BoxedDict*>(s);
     return new BoxedDictIterator(self, BoxedDictIterator::KeyIterator);
 }
 
+Box* dictIterKeys(Box* s) {
+    return dict_iter(s);
+}
+
 Box* dictIterValues(Box* s) {
     assert(PyDict_Check(s));
     BoxedDict* self = static_cast<BoxedDict*>(s);

src/runtime/inline/xrange.cpp

@@ -244,5 +244,6 @@ void setupXrange() {
     xrange_iterator_cls->freeze();
     xrange_iterator_cls->tpp_hasnext = BoxedXrangeIterator::xrangeIteratorHasnextUnboxed;
     xrange_iterator_cls->tp_iternext = BoxedXrangeIterator::xrangeIterator_next;
+    xrange_iterator_cls->tp_iter = PyObject_SelfIter;
 }
 }

src/runtime/iterobject.cpp

@@ -40,69 +40,82 @@ Box* seqiterIter(Box* s) {
     return s;
 }
 
-bool seqiterHasnextUnboxed(Box* s) {
+static Box* seqiterHasnext_capi(Box* s) noexcept {
     RELEASE_ASSERT(s->cls == seqiter_cls || s->cls == seqreviter_cls, "");
     BoxedSeqIter* self = static_cast<BoxedSeqIter*>(s);
 
     if (!self->b) {
-        return false;
+        return False;
     }
 
-    Box* next = getitemInternal<ExceptionStyle::CAPI>(self->b, boxInt(self->idx), NULL);
+    Box* next = PySequence_GetItem(self->b, self->idx);
     if (!next) {
         if (PyErr_ExceptionMatches(IndexError) || PyErr_ExceptionMatches(StopIteration)) {
             PyErr_Clear();
             self->b = NULL;
-            return false;
-        } else {
-            throwCAPIException();
+            return False;
         }
+        return NULL;
     }
 
     self->idx++;
     self->next = next;
-    return true;
+    return True;
 }
 
 Box* seqiterHasnext(Box* s) {
-    return boxBool(seqiterHasnextUnboxed(s));
+    Box* rtn = seqiterHasnext_capi(s);
+    if (!rtn)
+        throwCAPIException();
+    return rtn;
 }
 
-Box* seqreviterHasnext(Box* s) {
+bool seqiterHasnextUnboxed(Box* s) {
+    return unboxBool(seqiterHasnext(s));
+}
+
+Box* seqreviterHasnext_capi(Box* s) noexcept {
     RELEASE_ASSERT(s->cls == seqiter_cls || s->cls == seqreviter_cls, "");
     BoxedSeqIter* self = static_cast<BoxedSeqIter*>(s);
 
     if (self->idx == -1 || !self->b)
         return False;
-    Box* next;
-    try {
-        next = getitem(self->b, boxInt(self->idx));
-    } catch (ExcInfo e) {
-        if (e.matches(IndexError) || e.matches(StopIteration)) {
+
+    Box* next = PySequence_GetItem(self->b, self->idx);
+    if (!next) {
+        if (PyErr_ExceptionMatches(IndexError) || PyErr_ExceptionMatches(StopIteration)) {
+            PyErr_Clear();
             self->b = NULL;
             return False;
-        } else
-            throw e;
+        }
+        return NULL;
     }
     self->idx--;
     self->next = next;
     return True;
 }
 
-Box* seqiterNext(Box* s) {
+Box* seqreviterHasnext(Box* s) {
+    Box* rtn = seqreviterHasnext_capi(s);
+    if (!rtn)
+        throwCAPIException();
+    return rtn;
+}
+
+Box* seqiter_next(Box* s) noexcept {
     RELEASE_ASSERT(s->cls == seqiter_cls || s->cls == seqreviter_cls, "");
     BoxedSeqIter* self = static_cast<BoxedSeqIter*>(s);
 
     if (!self->next) {
         Box* hasnext = NULL;
         if (s->cls == seqiter_cls)
-            hasnext = seqiterHasnext(s);
+            hasnext = seqiterHasnext_capi(s);
         else if (s->cls == seqreviter_cls)
-            hasnext = seqreviterHasnext(s);
+            hasnext = seqreviterHasnext_capi(s);
         else
             RELEASE_ASSERT(0, "");
-        if (hasnext == False)
-            raiseExcHelper(StopIteration, (const char*)NULL);
+        if (hasnext != True)
+            return NULL;
     }
 
     RELEASE_ASSERT(self->next, "");
@@ -111,6 +124,13 @@ Box* seqiterNext(Box* s) {
     return r;
 }
 
+Box* seqiterNext(Box* s) {
+    Box* rtn = seqiter_next(s);
+    if (!rtn)
+        raiseExcHelper(StopIteration, (const char*)NULL);
+    return rtn;
+}
+
 void BoxedSeqIter::gcHandler(GCVisitor* v, Box* b) {
     assert(b->cls == seqiter_cls || b->cls == seqreviter_cls);
     Box::gcHandler(v, b);
@@ -149,16 +169,24 @@ Box* iterwrapperHasnext(Box* s) {
     return boxBool(iterwrapperHasnextUnboxed(s));
 }
 
-Box* iterwrapperNext(Box* s) {
+Box* iterwrapper_next(Box* s) noexcept {
     RELEASE_ASSERT(s->cls == iterwrapper_cls, "");
     BoxedIterWrapper* self = static_cast<BoxedIterWrapper*>(s);
 
-    RELEASE_ASSERT(self->next, "");
+    if (!self->next)
+        return NULL;
     Box* r = self->next;
     self->next = NULL;
     return r;
 }
 
+Box* iterwrapperNext(Box* s) {
+    Box* rtn = iterwrapper_next(s);
+    if (!rtn)
+        raiseExcHelper(StopIteration, (const char*)NULL);
+    return rtn;
+}
+
 extern "C" PyObject* PySeqIter_New(PyObject* seq) noexcept {
     try {
         return new BoxedSeqIter(seq, 0);
@@ -190,6 +218,8 @@ void setupIter() {
 
     seqiter_cls->freeze();
     seqiter_cls->tpp_hasnext = seqiterHasnextUnboxed;
+    seqiter_cls->tp_iter = PyObject_SelfIter;
+    seqiter_cls->tp_iternext = seqiter_next;
 
     seqreviter_cls = BoxedClass::create(type_cls, object_cls, &BoxedSeqIter::gcHandler, 0, 0, sizeof(BoxedSeqIter),
                                         false, "reversed");
@@ -199,6 +229,8 @@ void setupIter() {
     seqreviter_cls->giveAttr("__iter__", new BoxedFunction(boxRTFunction((void*)seqiterIter, UNKNOWN, 1)));
 
     seqreviter_cls->freeze();
+    seqreviter_cls->tp_iter = PyObject_SelfIter;
+    seqreviter_cls->tp_iternext = seqiter_next;
 
     iterwrapper_cls = BoxedClass::create(type_cls, object_cls, &BoxedIterWrapper::gcHandler, 0, 0,
                                          sizeof(BoxedIterWrapper), false, "iterwrapper");
@@ -209,5 +241,6 @@ void setupIter() {
 
     iterwrapper_cls->freeze();
     iterwrapper_cls->tpp_hasnext = iterwrapperHasnextUnboxed;
+    iterwrapper_cls->tp_iternext = iterwrapper_next;
 }
 }

src/runtime/list.cpp

@@ -1383,6 +1383,7 @@ void setupList() {
     list_iterator_cls->freeze();
     list_iterator_cls->tpp_hasnext = listiterHasnextUnboxed;
     list_iterator_cls->tp_iternext = listiter_next;
+    list_iterator_cls->tp_iter = PyObject_SelfIter;
 
     list_reverse_iterator_cls->giveAttr("__name__", boxString("listreverseiterator"));
 
@@ -1395,6 +1396,7 @@ void setupList() {
 
     list_reverse_iterator_cls->freeze();
     list_reverse_iterator_cls->tp_iternext = listreviter_next;
+    list_reverse_iterator_cls->tp_iter = PyObject_SelfIter;
 }
 
 void teardownList() {

src/runtime/objmodel.cpp

@@ -5229,8 +5229,7 @@ extern "C" Box* getPystonIter(Box* o) {
 }
 
 extern "C" Box* getiterHelper(Box* o) {
-    static BoxedString* getitem_str = internStringImmortal("__getitem__");
-    if (typeLookup(o->cls, getitem_str, NULL))
+    if (PySequence_Check(o))
         return new BoxedSeqIter(o, 0);
     raiseExcHelper(TypeError, "'%s' object is not iterable", getTypeName(o));
 }

src/runtime/set.cpp

@@ -699,6 +699,7 @@ void setupSet() {
     set_iterator_cls->giveAttr("next", new BoxedFunction(boxRTFunction((void*)setiteratorNext, UNKNOWN, 1)));
     set_iterator_cls->freeze();
     set_iterator_cls->tp_iternext = setiter_next;
+    set_iterator_cls->tp_iter = PyObject_SelfIter;
 
     set_cls->giveAttr("__new__", new BoxedFunction(boxRTFunction((void*)setNew, UNKNOWN, 2, false, false), { NULL }));
     set_cls->giveAttr("__init__", new BoxedFunction(boxRTFunction((void*)setInit, UNKNOWN, 2, false, false), { NULL }));

src/runtime/str.cpp

@@ -2834,6 +2834,7 @@ void setupStr() {
     str_iterator_cls->freeze();
     str_iterator_cls->tpp_hasnext = (BoxedClass::pyston_inquiry)BoxedStringIterator::hasnextUnboxed;
     str_iterator_cls->tp_iternext = BoxedStringIterator::next_capi;
+    str_iterator_cls->tp_iter = PyObject_SelfIter;
 
     str_cls->tp_as_buffer = &string_as_buffer;
     str_cls->tp_print = string_print;

src/runtime/tuple.cpp

@@ -712,6 +712,7 @@ void setupTuple() {
     tuple_iterator_cls->freeze();
     tuple_iterator_cls->tpp_hasnext = tupleiterHasnextUnboxed;
     tuple_iterator_cls->tp_iternext = tupleiter_next;
+    tuple_iterator_cls->tp_iter = PyObject_SelfIter;
 }
 
 void teardownTuple() {

src/runtime/types.cpp

@@ -2208,6 +2208,7 @@ public:
 
     static Box* hasnext(Box* _self);
     static Box* next(Box* _self);
+    static Box* next_capi(Box* _self) noexcept;
 };
 
 // A dictionary-like wrapper around the attributes array.
@@ -2523,7 +2524,7 @@ public:
         return None;
     }
 
-    static Box* iter(Box* _self) {
+    static Box* iter(Box* _self) noexcept {
         RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
         AttrWrapper* self = static_cast<AttrWrapper*>(_self);
 
@@ -2573,6 +2574,18 @@ Box* AttrWrapperIter::next(Box* _self) {
     return r;
 }
 
+Box* AttrWrapperIter::next_capi(Box* _self) noexcept {
+    RELEASE_ASSERT(_self->cls == attrwrapperiter_cls, "");
+    AttrWrapperIter* self = static_cast<AttrWrapperIter*>(_self);
+    RELEASE_ASSERT(self->hcls->type == HiddenClass::NORMAL || self->hcls->type == HiddenClass::SINGLETON, "");
+
+    if (self->it == self->hcls->getStrAttrOffsets().end())
+        return NULL;
+    Box* r = self->it->first;
+    ++self->it;
+    return r;
+}
+
 Box* Box::getAttrWrapper() {
     assert(cls->instancesHaveHCAttrs());
     HCAttrs* attrs = getHCAttrsPtr();
@@ -3854,11 +3867,14 @@ void setupRuntime() {
     attrwrapper_cls->giveAttr("update",
                               new BoxedFunction(boxRTFunction((void*)AttrWrapper::update, NONE, 1, true, true)));
     attrwrapper_cls->freeze();
+    attrwrapper_cls->tp_iter = AttrWrapper::iter;
 
     attrwrapperiter_cls->giveAttr("__hasnext__",
                                   new BoxedFunction(boxRTFunction((void*)AttrWrapperIter::hasnext, UNKNOWN, 1)));
     attrwrapperiter_cls->giveAttr("next", new BoxedFunction(boxRTFunction((void*)AttrWrapperIter::next, UNKNOWN, 1)));
     attrwrapperiter_cls->freeze();
+    attrwrapperiter_cls->tp_iter = PyObject_SelfIter;
+    attrwrapperiter_cls->tp_iternext = AttrWrapperIter::next_capi;
 
     setupBuiltins();
     _PyExc_Init();

test/tests/dict_setting.py

@@ -16,6 +16,7 @@ c2.b = 5
 
 def p():
     print sorted(c1.__dict__.items()), sorted(c2.__dict__.items())
+    print sorted(list(iter(c1.__dict__))), sorted(list(iter(c2.__dict__)))
 
 p()