Merge pull request #1042 from undingen/minor_compat

fix minor compatibility issues

from_cpython/CMakeLists.txt

@@ -85,6 +85,7 @@ file(GLOB_RECURSE STDOBJECT_SRCS Objects
     capsule.c
     cobject.c
     complexobject.c
+    dictobject.c
     dictproxy.c
     exceptions.c
     floatobject.c

from_cpython/Include/dictobject.h

@@ -103,10 +103,11 @@ PyAPI_DATA(PyTypeObject) PyDict_Type;
 PyAPI_DATA(PyTypeObject) PyDictIterKey_Type;
 PyAPI_DATA(PyTypeObject) PyDictIterValue_Type;
 PyAPI_DATA(PyTypeObject) PyDictIterItem_Type;
+#endif
 PyAPI_DATA(PyTypeObject) PyDictKeys_Type;
 PyAPI_DATA(PyTypeObject) PyDictItems_Type;
 PyAPI_DATA(PyTypeObject) PyDictValues_Type;
-#endif
+
 PyAPI_DATA(PyTypeObject*) dict_cls;
 #define PyDict_Type (*dict_cls)
 PyAPI_DATA(PyTypeObject*) dictiterkey_cls;
@@ -115,12 +116,6 @@ PyAPI_DATA(PyTypeObject*) dictitervalue_cls;
 #define PyDictIterValue_Type (*dictitervalue_cls)
 PyAPI_DATA(PyTypeObject*) dictiteritem_cls;
 #define PyDictIterItem_Type (*dictiteritem_cls)
-PyAPI_DATA(PyTypeObject*) dictkeys_cls;
-#define PyDictKeys_Type (*dictkeys_cls)
-PyAPI_DATA(PyTypeObject*) dictitems_cls;
-#define PyDictItems_Type (*dictitems_cls)
-PyAPI_DATA(PyTypeObject*) dictvalues_cls;
-#define PyDictValues_Type (*dictvalues_cls)
 
 #define PyDict_Check(op) \
                  PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_DICT_SUBCLASS)

src/capi/typeobject.cpp

@@ -3427,6 +3427,9 @@ extern "C" int PyType_Ready(PyTypeObject* cls) noexcept {
     // RELEASE_ASSERT(cls->tp_traverse == NULL, "");
     // RELEASE_ASSERT(cls->tp_clear == NULL, "");
 
+    // set this flag early because some function check if it is set pretty early
+    cls->is_user_defined = true;
+
     assert(cls->attrs.hcls == NULL);
     new (&cls->attrs) HCAttrs(HiddenClass::makeSingleton());
 #define INITIALIZE(a) new (&(a)) decltype(a)
@@ -3510,7 +3513,6 @@ extern "C" int PyType_Ready(PyTypeObject* cls) noexcept {
         cls->gc_visit = &conservativeAndBasesGCHandler;
     else
         cls->gc_visit = &conservativeGCHandler;
-    cls->is_user_defined = true;
 
 
     if (!cls->instancesHaveHCAttrs() && cls->tp_base) {

src/runtime/dict.cpp

@@ -28,10 +28,11 @@
 
 namespace pyston {
 
-BoxedClass* dict_iterator_cls = NULL;
-BoxedClass* dict_keys_cls = NULL;
-BoxedClass* dict_values_cls = NULL;
-BoxedClass* dict_items_cls = NULL;
+extern "C" {
+BoxedClass* dictiterkey_cls = NULL;
+BoxedClass* dictitervalue_cls = NULL;
+BoxedClass* dictiteritem_cls = NULL;
+}
 
 Box* dictRepr(BoxedDict* self) {
     std::vector<char> chars;
@@ -148,33 +149,6 @@ extern "C" PyObject* PyDict_Items(PyObject* mp) noexcept {
     return dict_helper(mp, dictItems);
 }
 
-Box* dictViewKeys(BoxedDict* self) {
-    if (!PyDict_Check(self)) {
-        raiseExcHelper(TypeError, "descriptor 'viewkeys' requires a 'dict' object but received a '%s'",
-                       getTypeName(self));
-    }
-    BoxedDictView* rtn = new (dict_keys_cls) BoxedDictView(self);
-    return rtn;
-}
-
-Box* dictViewValues(BoxedDict* self) {
-    if (!PyDict_Check(self)) {
-        raiseExcHelper(TypeError, "descriptor 'viewvalues' requires a 'dict' object but received a '%s'",
-                       getTypeName(self));
-    }
-    BoxedDictView* rtn = new (dict_values_cls) BoxedDictView(self);
-    return rtn;
-}
-
-Box* dictViewItems(BoxedDict* self) {
-    if (!PyDict_Check(self)) {
-        raiseExcHelper(TypeError, "descriptor 'viewitems' requires a 'dict' object but received a '%s'",
-                       getTypeName(self));
-    }
-    BoxedDictView* rtn = new (dict_items_cls) BoxedDictView(self);
-    return rtn;
-}
-
 // Analoguous to CPython's, used for sq_ slots.
 static Py_ssize_t dict_length(PyDictObject* mp) {
     return ((BoxedDict*)mp)->d.size();
@@ -747,21 +721,12 @@ void BoxedDict::gcHandler(GCVisitor* v, Box* b) {
 }
 
 void BoxedDictIterator::gcHandler(GCVisitor* v, Box* b) {
-    assert(b->cls == dict_iterator_cls);
     Box::gcHandler(v, b);
 
     BoxedDictIterator* it = static_cast<BoxedDictIterator*>(b);
     v->visit(&it->d);
 }
 
-void BoxedDictView::gcHandler(GCVisitor* v, Box* b) {
-    assert(b->cls == dict_items_cls || b->cls == dict_values_cls || b->cls == dict_keys_cls);
-    Box::gcHandler(v, b);
-
-    BoxedDictView* view = static_cast<BoxedDictView*>(b);
-    v->visit(&view->d);
-}
-
 static int dict_init(PyObject* self, PyObject* args, PyObject* kwds) noexcept {
     assert(PyDict_Check(self));
     try {
@@ -788,24 +753,49 @@ void BoxedDict::dealloc(Box* b) noexcept {
     static_cast<BoxedDict*>(b)->d.freeAllMemory();
 }
 
+// We use cpythons dictview implementation from dictobject.c
+extern "C" PyObject* dictview_new(PyObject* dict, PyTypeObject* type) noexcept;
+Box* dictViewKeys(BoxedDict* d) {
+    Box* rtn = dictview_new(d, &PyDictKeys_Type);
+    if (!rtn)
+        throwCAPIException();
+    return rtn;
+}
+Box* dictViewValues(BoxedDict* d) {
+    Box* rtn = dictview_new(d, &PyDictValues_Type);
+    if (!rtn)
+        throwCAPIException();
+    return rtn;
+}
+Box* dictViewItems(BoxedDict* d) {
+    Box* rtn = dictview_new(d, &PyDictItems_Type);
+    if (!rtn)
+        throwCAPIException();
+    return rtn;
+}
+
+
+// This function gets called from dictobject.c
+extern "C" PyObject* dictiter_new(PyDictObject* dict, PyTypeObject* iter_type) noexcept {
+    return new (iter_type) BoxedDictIterator((BoxedDict*)dict);
+}
+
+
 void setupDict() {
     static PyMappingMethods dict_as_mapping;
     dict_cls->tp_as_mapping = &dict_as_mapping;
     static PySequenceMethods dict_as_sequence;
     dict_cls->tp_as_sequence = &dict_as_sequence;
 
-    dict_iterator_cls = BoxedClass::create(type_cls, object_cls, &BoxedDictIterator::gcHandler, 0, 0,
-                                           sizeof(BoxedDictIterator), false, "dictionary-itemiterator");
-
-    dict_keys_cls = BoxedClass::create(type_cls, object_cls, &BoxedDictView::gcHandler, 0, 0, sizeof(BoxedDictView),
-                                       false, "dict_keys");
-    dict_values_cls = BoxedClass::create(type_cls, object_cls, &BoxedDictView::gcHandler, 0, 0, sizeof(BoxedDictView),
-                                         false, "dict_values");
-    dict_items_cls = BoxedClass::create(type_cls, object_cls, &BoxedDictView::gcHandler, 0, 0, sizeof(BoxedDictView),
-                                        false, "dict_items");
+    dictiterkey_cls = BoxedClass::create(type_cls, object_cls, &BoxedDictIterator::gcHandler, 0, 0,
+                                         sizeof(BoxedDictIterator), false, "dictionary-keyiterator");
+    dictitervalue_cls = BoxedClass::create(type_cls, object_cls, &BoxedDictIterator::gcHandler, 0, 0,
+                                           sizeof(BoxedDictIterator), false, "dictionary-valueiterator");
+    dictiteritem_cls = BoxedClass::create(type_cls, object_cls, &BoxedDictIterator::gcHandler, 0, 0,
+                                          sizeof(BoxedDictIterator), false, "dictionary-itemiterator");
 
-    dict_iterator_cls->instances_are_nonzero = dict_keys_cls->instances_are_nonzero
-        = dict_values_cls->instances_are_nonzero = dict_items_cls->instances_are_nonzero = true;
+    dictiterkey_cls->instances_are_nonzero = dictitervalue_cls->instances_are_nonzero
+        = dictiteritem_cls->instances_are_nonzero = true;
 
     dict_cls->tp_dealloc = &BoxedDict::dealloc;
     dict_cls->has_safe_tp_dealloc = true;
@@ -819,7 +809,7 @@ void setupDict() {
     dict_cls->giveAttr("__ne__", new BoxedFunction(FunctionMetadata::create((void*)dictNe, UNKNOWN, 2)));
 
     dict_cls->giveAttr("__iter__", new BoxedFunction(FunctionMetadata::create((void*)dictIterKeys,
-                                                                              typeFromClass(dict_iterator_cls), 1)));
+                                                                              typeFromClass(dictiterkey_cls), 1)));
 
     dict_cls->giveAttr("update", new BoxedFunction(FunctionMetadata::create((void*)dictUpdate, NONE, 1, true, true)));
 
@@ -829,11 +819,11 @@ void setupDict() {
     dict_cls->giveAttr("has_key", new BoxedFunction(FunctionMetadata::create((void*)dictContains, BOXED_BOOL, 2)));
     dict_cls->giveAttr("items", new BoxedFunction(FunctionMetadata::create((void*)dictItems, LIST, 1)));
     dict_cls->giveAttr("iteritems", new BoxedFunction(FunctionMetadata::create((void*)dictIterItems,
-                                                                               typeFromClass(dict_iterator_cls), 1)));
+                                                                               typeFromClass(dictiteritem_cls), 1)));
 
     dict_cls->giveAttr("values", new BoxedFunction(FunctionMetadata::create((void*)dictValues, LIST, 1)));
     dict_cls->giveAttr("itervalues", new BoxedFunction(FunctionMetadata::create((void*)dictIterValues,
-                                                                                typeFromClass(dict_iterator_cls), 1)));
+                                                                                typeFromClass(dictitervalue_cls), 1)));
 
     dict_cls->giveAttr("keys", new BoxedFunction(FunctionMetadata::create((void*)dictKeys, LIST, 1)));
     dict_cls->giveAttr("iterkeys", dict_cls->getattr(internStringMortal("__iter__")));
@@ -868,15 +858,20 @@ void setupDict() {
 
     dict_cls->freeze();
 
-    FunctionMetadata* hasnext = FunctionMetadata::create((void*)dictIterHasnextUnboxed, BOOL, 1);
-    hasnext->addVersion((void*)dictIterHasnext, BOXED_BOOL);
-    dict_iterator_cls->giveAttr("__hasnext__", new BoxedFunction(hasnext));
-    dict_iterator_cls->giveAttr("__iter__", new BoxedFunction(FunctionMetadata::create(
-                                                (void*)dictIterIter, typeFromClass(dict_iterator_cls), 1)));
-    dict_iterator_cls->giveAttr("next", new BoxedFunction(FunctionMetadata::create((void*)dictIterNext, UNKNOWN, 1)));
-    dict_iterator_cls->freeze();
-    dict_iterator_cls->tp_iter = PyObject_SelfIter;
-    dict_iterator_cls->tp_iternext = dictiter_next;
+    // create the dictonary iterator types
+    for (BoxedClass* iter_type : { dictiterkey_cls, dictitervalue_cls, dictiteritem_cls }) {
+        FunctionMetadata* hasnext = FunctionMetadata::create((void*)dictIterHasnextUnboxed, BOOL, 1);
+        hasnext->addVersion((void*)dictIterHasnext, BOXED_BOOL);
+        iter_type->giveAttr("__hasnext__", new BoxedFunction(hasnext));
+        iter_type->giveAttr(
+            "__iter__", new BoxedFunction(FunctionMetadata::create((void*)dictIterIter, typeFromClass(iter_type), 1)));
+        iter_type->giveAttr("next", new BoxedFunction(FunctionMetadata::create((void*)dictIterNext, UNKNOWN, 1)));
+        iter_type->freeze();
+        iter_type->tp_iter = PyObject_SelfIter;
+        iter_type->tp_iternext = dictiter_next;
+        iter_type->tp_flags &= ~Py_TPFLAGS_BASETYPE; // subclassing is not allowed
+    }
+
 
     // Manually set some tp_* slots *after* calling freeze() -> fixup_slot_dispatchers().
     // fixup_slot_dispatchers will insert a wrapper like slot_tp_init into tp_init, which calls the python-level
@@ -899,15 +894,9 @@ void setupDict() {
 
     dict_cls->tp_as_sequence->sq_contains = (objobjproc)PyDict_Contains;
 
-    dict_keys_cls->giveAttr("__iter__", new BoxedFunction(FunctionMetadata::create(
-                                            (void*)dictViewKeysIter, typeFromClass(dict_iterator_cls), 1)));
-    dict_keys_cls->freeze();
-    dict_values_cls->giveAttr("__iter__", new BoxedFunction(FunctionMetadata::create(
-                                              (void*)dictViewValuesIter, typeFromClass(dict_iterator_cls), 1)));
-    dict_values_cls->freeze();
-    dict_items_cls->giveAttr("__iter__", new BoxedFunction(FunctionMetadata::create(
-                                             (void*)dictViewItemsIter, typeFromClass(dict_iterator_cls), 1)));
-    dict_items_cls->freeze();
+    PyType_Ready(&PyDictKeys_Type);
+    PyType_Ready(&PyDictValues_Type);
+    PyType_Ready(&PyDictItems_Type);
 }
 
 void teardownDict() {

src/runtime/dict.h

@@ -20,23 +20,13 @@
 
 namespace pyston {
 
-extern BoxedClass* dict_iterator_cls;
-extern BoxedClass* dict_keys_cls;
-extern BoxedClass* dict_values_cls;
-extern BoxedClass* dict_items_cls;
-
 class BoxedDictIterator : public Box {
 public:
-    enum IteratorType { KeyIterator, ValueIterator, ItemIterator };
-
     BoxedDict* d;
     BoxedDict::DictMap::iterator it;
     const BoxedDict::DictMap::iterator itEnd;
-    const IteratorType type;
 
-    BoxedDictIterator(BoxedDict* d, IteratorType type);
-
-    DEFAULT_CLASS(dict_iterator_cls);
+    BoxedDictIterator(BoxedDict* d);
 
     static void gcHandler(GCVisitor* v, Box* self);
 };
@@ -53,17 +43,7 @@ llvm_compat_bool dictIterHasnextUnboxed(Box* self);
 Box* dictiter_next(Box* self) noexcept;
 Box* dictIterNext(Box* self);
 
-class BoxedDictView : public Box {
-public:
-    BoxedDict* d;
-    BoxedDictView(BoxedDict* d);
-
-    static void gcHandler(GCVisitor* v, Box* self);
-};
 
-Box* dictViewKeysIter(Box* self);
-Box* dictViewValuesIter(Box* self);
-Box* dictViewItemsIter(Box* self);
 void dictMerge(BoxedDict* self, Box* other);
 Box* dictUpdate(BoxedDict* self, BoxedTuple* args, BoxedDict* kwargs);
 }

src/runtime/inline/dict.cpp

@@ -19,14 +19,13 @@
 
 namespace pyston {
 
-BoxedDictIterator::BoxedDictIterator(BoxedDict* d, IteratorType type)
-    : d(d), it(d->d.begin()), itEnd(d->d.end()), type(type) {
+BoxedDictIterator::BoxedDictIterator(BoxedDict* d) : d(d), it(d->d.begin()), itEnd(d->d.end()) {
 }
 
 Box* dict_iter(Box* s) noexcept {
     assert(PyDict_Check(s));
     BoxedDict* self = static_cast<BoxedDict*>(s);
-    return new BoxedDictIterator(self, BoxedDictIterator::KeyIterator);
+    return new (&PyDictIterKey_Type) BoxedDictIterator(self);
 }
 
 Box* dictIterKeys(Box* s) {
@@ -36,13 +35,13 @@ Box* dictIterKeys(Box* s) {
 Box* dictIterValues(Box* s) {
     assert(PyDict_Check(s));
     BoxedDict* self = static_cast<BoxedDict*>(s);
-    return new BoxedDictIterator(self, BoxedDictIterator::ValueIterator);
+    return new (&PyDictIterValue_Type) BoxedDictIterator(self);
 }
 
 Box* dictIterItems(Box* s) {
     assert(PyDict_Check(s));
     BoxedDict* self = static_cast<BoxedDict*>(s);
-    return new BoxedDictIterator(self, BoxedDictIterator::ItemIterator);
+    return new (&PyDictIterItem_Type) BoxedDictIterator(self);
 }
 
 Box* dictIterIter(Box* s) {
@@ -50,7 +49,6 @@ Box* dictIterIter(Box* s) {
 }
 
 llvm_compat_bool dictIterHasnextUnboxed(Box* s) {
-    assert(s->cls == dict_iterator_cls);
     BoxedDictIterator* self = static_cast<BoxedDictIterator*>(s);
 
     return self->it != self->itEnd;
@@ -61,19 +59,20 @@ Box* dictIterHasnext(Box* s) {
 }
 
 Box* dictiter_next(Box* s) noexcept {
-    assert(s->cls == dict_iterator_cls);
     BoxedDictIterator* self = static_cast<BoxedDictIterator*>(s);
 
     if (self->it == self->itEnd)
         return NULL;
 
     Box* rtn = nullptr;
-    if (self->type == BoxedDictIterator::KeyIterator) {
+    if (self->cls == &PyDictIterKey_Type) {
         rtn = self->it->first.value;
-    } else if (self->type == BoxedDictIterator::ValueIterator) {
+    } else if (self->cls == &PyDictIterValue_Type) {
         rtn = self->it->second;
-    } else if (self->type == BoxedDictIterator::ItemIterator) {
+    } else if (self->cls == &PyDictIterItem_Type) {
         rtn = BoxedTuple::create({ self->it->first.value, self->it->second });
+    } else {
+        RELEASE_ASSERT(0, "");
     }
     ++self->it;
     return rtn;
@@ -85,25 +84,4 @@ Box* dictIterNext(Box* s) {
         raiseExcHelper(StopIteration, "");
     return rtn;
 }
-
-BoxedDictView::BoxedDictView(BoxedDict* d) : d(d) {
-}
-
-Box* dictViewKeysIter(Box* s) {
-    assert(s->cls == dict_keys_cls);
-    BoxedDictView* self = static_cast<BoxedDictView*>(s);
-    return dictIterKeys(self->d);
-}
-
-Box* dictViewValuesIter(Box* s) {
-    assert(s->cls == dict_values_cls);
-    BoxedDictView* self = static_cast<BoxedDictView*>(s);
-    return dictIterValues(self->d);
-}
-
-Box* dictViewItemsIter(Box* s) {
-    assert(s->cls == dict_items_cls);
-    BoxedDictView* self = static_cast<BoxedDictView*>(s);
-    return dictIterItems(self->d);
-}
 }

src/runtime/types.cpp

@@ -2511,6 +2511,12 @@ public:
         return r ? True : False;
     }
 
+    static Box* hasKey(Box* _self, Box* _key) {
+        if (PyErr_WarnPy3k("dict.has_key() not supported in 3.x; use the in operator", 1) < 0)
+            throwCAPIException();
+        return contains<CXX>(_self, _key);
+    }
+
     static int sq_contains(Box* _self, Box* _key) noexcept {
         Box* rtn = contains<CAPI>(_self, _key);
         if (!rtn)
@@ -3837,8 +3843,10 @@ void setupRuntime() {
     attrwrapperiter_cls = BoxedClass::create(type_cls, object_cls, &AttrWrapperIter::gcHandler, 0, 0,
                                              sizeof(AttrWrapperIter), false, "attrwrapperiter");
 
-    pyston_getset_cls->giveAttr("__get__", new BoxedFunction(FunctionMetadata::create((void*)getsetGet, UNKNOWN, 3)));
-    capi_getset_cls->giveAttr("__get__", new BoxedFunction(FunctionMetadata::create((void*)getsetGet, UNKNOWN, 3)));
+    pyston_getset_cls->giveAttr("__get__",
+                                new BoxedFunction(FunctionMetadata::create((void*)getsetGet, UNKNOWN, 3), { None }));
+    capi_getset_cls->giveAttr("__get__",
+                              new BoxedFunction(FunctionMetadata::create((void*)getsetGet, UNKNOWN, 3), { None }));
     pyston_getset_cls->giveAttr("__set__", new BoxedFunction(FunctionMetadata::create((void*)getsetSet, UNKNOWN, 3)));
     capi_getset_cls->giveAttr("__set__", new BoxedFunction(FunctionMetadata::create((void*)getsetSet, UNKNOWN, 3)));
     pyston_getset_cls->giveAttr("__delete__",
@@ -4043,7 +4051,9 @@ void setupRuntime() {
     attrwrapper_cls->giveAttr("__str__",
                               new BoxedFunction(FunctionMetadata::create((void*)AttrWrapper::str, UNKNOWN, 1)));
     attrwrapper_cls->giveAttr(
-        "__contains__", new BoxedFunction(FunctionMetadata::create((void*)AttrWrapper::contains<CXX>, UNKNOWN, 2)));
+        "__contains__", new BoxedFunction(FunctionMetadata::create((void*)AttrWrapper::contains<CXX>, BOXED_BOOL, 2)));
+    attrwrapper_cls->giveAttr("has_key",
+                              new BoxedFunction(FunctionMetadata::create((void*)AttrWrapper::hasKey, BOXED_BOOL, 2)));
     attrwrapper_cls->giveAttr("__eq__",
                               new BoxedFunction(FunctionMetadata::create((void*)AttrWrapper::eq, UNKNOWN, 2)));
     attrwrapper_cls->giveAttr("__ne__",

test/tests/builtins.py

@@ -72,6 +72,7 @@ except TypeError, e:
 
 print globals().get("not a real variable")
 print globals().get("not a real variable", 1)
+print globals().has_key("C"), globals().has_key("CC")
 
 print hex(12345)
 print oct(234)

test/tests/capi_slots.py

@@ -44,6 +44,14 @@ print slots_test.SlotsTesterNum(0) == slots_test.SlotsTesterNum(1)
 for i in slots_test.SlotsTesterSeq(6):
     print i
 
+try:
+    # seqiter.tp_new is NULL so we should not be allowed to create an instance
+    slot_tester_seqiter = type(iter(slots_test.SlotsTesterSeq(6)))
+    print slot_tester_seqiter
+    slot_tester_seqiter()
+except Exception as e:
+    print e
+
 su = slots_test.SlotsTesterSub(5)
 print su
 

test/tests/descriptors_double.py

@@ -117,3 +117,7 @@ print type(f.at) # should not call __get__
 f.at = 12 # should not call __set__, should print nothing
 #TODO uncomment this:
 #print f.__dict__['at']
+
+
+# test if we support getset.__get__ with default args
+print type(5).__dict__["real"].__get__(42)