Commit e47bcc85 authored by Marius Wachtler's avatar Marius Wachtler

Merge commit 'ca7c5b86' into refcounting

Conflicts:
	src/runtime/types.cpp

Had todo some bigger changes to AttrWrapper and Box::setDictBacked
parents 9404c04a ca7c5b86
......@@ -690,7 +690,7 @@ public:
void setDictBacked(STOLEN(Box*) d);
// For instances with dict attrs:
BoxedDict* getDict();
void setDict(BoxedDict* d);
void setDict(STOLEN(BoxedDict*) d);
// Note, setattr does *not* steal a reference, but it probably should
......
......@@ -965,7 +965,7 @@ BoxedDict** Box::getDictPtr() {
return d_ptr;
}
void Box::setDict(BoxedDict* d) {
void Box::setDict(STOLEN(BoxedDict*) d) {
assert(0 && "check refcounting");
assert(cls->instancesHaveDictAttrs());
......@@ -6547,7 +6547,6 @@ Box* _typeNew(BoxedClass* metatype, BoxedString* name, BoxedTuple* bases, BoxedD
Py_DECREF(s);
}
} else {
assert(0 && "check refcounting");
Box* copy = PyDict_Copy(attr_dict);
RELEASE_ASSERT(copy, "");
made->setDictBacked(copy);
......
......@@ -1344,23 +1344,8 @@ static Box* typeSubDict(Box* obj, void* context) {
abort();
}
void Box::setDictBacked(STOLEN(Box*) val) {
assert(0 && "check refcounting");
assert(this->cls->instancesHaveHCAttrs());
RELEASE_ASSERT(val->cls == dict_cls || val->cls == attrwrapper_cls, "");
auto new_attr_list = (HCAttrs::AttrList*)PyMem_MALLOC(sizeof(HCAttrs::AttrList) + sizeof(Box*));
new_attr_list->attrs[0] = val;
HCAttrs* hcattrs = this->getHCAttrsPtr();
hcattrs->hcls = HiddenClass::dict_backed;
hcattrs->attr_list = new_attr_list;
}
static void typeSubSetDict(Box* obj, Box* val, void* context) {
assert(0 && "check refcounting");
static void typeSubSetDict(BORROWED(Box*) obj, BORROWED(Box*) val, void* context) {
Py_INCREF(val);
if (obj->cls->instancesHaveDictAttrs()) {
RELEASE_ASSERT(val->cls == dict_cls, "");
obj->setDict(static_cast<BoxedDict*>(val));
......@@ -1368,16 +1353,7 @@ static void typeSubSetDict(Box* obj, Box* val, void* context) {
}
if (obj->cls->instancesHaveHCAttrs()) {
RELEASE_ASSERT(PyDict_Check(val) || val->cls == attrwrapper_cls, "%s", val->cls->tp_name);
auto new_attr_list
= (HCAttrs::AttrList*)PyMem_MALLOC(sizeof(HCAttrs::AttrList) + sizeof(Box*));
new_attr_list->attrs[0] = val;
HCAttrs* hcattrs = obj->getHCAttrsPtr();
hcattrs->hcls = HiddenClass::dict_backed;
hcattrs->attr_list = new_attr_list;
obj->setDictBacked(val);
return;
}
......@@ -2144,35 +2120,45 @@ public:
// or PyModule_GetDict to return real dicts.
class AttrWrapper : public Box {
private:
// TODO: need to merge this with marius's change. this will need to become
// an owned reference.
BORROWED(Box*) b; // The parent object ('b') will keep the attrwrapper alive (forever)
// TODO: not sure if b should become owned (it would create a cycle)
// The fields are not not allowed to be set at the same time
BORROWED(Box* b); // set when this attribute wrapper wraps a Box (default mode)
BoxedDict* private_dict; // set when the attribute wrapper does not wrap a Box anymore because the Box.__dict__
// changed to another dict.
// This is useful when we assign non str_cls keys, because the attribute array only supports strs.
void convertToDictBacked() {
assert(0 && "check refcounting");
HCAttrs* attrs = this->b->getHCAttrsPtr();
if (attrs->hcls->type == HiddenClass::DICT_BACKED)
if (isDictBacked())
return;
BoxedDict* d = new BoxedDict();
RELEASE_ASSERT(attrs->hcls->type == HiddenClass::NORMAL || attrs->hcls->type == HiddenClass::SINGLETON, "");
for (const auto& p : attrs->hcls->getStrAttrOffsets()) {
d->d[p.first] = attrs->attr_list->attrs[p.second];
}
HCAttrs* attrs = this->b->getHCAttrsPtr();
assert(attrs->hcls->type != HiddenClass::DICT_BACKED);
BoxedDict* d = (BoxedDict*)AttrWrapper::copy(this);
b->clearAttrs();
HCAttrs* hcattrs = b->getHCAttrsPtr();
auto new_attr_list = (HCAttrs::AttrList*)PyMem_MALLOC(sizeof(HCAttrs::AttrList) + sizeof(Box*));
new_attr_list->attrs[0] = d;
b->setDictBacked(d);
hcattrs->hcls = HiddenClass::dict_backed;
hcattrs->attr_list = new_attr_list;
}
bool isDictBacked() { return b->getHCAttrsPtr()->hcls->type == HiddenClass::DICT_BACKED; }
bool isDictBacked() {
if (private_dict)
return private_dict;
return b->getHCAttrsPtr()->hcls->type == HiddenClass::DICT_BACKED;
}
Box* getDictBacking() {
assert(isDictBacked());
if (private_dict)
return private_dict;
return b->getHCAttrsPtr()->attr_list->attrs[0];
}
public:
AttrWrapper(Box* b) : b(b) {
AttrWrapper(Box* b) : b(b), private_dict(NULL) {
assert(b->cls->instancesHaveHCAttrs());
// We currently don't support creating an attrwrapper around a dict-backed object,
......@@ -2184,9 +2170,43 @@ public:
|| b->getHCAttrsPtr()->hcls->type == HiddenClass::SINGLETON);
}
void convertToPrivateDict() {
RELEASE_ASSERT(!private_dict, "");
RELEASE_ASSERT(b, "");
private_dict = (BoxedDict*)AttrWrapper::copy(this);
assert(PyDict_CheckExact(private_dict));
b->clearAttrs();
b = NULL;
}
DEFAULT_CLASS(attrwrapper_cls);
BORROWED(Box*) getUnderlying() { return b; }
BORROWED(Box*) getUnderlying() {
if (private_dict)
return private_dict;
return b;
}
static void dealloc(Box* b) noexcept {
if (_PyObject_GC_IS_TRACKED(b))
_PyObject_GC_UNTRACK(b);
AttrWrapper::tp_clear(b);
b->cls->tp_free(b);
}
static int traverse(PyObject* op, visitproc visit, void* arg) noexcept {
AttrWrapper* attr_wrapper = (AttrWrapper*)op;
Py_VISIT(attr_wrapper->private_dict);
return 0;
}
static int tp_clear(PyObject* op) noexcept {
AttrWrapper* attr_wrapper = (AttrWrapper*)op;
Py_CLEAR(attr_wrapper->private_dict);
return 0;
}
static Box* setitem(Box* _self, Box* _key, Box* value) {
RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
......@@ -2259,7 +2279,14 @@ public:
RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
AttrWrapper* self = static_cast<AttrWrapper*>(_self);
_key = coerceUnicodeToStr<CXX>(_key);
if (_key->cls != str_cls)
self->convertToDictBacked();
if (self->isDictBacked()) {
static BoxedString* get_str = getStaticString("get");
return callattrInternal<CXX, NOT_REWRITABLE>(self->getDictBacking(), get_str, LookupScope::CLASS_ONLY, NULL,
ArgPassSpec(2), _key, def, NULL, NULL, NULL);
}
RELEASE_ASSERT(_key->cls == str_cls, "");
BoxedString* key = static_cast<BoxedString*>(_key);
......@@ -2276,9 +2303,14 @@ public:
RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
AttrWrapper* self = static_cast<AttrWrapper*>(_self);
_key = coerceUnicodeToStr<S>(_key);
if (S == CAPI && !_key)
return NULL;
if (_key->cls != str_cls)
self->convertToDictBacked();
if (self->isDictBacked()) {
static BoxedString* getitem_str = getStaticString("__getitem__");
return callattrInternal<S, NOT_REWRITABLE>(self->getDictBacking(), getitem_str, LookupScope::CLASS_ONLY,
NULL, ArgPassSpec(1), _key, NULL, NULL, NULL, NULL);
}
RELEASE_ASSERT(_key->cls == str_cls, "");
BoxedString* key = static_cast<BoxedString*>(_key);
......@@ -2301,7 +2333,14 @@ public:
RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
AttrWrapper* self = static_cast<AttrWrapper*>(_self);
_key = coerceUnicodeToStr<CXX>(_key);
if (_key->cls != str_cls)
self->convertToDictBacked();
if (self->isDictBacked()) {
static BoxedString* pop_str = getStaticString("pop");
return callattrInternal<CXX, NOT_REWRITABLE>(self->getDictBacking(), pop_str, LookupScope::CLASS_ONLY, NULL,
ArgPassSpec(2), _key, default_, NULL, NULL, NULL);
}
RELEASE_ASSERT(_key->cls == str_cls, "");
BoxedString* key = static_cast<BoxedString*>(_key);
......@@ -2323,7 +2362,14 @@ public:
RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
AttrWrapper* self = static_cast<AttrWrapper*>(_self);
_key = coerceUnicodeToStr<CXX>(_key);
if (_key->cls != str_cls)
self->convertToDictBacked();
if (self->isDictBacked()) {
static BoxedString* delitem_str = getStaticString("__delitem__");
return callattrInternal<CXX, NOT_REWRITABLE>(self->getDictBacking(), delitem_str, LookupScope::CLASS_ONLY,
NULL, ArgPassSpec(1), _key, NULL, NULL, NULL, NULL);
}
RELEASE_ASSERT(_key->cls == str_cls, "");
BoxedString* key = static_cast<BoxedString*>(_key);
......@@ -2344,21 +2390,33 @@ public:
std::string O("");
llvm::raw_string_ostream os(O);
os << "attrwrapper({";
HCAttrs* attrs = self->b->getHCAttrsPtr();
RELEASE_ASSERT(attrs->hcls->type == HiddenClass::NORMAL || attrs->hcls->type == HiddenClass::SINGLETON, "");
bool first = true;
for (const auto& p : attrs->hcls->getStrAttrOffsets()) {
if (!first)
os << ", ";
first = false;
BoxedString* v = attrs->attr_list->attrs[p.second]->reprICAsString();
os << p.first->s() << ": " << v->s();
Py_DECREF(v);
os << "attrwrapper(";
if (self->isDictBacked()) {
static BoxedString* repr_str = getStaticString("__repr__");
Box* dict_repr
= callattrInternal<CXX, NOT_REWRITABLE>(self->getDictBacking(), repr_str, LookupScope::CLASS_ONLY, NULL,
ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
RELEASE_ASSERT(PyString_CheckExact(dict_repr), "");
os << ((BoxedString*)dict_repr)->s();
Py_DECREF(dict_repr);
} else {
HCAttrs* attrs = self->b->getHCAttrsPtr();
os << "{";
RELEASE_ASSERT(attrs->hcls->type == HiddenClass::NORMAL || attrs->hcls->type == HiddenClass::SINGLETON, "");
bool first = true;
for (const auto& p : attrs->hcls->getStrAttrOffsets()) {
if (!first)
os << ", ";
first = false;
BoxedString* v = attrs->attr_list->attrs[p.second]->reprICAsString();
os << p.first->s() << ": " << v->s();
Py_DECREF(v);
}
os << "}";
}
os << "})";
os << ")";
return boxString(os.str());
}
......@@ -2366,9 +2424,14 @@ public:
RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
AttrWrapper* self = static_cast<AttrWrapper*>(_self);
_key = coerceUnicodeToStr<S>(_key);
if (S == CAPI && !_key)
return NULL;
if (_key->cls != str_cls)
self->convertToDictBacked();
if (self->isDictBacked()) {
static BoxedString* contains_str = getStaticString("__contains__");
return callattrInternal<S, NOT_REWRITABLE>(self->getDictBacking(), contains_str, LookupScope::CLASS_ONLY,
NULL, ArgPassSpec(1), _key, NULL, NULL, NULL, NULL);
}
RELEASE_ASSERT(_key->cls == str_cls, "");
BoxedString* key = static_cast<BoxedString*>(_key);
......@@ -2397,6 +2460,13 @@ public:
RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
AttrWrapper* self = static_cast<AttrWrapper*>(_self);
if (self->isDictBacked()) {
static BoxedString* keys_str = getStaticString("keys");
return callattrInternal<CXX, NOT_REWRITABLE>(self->getDictBacking(), keys_str, LookupScope::CLASS_ONLY,
NULL, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
}
BoxedList* rtn = new BoxedList();
HCAttrs* attrs = self->b->getHCAttrsPtr();
......@@ -2411,6 +2481,12 @@ public:
RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
AttrWrapper* self = static_cast<AttrWrapper*>(_self);
if (self->isDictBacked()) {
static BoxedString* values_str = getStaticString("values");
return callattrInternal<CXX, NOT_REWRITABLE>(self->getDictBacking(), values_str, LookupScope::CLASS_ONLY,
NULL, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
}
BoxedList* rtn = new BoxedList();
HCAttrs* attrs = self->b->getHCAttrsPtr();
......@@ -2425,6 +2501,12 @@ public:
RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
AttrWrapper* self = static_cast<AttrWrapper*>(_self);
if (self->isDictBacked()) {
static BoxedString* items_str = getStaticString("items");
return callattrInternal<CXX, NOT_REWRITABLE>(self->getDictBacking(), items_str, LookupScope::CLASS_ONLY,
NULL, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
}
BoxedList* rtn = new BoxedList();
HCAttrs* attrs = self->b->getHCAttrsPtr();
......@@ -2455,6 +2537,12 @@ public:
RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
AttrWrapper* self = static_cast<AttrWrapper*>(_self);
if (self->isDictBacked()) {
static BoxedString* copy_str = getStaticString("copy");
return callattrInternal<CXX, NOT_REWRITABLE>(self->getDictBacking(), copy_str, LookupScope::CLASS_ONLY,
NULL, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
}
BoxedDict* rtn = new BoxedDict();
HCAttrs* attrs = self->b->getHCAttrsPtr();
......@@ -2470,6 +2558,15 @@ public:
RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
AttrWrapper* self = static_cast<AttrWrapper*>(_self);
if (self->isDictBacked()) {
static BoxedString* clear_str = getStaticString("clear");
auto rtn = callattrInternal<CXX, NOT_REWRITABLE>(self->getDictBacking(), clear_str, LookupScope::CLASS_ONLY,
NULL, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
assert(rtn == None);
Py_DECREF(rtn);
return;
}
HCAttrs* attrs = self->b->getHCAttrsPtr();
RELEASE_ASSERT(attrs->hcls->type == HiddenClass::NORMAL || attrs->hcls->type == HiddenClass::SINGLETON, "");
attrs->clear();
......@@ -2480,16 +2577,22 @@ public:
}
static Box* clear(Box* _self) {
RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
_clear(_self);
Py_RETURN_NONE;
}
static Box* len(Box* _self) {
RELEASE_ASSERT(_self->cls == attrwrapper_cls, "");
AttrWrapper* self = static_cast<AttrWrapper*>(_self);
if (self->isDictBacked()) {
static BoxedString* len_str = getStaticString("__len__");
return callattrInternal<CXX, NOT_REWRITABLE>(self->getDictBacking(), len_str, LookupScope::CLASS_ONLY, NULL,
ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
}
HCAttrs* attrs = self->b->getHCAttrsPtr();
RELEASE_ASSERT(attrs->hcls->type == HiddenClass::NORMAL || attrs->hcls->type == HiddenClass::SINGLETON, "");
return boxInt(attrs->hcls->getStrAttrOffsets().size());
......@@ -2504,9 +2607,17 @@ public:
RELEASE_ASSERT(args->size() <= 1, ""); // should throw a TypeError
if (self->isDictBacked()) {
static BoxedString* update_str = getStaticString("update");
return callattrInternal<CXX, NOT_REWRITABLE>(self->getDictBacking(), update_str, LookupScope::CLASS_ONLY,
NULL, ArgPassSpec(0, 0, true, true), args, kwargs, NULL, NULL,
NULL);
}
auto handle = [&](Box* _container) {
if (_container->cls == attrwrapper_cls) {
AttrWrapper* container = static_cast<AttrWrapper*>(_container);
RELEASE_ASSERT(!container->isDictBacked(), "not implemented");
HCAttrs* attrs = container->b->getHCAttrsPtr();
RELEASE_ASSERT(attrs->hcls->type == HiddenClass::NORMAL || attrs->hcls->type == HiddenClass::SINGLETON,
......@@ -2564,6 +2675,7 @@ public:
// In order to not have to reimplement dict cmp: just create a real dict for now and us it.
BoxedDict* dict = (BoxedDict*)AttrWrapper::copy(_self);
AUTO_DECREF(dict);
assert(dict->cls == dict_cls);
static BoxedString* eq_str = getStaticString("__eq__");
return callattrInternal<CXX, NOT_REWRITABLE>(dict, eq_str, LookupScope::CLASS_ONLY, NULL, ArgPassSpec(1),
......@@ -2623,6 +2735,9 @@ Box* Box::getAttrWrapper() {
HCAttrs* attrs = getHCAttrsPtr();
HiddenClass* hcls = attrs->hcls;
if (!hcls)
hcls = attrs->hcls = root_hcls;
if (hcls->type == HiddenClass::DICT_BACKED) {
return incref(attrs->attr_list->attrs[0]);
}
......@@ -2654,6 +2769,38 @@ BORROWED(Box*) unwrapAttrWrapper(Box* b) {
return static_cast<AttrWrapper*>(b)->getUnderlying();
}
void Box::setDictBacked(STOLEN(Box*) val) {
// this checks for: v.__dict__ = v.__dict__
if (val->cls == attrwrapper_cls && unwrapAttrWrapper(val) == this) {
Py_DECREF(val);
return;
}
assert(this->cls->instancesHaveHCAttrs());
HCAttrs* hcattrs = this->getHCAttrsPtr();
RELEASE_ASSERT(PyDict_Check(val) || val->cls == attrwrapper_cls, "");
// If there is an old attrwrapper it is not allowed to wrap the instance anymore instead it has to switch to a
// private dictonary.
// e.g.:
// a = v.__dict__
// v.__dict__ = {} # 'a' must switch now from wrapping 'v' to a the private dict.
int offset = hcattrs->hcls->type != HiddenClass::DICT_BACKED ? hcattrs->hcls->getAttrwrapperOffset() : -1;
if (offset != -1) {
AttrWrapper* wrapper = (AttrWrapper*)hcattrs->attr_list->attrs[offset];
RELEASE_ASSERT(wrapper->cls == attrwrapper_cls, "");
wrapper->convertToPrivateDict();
}
// assign the dict to the attribute list and switch to the dict backed strategy
auto new_attr_list = (HCAttrs::AttrList*)PyMem_MALLOC(sizeof(HCAttrs::AttrList) + sizeof(Box*));
new_attr_list->attrs[0] = val;
hcattrs->hcls = HiddenClass::dict_backed;
hcattrs->attr_list = new_attr_list;
}
Box* attrwrapperKeys(Box* b) {
return AttrWrapper::keys(b);
}
......@@ -4032,7 +4179,8 @@ void setupRuntime() {
pyston_getset_cls = new (0) BoxedClass(object_cls, 0, 0, sizeof(BoxedGetsetDescriptor), false, "getset_descriptor",
false, BoxedGetsetDescriptor::dealloc, NULL, false);
attrwrapper_cls = new (0)
BoxedClass(object_cls, 0, 0, sizeof(AttrWrapper), false, "attrwrapper", false, NULL, NULL, false);
BoxedClass(object_cls, 0, 0, sizeof(AttrWrapper), false, "attrwrapper", false, AttrWrapper::dealloc, NULL,
true, AttrWrapper::traverse, AttrWrapper::tp_clear);
dict_cls = new (0) BoxedClass(object_cls, 0, 0, sizeof(BoxedDict), false, "dict", true, BoxedDict::dealloc, NULL,
true, BoxedDict::traverse, BoxedDict::clear);
dict_cls->tp_flags |= Py_TPFLAGS_DICT_SUBCLASS;
......
......@@ -51,6 +51,19 @@ print hasattr(c, "attr")
print hasattr(c, "foo")
print c.__dict__ is d1
c = C()
a = c.__dict__
c.__dict__ = c.__dict__
c.__dict__[u"uni"] = "u"
c.__dict__[u"\u20ac"] = "u"
c.__dict__[(1, 2, 3)] = "t"
print sorted(c.__dict__.keys()), sorted(a.keys())
print "all non str attributes:", sorted([item for item in dir(c) if not isinstance(item, str)])
print "can we retrieve unicode attributes by there ascii value?", c.uni, c.__dict__["uni"]
print "attrwrapper:", a["uni"], a[(1, 2, 3)]
setattr(c, "uni", 1)
print "test setattr()", c.uni, c.__dict__["uni"], a["uni"], len(c.__dict__), len(a)
dictproxy = C.__dict__
print type(dictproxy)
print "foo" in dictproxy
......
# expected: fail
# - haven't bothered to implement this yet
# Funny case: if we get the attrwrapper of an object, then change it to be dict-backed,
# the original attrwrapper should remain valid but no longer connected to that object.
......@@ -13,3 +10,10 @@ c1.a = 1
print aw.items()
c1.__dict__ = d = {}
print aw.items()
c2 = C()
olddict = c2.__dict__
c2.__dict__ = { "should not be there" : 1 }
print olddict.has_key("should not be there")
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment