Change exceptions to hcattrs

The rest of our builtin types use our "hc attrs" (hidden-class-based attribute
storage), but we use the CPython implementation of exceptions which ask for
dict attributes.  So, expose our hcattrs via the C API and change the exceptions
classes to use it.

Hopefully we can eventually find some more general way of providing this feature
(ex for all string-keyed dicts) without needing these changes.

from_cpython/Include/Python.h

@@ -86,6 +86,36 @@
 #include "warnings.h"
 #include "weakrefobject.h"
 
+// Pyston additions:
+// These new APIS give access to our fast hidden-class-based attributes implementation.
+// Ideally in the future this will just be "storage strategy" of dicts and all Python
+// dicts will benefit from it, but for now classes have to explicitly opt-in to having
+// these kinds of attrs.
+struct _hcattrs {
+    char _data[16];
+};
+#ifndef _PYSTON_API
+typedef struct _hcattrs PyHcAttrs;
+#else
+namespace pyston {
+class HCAttrs;
+}
+typedef int PyHcAttrs;
+#endif
+PyAPI_FUNC(void) PyObject_InitHcAttrs(PyHcAttrs*) PYSTON_NOEXCEPT;
+PyAPI_FUNC(PyObject*) PyObject_GetAttrWrapper(PyObject*) PYSTON_NOEXCEPT;
+PyAPI_FUNC(void) PyType_RequestHcAttrs(PyTypeObject*, int offset) PYSTON_NOEXCEPT;
+// Sets a descriptor on the type so that the attrs are available via __dict__
+PyAPI_FUNC(void) PyType_GiveHcAttrsDictDescr(PyTypeObject*) PYSTON_NOEXCEPT;
+// These functions directly manipulate the hcattrs storage, bypassing any getattro
+// or descriptor logic.  This is the equivallent of callling PyDict_GetItemString
+// on an instance's dict.
+// These functions try to mimic the Dict versions as much as possible, so for example
+// the PyObject_GetHcAttrString function does not set an exception.
+PyAPI_FUNC(PyObject*) PyObject_GetHcAttrString(PyObject*, const char*) PYSTON_NOEXCEPT;
+PyAPI_FUNC(int) PyObject_SetHcAttrString(PyObject*, const char*, PyObject*) PYSTON_NOEXCEPT;
+PyAPI_FUNC(int) PyObject_DelHcAttrString(PyObject*, const char*) PYSTON_NOEXCEPT;
+
 #include "codecs.h"
 #include "pyerrors.h"
 

from_cpython/Include/pyerrors.h

@@ -10,14 +10,18 @@ extern "C" {
 
 typedef struct {
     PyObject_HEAD
-    PyObject *dict;
+    // Pyston change: changed from dict to hcattrs
+    // PyObject *dict;
+    struct _hcattrs hcattrs;
     PyObject *args;
     PyObject *message;
 } PyBaseExceptionObject;
 
 typedef struct {
     PyObject_HEAD
-    PyObject *dict;
+    // Pyston change: changed from dict to hcattrs
+    // PyObject *dict;
+    struct _hcattrs hcattrs;
     PyObject *args;
     PyObject *message;
     PyObject *msg;
@@ -31,7 +35,9 @@ typedef struct {
 #ifdef Py_USING_UNICODE
 typedef struct {
     PyObject_HEAD
-    PyObject *dict;
+    // Pyston change: changed from dict to hcattrs
+    // PyObject *dict;
+    struct _hcattrs hcattrs;
     PyObject *args;
     PyObject *message;
     PyObject *encoding;
@@ -44,7 +50,9 @@ typedef struct {
 
 typedef struct {
     PyObject_HEAD
-    PyObject *dict;
+    // Pyston change: changed from dict to hcattrs
+    // PyObject *dict;
+    struct _hcattrs hcattrs;
     PyObject *args;
     PyObject *message;
     PyObject *code;
@@ -52,7 +60,9 @@ typedef struct {
 
 typedef struct {
     PyObject_HEAD
-    PyObject *dict;
+    // Pyston change: changed from dict to hcattrs
+    // PyObject *dict;
+    struct _hcattrs hcattrs;
     PyObject *args;
     PyObject *message;
     PyObject *myerrno;
@@ -63,7 +73,9 @@ typedef struct {
 #ifdef MS_WINDOWS
 typedef struct {
     PyObject_HEAD
-    PyObject *dict;
+    // Pyston change: changed from dict to hcattrs
+    // PyObject *dict;
+    struct _hcattrs hcattrs;
     PyObject *args;
     PyObject *message;
     PyObject *myerrno;

from_cpython/Objects/exceptions.c

@@ -36,8 +36,8 @@ BaseException_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     self = (PyBaseExceptionObject *)type->tp_alloc(type, 0);
     if (!self)
         return NULL;
-    /* the dict is created on the fly in PyObject_GenericSetAttr */
-    self->message = self->dict = NULL;
+    self->message = NULL;
+    PyObject_InitHcAttrs(&self->hcattrs);
 
     self->args = PyTuple_New(0);
     if (!self->args) {
@@ -84,7 +84,7 @@ PyObject* PyErr_CreateExceptionInstance(PyObject* _type, PyObject* arg) {
         if (!self)
             return NULL;
 
-        self->dict = NULL;
+        PyObject_InitHcAttrs(&self->hcattrs);
         if (arg) {
             self->args = PyTuple_Pack(1, arg);
             if (!self->args)
@@ -115,7 +115,8 @@ PyObject* PyErr_CreateExceptionInstance(PyObject* _type, PyObject* arg) {
 static int
 BaseException_clear(PyBaseExceptionObject *self)
 {
-    Py_CLEAR(self->dict);
+    // Pyston change:
+    // Py_CLEAR(self->dict);
     Py_CLEAR(self->args);
     Py_CLEAR(self->message);
     return 0;
@@ -132,7 +133,8 @@ BaseException_dealloc(PyBaseExceptionObject *self)
 static int
 BaseException_traverse(PyBaseExceptionObject *self, visitproc visit, void *arg)
 {
-    Py_VISIT(self->dict);
+    // Pyston change:
+    // Py_VISIT(self->dict);
     Py_VISIT(self->args);
     Py_VISIT(self->message);
     return 0;
@@ -225,10 +227,16 @@ BaseException_repr(PyBaseExceptionObject *self)
 static PyObject *
 BaseException_reduce(PyBaseExceptionObject *self)
 {
+    /* Pyston change:
     if (self->args && self->dict)
         return PyTuple_Pack(3, Py_TYPE(self), self->args, self->dict);
     else
         return PyTuple_Pack(2, Py_TYPE(self), self->args);
+    */
+    PyObject* attr_wrapper = PyObject_GetAttrWrapper((PyObject*)self);
+    if (!attr_wrapper)
+        return NULL;
+    return PyTuple_Pack(3, Py_TYPE(self), self->args, attr_wrapper);
 }
 
 /*
@@ -299,6 +307,7 @@ static PySequenceMethods BaseException_as_sequence = {
     0                       /* sq_inplace_repeat; */
 };
 
+/* Pyston change: we use the standard dict descriptor for these types now
 static PyObject *
 BaseException_get_dict(PyBaseExceptionObject *self)
 {
@@ -327,6 +336,7 @@ BaseException_set_dict(PyBaseExceptionObject *self, PyObject *val)
     self->dict = val;
     return 0;
 }
+*/
 
 static PyObject *
 BaseException_get_args(PyBaseExceptionObject *self)
@@ -361,8 +371,7 @@ BaseException_get_message(PyBaseExceptionObject *self)
     PyObject *msg;
 
     /* if "message" is in self->dict, accessing a user-set message attribute */
-    if (self->dict &&
-        (msg = PyDict_GetItemString(self->dict, "message"))) {
+    if ((msg = PyObject_GetHcAttrString((PyObject*)self, "message"))) {
         Py_INCREF(msg);
         return msg;
     }
@@ -387,25 +396,18 @@ BaseException_set_message(PyBaseExceptionObject *self, PyObject *val)
 {
     /* if val is NULL, delete the message attribute */
     if (val == NULL) {
-        if (self->dict && PyDict_GetItemString(self->dict, "message")) {
-            if (PyDict_DelItemString(self->dict, "message") < 0)
+        if (PyObject_GetHcAttrString((PyObject*)self, "message")) {
+            if (PyObject_DelHcAttrString((PyObject*)self, "message") < 0)
                 return -1;
         }
         Py_CLEAR(self->message);
         return 0;
     }
 
-    /* else set it in __dict__, but may need to create the dict first */
-    if (self->dict == NULL) {
-        self->dict = PyDict_New();
-        if (!self->dict)
-            return -1;
-    }
-    return PyDict_SetItemString(self->dict, "message", val);
+    return PyObject_SetHcAttrString((PyObject*)self, "message", val);
 }
 
 static PyGetSetDef BaseException_getset[] = {
-    {"__dict__", (getter)BaseException_get_dict, (setter)BaseException_set_dict},
     {"args", (getter)BaseException_get_args, (setter)BaseException_set_args},
     {"message", (getter)BaseException_get_message,
             (setter)BaseException_set_message},
@@ -450,7 +452,7 @@ static PyTypeObject _PyExc_BaseException = {
     0,                          /* tp_dict */
     0,                          /* tp_descr_get */
     0,                          /* tp_descr_set */
-    offsetof(PyBaseExceptionObject, dict), /* tp_dictoffset */
+    /* Pyston change: offsetof(PyBaseExceptionObject, dict)*/ 0, /* tp_dictoffset */
     (initproc)BaseException_init, /* tp_init */
     0,                          /* tp_alloc */
     BaseException_new,          /* tp_new */
@@ -474,7 +476,7 @@ static PyTypeObject _PyExc_ ## EXCNAME = { \
     Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, \
     PyDoc_STR(EXCDOC), (traverseproc)BaseException_traverse, \
     (inquiry)BaseException_clear, 0, 0, 0, 0, 0, 0, 0, &_ ## EXCBASE, \
-    0, 0, 0, offsetof(PyBaseExceptionObject, dict), \
+    0, 0, 0, /* Pyston change: offsetof(PyBaseExceptionObject, dict) */ 0, \
     (initproc)BaseException_init, 0, BaseException_new,\
 }; \
 PyObject *PyExc_ ## EXCNAME = (PyObject *)&_PyExc_ ## EXCNAME
@@ -490,7 +492,7 @@ static PyTypeObject _PyExc_ ## EXCNAME = { \
     Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, \
     PyDoc_STR(EXCDOC), (traverseproc)EXCSTORE ## _traverse, \
     (inquiry)EXCSTORE ## _clear, 0, 0, 0, 0, 0, 0, 0, &_ ## EXCBASE, \
-    0, 0, 0, offsetof(Py ## EXCSTORE ## Object, dict), \
+    0, 0, 0, /* Pyston change: offsetof(Py ## EXCSTORE ## Object, dict) */ 0, \
     (initproc)EXCSTORE ## _init, 0, BaseException_new,\
 }; \
 PyObject *PyExc_ ## EXCNAME = (PyObject *)&_PyExc_ ## EXCNAME
@@ -507,7 +509,7 @@ static PyTypeObject _PyExc_ ## EXCNAME = { \
     PyDoc_STR(EXCDOC), (traverseproc)EXCSTORE ## _traverse, \
     (inquiry)EXCSTORE ## _clear, 0, 0, 0, 0, EXCMETHODS, \
     EXCMEMBERS, 0, &_ ## EXCBASE, \
-    0, 0, 0, offsetof(Py ## EXCSTORE ## Object, dict), \
+    0, 0, 0, /* Pyston change: offsetof(Py ## EXCSTORE ## Object, dict) */ 0, \
     (initproc)EXCSTORE ## _init, 0, BaseException_new,\
 }; \
 PyObject *PyExc_ ## EXCNAME = (PyObject *)&_PyExc_ ## EXCNAME
@@ -831,12 +833,17 @@ EnvironmentError_reduce(PyEnvironmentErrorObject *self)
     } else
         Py_INCREF(args);
 
+    /* Pyston change:
     if (self->dict)
         res = PyTuple_Pack(3, Py_TYPE(self), args, self->dict);
     else
         res = PyTuple_Pack(2, Py_TYPE(self), args);
     Py_DECREF(args);
-    return res;
+    return res; */
+    PyObject* attr_wrapper = PyObject_GetAttrWrapper((PyObject*)self);
+    if (!attr_wrapper)
+        return NULL;
+    return PyTuple_Pack(3, Py_TYPE(self), args, attr_wrapper);
 }
 
 
@@ -1743,7 +1750,7 @@ static PyTypeObject _PyExc_UnicodeEncodeError = {
     Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
     PyDoc_STR("Unicode encoding error."), (traverseproc)UnicodeError_traverse,
     (inquiry)UnicodeError_clear, 0, 0, 0, 0, 0, UnicodeError_members,
-    0, &_PyExc_UnicodeError, 0, 0, 0, offsetof(PyUnicodeErrorObject, dict),
+    0, &_PyExc_UnicodeError, 0, 0, 0, /* Pyston change: offsetof(PyUnicodeErrorObject, dict) */ 0,
     (initproc)UnicodeEncodeError_init, 0, BaseException_new,
 };
 PyObject *PyExc_UnicodeEncodeError = (PyObject *)&_PyExc_UnicodeEncodeError;
@@ -1828,7 +1835,7 @@ static PyTypeObject _PyExc_UnicodeDecodeError = {
     Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
     PyDoc_STR("Unicode decoding error."), (traverseproc)UnicodeError_traverse,
     (inquiry)UnicodeError_clear, 0, 0, 0, 0, 0, UnicodeError_members,
-    0, &_PyExc_UnicodeError, 0, 0, 0, offsetof(PyUnicodeErrorObject, dict),
+    0, &_PyExc_UnicodeError, 0, 0, 0, /* Pyston change: offsetof(PyUnicodeErrorObject, dict) */ 0,
     (initproc)UnicodeDecodeError_init, 0, BaseException_new,
 };
 PyObject *PyExc_UnicodeDecodeError = (PyObject *)&_PyExc_UnicodeDecodeError;
@@ -1926,7 +1933,7 @@ static PyTypeObject _PyExc_UnicodeTranslateError = {
     Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
     PyDoc_STR("Unicode translation error."), (traverseproc)UnicodeError_traverse,
     (inquiry)UnicodeError_clear, 0, 0, 0, 0, 0, UnicodeError_members,
-    0, &_PyExc_UnicodeError, 0, 0, 0, offsetof(PyUnicodeErrorObject, dict),
+    0, &_PyExc_UnicodeError, 0, 0, 0, /* Pyston change: offsetof(PyUnicodeErrorObject, dict) */ 0,
     (initproc)UnicodeTranslateError_init, 0, BaseException_new,
 };
 PyObject *PyExc_UnicodeTranslateError = (PyObject *)&_PyExc_UnicodeTranslateError;
@@ -2110,7 +2117,10 @@ _PyExc_Init(void)
 {
     PyObject *m, *bltinmod, *bdict;
 
+    PyType_RequestHcAttrs(&_PyExc_BaseException, offsetof(PyBaseExceptionObject, hcattrs));
     PRE_INIT(BaseException)
+    PyType_GiveHcAttrsDictDescr(&_PyExc_BaseException);
+
     PRE_INIT(Exception)
     PRE_INIT(StandardError)
     PRE_INIT(TypeError)

src/capi/typeobject.cpp

@@ -3262,6 +3262,18 @@ static Box* tppProxyToTpCall(Box* self, CallRewriteArgs* rewrite_args, ArgPassSp
     return r;
 }
 
+extern "C" void PyType_RequestHcAttrs(PyTypeObject* cls, int offset) noexcept {
+    assert(cls->attrs_offset == 0);
+    assert(cls->tp_dictoffset == 0);
+    cls->attrs_offset = offset;
+}
+
+extern "C" void PyType_GiveHcAttrsDictDescr(PyTypeObject* cls) noexcept {
+    static BoxedString* dict_str = internStringImmortal("__dict__");
+    assert(!cls->hasattr(dict_str));
+    cls->giveAttr(dict_str, dict_descr);
+}
+
 extern "C" int PyType_Ready(PyTypeObject* cls) noexcept {
     ASSERT(!cls->is_pyston_class, "should not call this on Pyston classes");
 
@@ -3389,11 +3401,10 @@ extern "C" int PyType_Ready(PyTypeObject* cls) noexcept {
         // e.g. CTypes does this.
         bool is_metaclass = PyType_IsSubtype(cls, type_cls);
         assert(!is_metaclass || cls->instancesHaveHCAttrs() || cls->instancesHaveDictAttrs());
-    } else {
-        // this should get automatically initialized to 0 on this path:
-        assert(cls->attrs_offset == 0);
     }
 
+    assert(!(cls->instancesHaveHCAttrs() && cls->instancesHaveDictAttrs()));
+
     if (Py_TPFLAGS_BASE_EXC_SUBCLASS & cls->tp_flags) {
         exception_types.push_back(cls);
     }

src/core/types.h

@@ -505,6 +505,7 @@ public:
 
     HCAttrs(HiddenClass* hcls = root_hcls) : hcls(hcls), attr_list(nullptr) {}
 };
+static_assert(sizeof(HCAttrs) == sizeof(struct _hcattrs), "");
 
 class BoxedDict;
 class BoxedString;

src/runtime/types.cpp

@@ -2480,6 +2480,10 @@ Box* Box::getAttrWrapper() {
     return attrs->attr_list->attrs[offset];
 }
 
+extern "C" PyObject* PyObject_GetAttrWrapper(PyObject* obj) noexcept {
+    return obj->getAttrWrapper();
+}
+
 Box* unwrapAttrWrapper(Box* b) {
     assert(b->cls == attrwrapper_cls);
     return static_cast<AttrWrapper*>(b)->getUnderlying();
@@ -3000,6 +3004,26 @@ inline void initUserAttrs(Box* obj, BoxedClass* cls) {
     }
 }
 
+extern "C" void PyObject_InitHcAttrs(HCAttrs* attrs) noexcept {
+    new ((void*)attrs) HCAttrs();
+}
+
+extern "C" PyObject* PyObject_GetHcAttrString(PyObject* obj, const char* attr) PYSTON_NOEXCEPT {
+    return obj->getattr(internStringMortal(attr));
+}
+extern "C" int PyObject_SetHcAttrString(PyObject* obj, const char* attr, PyObject* val) PYSTON_NOEXCEPT {
+    obj->setattr(internStringMortal(attr), val, NULL);
+    return 0;
+}
+extern "C" int PyObject_DelHcAttrString(PyObject* obj, const char* attr) PYSTON_NOEXCEPT {
+    BoxedString* attr_str = internStringMortal(attr);
+    bool has = obj->hasattr(attr_str);
+    if (!has)
+        return -1;
+    obj->delattr(attr_str, NULL);
+    return 0;
+}
+
 extern "C" PyVarObject* PyObject_InitVar(PyVarObject* op, PyTypeObject* tp, Py_ssize_t size) noexcept {
     assert(op);
     assert(tp);

test/tests/exceptions_basic.py

@@ -104,7 +104,7 @@ f14()
 
 def test_set_state():
     exc = BaseException()
-    print exc.__dict__
+    print sorted(exc.__dict__.items())
     attrs = {"x": 1, "y": 2}
     exc.__setstate__(attrs)
     print exc.__dict__ == attrs