add 1 to str_cls->tp_basicsize for the null byte

src/runtime/inline/boxing.cpp

@@ -32,14 +32,14 @@ extern "C" Box* createList() {
 
 BoxedString* _boxStrConstant(const char* chars) {
     size_t len = strlen(chars);
-    return new (len + 1) BoxedString(chars, len);
+    return new (len) BoxedString(chars, len);
 }
 extern "C" BoxedString* boxStrConstant(const char* chars) {
     return _boxStrConstant(chars);
 }
 
 BoxedString* _boxStrConstantSize(const char* chars, size_t n) {
-    return new (n + 1) BoxedString(chars, n);
+    return new (n) BoxedString(chars, n);
 }
 
 extern "C" BoxedString* boxStrConstantSize(const char* chars, size_t n) {
@@ -47,18 +47,18 @@ extern "C" BoxedString* boxStrConstantSize(const char* chars, size_t n) {
 }
 
 extern "C" Box* boxStringPtr(const std::string* s) {
-    return new (s->size() + 1) BoxedString(s->c_str(), s->size());
+    return new (s->size()) BoxedString(s->c_str(), s->size());
 }
 
 Box* boxStringRef(llvm::StringRef s) {
-    return new (s.size() + 1) BoxedString(s);
+    return new (s.size()) BoxedString(s);
 }
 
 Box* boxString(const std::string& s) {
-    return new (s.size() + 1) BoxedString(s.c_str(), s.size());
+    return new (s.size()) BoxedString(s.c_str(), s.size());
 }
 Box* boxString(std::string&& s) {
-    return new (s.size() + 1) BoxedString(s.c_str(), s.size());
+    return new (s.size()) BoxedString(s.c_str(), s.size());
 }
 
 Box* boxStringTwine(const llvm::Twine& t) {

src/runtime/objmodel.cpp

@@ -369,7 +369,6 @@ BoxedClass::BoxedClass(BoxedClass* base, gcvisit_func gc_visit, int attrs_offset
     if (base && cls && str_cls)
         giveAttr("__base__", base);
 
-    assert(tp_basicsize % sizeof(void*) == 0); // Not critical I suppose, but probably signals a bug
     if (attrs_offset) {
         assert(tp_basicsize >= attrs_offset + sizeof(HCAttrs));
         assert(attrs_offset % sizeof(void*) == 0); // Not critical I suppose, but probably signals a bug

src/runtime/str.cpp

@@ -341,7 +341,7 @@ extern "C" Box* strAdd(BoxedString* lhs, Box* _rhs) {
     }
 
     BoxedString* rhs = static_cast<BoxedString*>(_rhs);
-    return new (lhs->size() + rhs->size() + 1) BoxedString(lhs->s, rhs->s);
+    return new (lhs->size() + rhs->size()) BoxedString(lhs->s, rhs->s);
 }
 
 static llvm::StringMap<Box*> interned_strings;
@@ -1547,7 +1547,7 @@ extern "C" Box* strNew(BoxedClass* cls, Box* obj) {
 
     BoxedString* _rtn = static_cast<BoxedString*>(rtn);
 
-    return new (cls, _rtn->size() + 1) BoxedString(_rtn->s);
+    return new (cls, _rtn->size()) BoxedString(_rtn->s);
 }
 
 extern "C" Box* basestringNew(BoxedClass* cls, Box* args, Box* kwargs) {
@@ -1571,7 +1571,7 @@ Box* _strSlice(BoxedString* self, i64 start, i64 stop, i64 step, i64 length) {
     if (length == 0)
         return EmptyString;
 
-    BoxedString* bs = new (length + 1) BoxedString(nullptr, length);
+    BoxedString* bs = new (length) BoxedString(nullptr, length);
     copySlice(bs->data(), s.data(), start, step, length);
     return bs;
 }
@@ -2289,7 +2289,7 @@ extern "C" int PyString_AsStringAndSize(register PyObject* obj, register char**
 
 BoxedString* createUninitializedString(ssize_t n) {
     // I *think* this should avoid doing any copies, by using move constructors:
-    return new (n + 1) BoxedString(n, 0);
+    return new (n) BoxedString(n, 0);
 }
 
 char* getWriteableStringContents(BoxedString* s) {
@@ -2340,7 +2340,7 @@ extern "C" int _PyString_Resize(PyObject** pv, Py_ssize_t newsize) noexcept {
     BoxedString* resized;
 
     if (s->cls == str_cls)
-        resized = new (newsize + 1) BoxedString(newsize, 0); // we need an uninitialized string, but this will memset
+        resized = new (newsize) BoxedString(newsize, 0); // we need an uninitialized string, but this will memset
     else
         resized = new (s->cls, newsize)
             BoxedString(newsize, 0); // we need an uninitialized string, but this will memset

src/runtime/types.cpp

@@ -2033,7 +2033,8 @@ void setupRuntime() {
     // You can't actually have an instance of basestring
     basestring_cls = new (0) BoxedHeapClass(object_cls, NULL, 0, 0, sizeof(Box), false, NULL);
 
-    str_cls = new (0) BoxedHeapClass(basestring_cls, NULL, 0, 0, sizeof(BoxedString), false, NULL);
+    // We add 1 to the tp_basicsize of the BoxedString in order to hold the null byte at the end.
+    str_cls = new (0) BoxedHeapClass(basestring_cls, NULL, 0, 0, sizeof(BoxedString) + 1, false, NULL);
     str_cls->tp_flags |= Py_TPFLAGS_STRING_SUBCLASS;
     str_cls->tp_itemsize = sizeof(char);
 

src/runtime/types.h

@@ -416,7 +416,26 @@ public:
     char* data() { return const_cast<char*>(s.data()); }
     size_t size() { return s.size(); }
 
-    DEFAULT_CLASS_VAR_SIMPLE(str_cls, sizeof(char));
+    // DEFAULT_CLASS_VAR_SIMPLE doesn't work because of the +1 for the null byte
+    void* operator new(size_t size, BoxedClass* cls, size_t nitems) __attribute__((visibility("default"))) {
+        assert(cls->tp_itemsize == sizeof(char));
+        return BoxVar::operator new(size, cls, nitems);
+    }
+    void* operator new(size_t size, size_t nitems) __attribute__((visibility("default"))) {
+        assert(str_cls->tp_alloc == PystonType_GenericAlloc);
+        assert(str_cls->tp_itemsize == 1);
+        assert(str_cls->tp_basicsize == sizeof(BoxedString) + 1);
+        assert(str_cls->is_pyston_class);
+        assert(str_cls->attrs_offset == 0);
+
+        void* mem = gc_alloc(sizeof(BoxedString) + 1 + nitems, gc::GCKind::PYTHON);
+        assert(mem);
+
+        BoxVar* rtn = static_cast<BoxVar*>(mem);
+        rtn->cls = str_cls;
+        rtn->ob_size = nitems;
+        return rtn;
+    }
 
     // these should be private, but strNew needs them
     BoxedString(const char* s, size_t n) __attribute__((visibility("default")));
@@ -426,6 +445,7 @@ public:
 
 private:
     // used only in ctors to give our llvm::StringRef the proper pointer
+    // Note: sizeof(BoxedString) = str_cls->tp_basicsize - 1
     char* storage() { return (char*)this + sizeof(BoxedString); }
 
     void* operator new(size_t size) = delete;