Free string memory via a string destructor

Also, register string memory as additional GC pressure.
The subprocess module repeatedly allocates a 1MB str as a buffer
for reading from file descriptors.  The first issue was that we would
never free this memory; but even once we would, we wouldn't trigger a GC
since there were only small GC allocations (sizeof(BoxedString) at a time)
but with large other memory pressure, so add a hook for adding GC pressure.

src/gc/gc_alloc.h

@@ -27,6 +27,13 @@
 namespace pyston {
 namespace gc {
 
+// Notify the gc of n bytes as being under GC management.
+// This is called internally for anything allocated through gc_alloc,
+// but it can also be called by clients to say that they have memory that
+// is ultimately GC managed but did not get allocated through gc_alloc,
+// such as memory that will get freed by a gc destructor.
+void registerGCManagedBytes(size_t bytes);
+
 extern "C" inline void* gc_alloc(size_t bytes, GCKind kind_id) __attribute__((visibility("default")));
 extern "C" inline void* gc_alloc(size_t bytes, GCKind kind_id) {
     size_t alloc_bytes = bytes + sizeof(GCAllocation);

src/gc/heap.cpp

@@ -38,7 +38,7 @@ static unsigned bytesAllocatedSinceCollection;
 static __thread unsigned thread_bytesAllocatedSinceCollection;
 #define ALLOCBYTES_PER_COLLECTION 10000000
 
-void _collectIfNeeded(size_t bytes) {
+void registerGCManagedBytes(size_t bytes) {
     thread_bytesAllocatedSinceCollection += bytes;
     if (unlikely(thread_bytesAllocatedSinceCollection > ALLOCBYTES_PER_COLLECTION / 4)) {
         bytesAllocatedSinceCollection += thread_bytesAllocatedSinceCollection;
@@ -111,7 +111,7 @@ struct LargeObj {
 };
 
 GCAllocation* Heap::allocLarge(size_t size) {
-    _collectIfNeeded(size);
+    registerGCManagedBytes(size);
 
     LOCK_REGION(lock);
 
@@ -223,7 +223,7 @@ static Block* claimBlock(size_t rounded_size, Block** free_head) {
 }
 
 GCAllocation* Heap::allocSmall(size_t rounded_size, int bucket_idx) {
-    _collectIfNeeded(rounded_size);
+    registerGCManagedBytes(rounded_size);
 
     Block** free_head = &heads[bucket_idx];
     Block** full_head = &full_heads[bucket_idx];

src/runtime/str.cpp

@@ -32,6 +32,18 @@
 
 namespace pyston {
 
+BoxedString::BoxedString(const char* s, size_t n) : s(s, n) {
+    gc::registerGCManagedBytes(this->s.size());
+}
+
+BoxedString::BoxedString(std::string&& s) : s(std::move(s)) {
+    gc::registerGCManagedBytes(this->s.size());
+}
+
+BoxedString::BoxedString(const std::string& s) : s(s) {
+    gc::registerGCManagedBytes(this->s.size());
+}
+
 extern "C" PyObject* PyString_FromFormatV(const char* format, va_list vargs) noexcept {
     va_list count;
     Py_ssize_t n = 0;
@@ -1882,7 +1894,16 @@ static PyBufferProcs string_as_buffer = {
     (releasebufferproc)NULL,
 };
 
+void strDestructor(Box* b) {
+    assert(isSubclass(b->cls, str_cls));
+    BoxedString* self = static_cast<BoxedString*>(b);
+
+    self->s.~basic_string();
+}
+
 void setupStr() {
+    str_cls->simple_destructor = strDestructor;
+
     str_iterator_cls
         = new BoxedHeapClass(object_cls, &strIteratorGCHandler, 0, sizeof(BoxedStringIterator), false, "striterator");
     str_iterator_cls->giveAttr("__hasnext__",

src/runtime/types.h

@@ -348,9 +348,9 @@ public:
     // const std::basic_string<char, std::char_traits<char>, StlCompatAllocator<char> > s;
     std::string s;
 
-    BoxedString(const char* s, size_t n) __attribute__((visibility("default"))) : s(s, n) {}
-    BoxedString(const std::string&& s) __attribute__((visibility("default"))) : s(std::move(s)) {}
-    BoxedString(const std::string& s) __attribute__((visibility("default"))) : s(s) {}
+    BoxedString(const char* s, size_t n) __attribute__((visibility("default")));
+    BoxedString(std::string&& s) __attribute__((visibility("default")));
+    BoxedString(const std::string& s) __attribute__((visibility("default")));
 
     DEFAULT_CLASS(str_cls);
 };

test/tests/str_allocation.py

+# Make sure that allocating large strings works.
+# (We've had issues with this due to string memory not being GC'd or tracked by the GC
+
+s = "." * 1000000 # 1MB
+
+for i in xrange(1000):
+    print i, len(s + '!')