Merge pull request #820 from rudi-c/gc_types

Refactors on types of GC objects

src/core/types.h

@@ -29,6 +29,7 @@
 #include "core/common.h"
 #include "core/stats.h"
 #include "core/stringpool.h"
+#include "gc/gc.h"
 
 namespace llvm {
 class Function;
@@ -38,29 +39,6 @@ class Value;
 
 namespace pyston {
 
-namespace gc {
-
-class TraceStack;
-class GCVisitor {
-private:
-    bool isValid(void* p);
-
-public:
-    TraceStack* stack;
-    GCVisitor(TraceStack* stack) : stack(stack) {}
-
-    // These all work on *user* pointers, ie pointers to the user_data section of GCAllocations
-    void visitIf(void* p) {
-        if (p)
-            visit(p);
-    }
-    void visit(void* p);
-    void visitRange(void* const* start, void* const* end);
-    void visitPotential(void* p);
-    void visitPotentialRange(void* const* start, void* const* end);
-};
-
-} // namespace gc
 using gc::GCVisitor;
 
 enum class EffortLevel {
@@ -459,33 +437,14 @@ class BinopIC;
 
 class Box;
 
-namespace gc {
-
-enum class GCKind : uint8_t {
-    PYTHON = 1,
-    CONSERVATIVE = 2,
-    PRECISE = 3,
-    UNTRACKED = 4,
-    HIDDEN_CLASS = 5,
-    CONSERVATIVE_PYTHON = 6,
-};
-
-extern "C" void* gc_alloc(size_t nbytes, GCKind kind);
-extern "C" void gc_free(void* ptr);
-}
-
-template <gc::GCKind gc_kind> class GCAllocated {
-public:
-    void* operator new(size_t size) __attribute__((visibility("default"))) { return gc::gc_alloc(size, gc_kind); }
-    void operator delete(void* ptr) __attribute__((visibility("default"))) { gc::gc_free(ptr); }
-};
-
-class BoxIteratorImpl : public GCAllocated<gc::GCKind::CONSERVATIVE> {
+class BoxIteratorImpl : public gc::GCAllocatedRuntime {
 public:
     virtual ~BoxIteratorImpl() = default;
     virtual void next() = 0;
     virtual Box* getValue() = 0;
     virtual bool isSame(const BoxIteratorImpl* rhs) = 0;
+
+    virtual void gc_visit(GCVisitor* v) = 0;
 };
 
 class BoxIterator {

src/gc/collector.cpp

@@ -404,9 +404,9 @@ static __attribute__((always_inline)) void visitByGCKind(void* p, GCVisitor& vis
             ASSERT(cls->gc_visit, "%s", getTypeName(b));
             cls->gc_visit(&visitor, b);
         }
-    } else if (kind_id == GCKind::HIDDEN_CLASS) {
-        HiddenClass* hcls = reinterpret_cast<HiddenClass*>(p);
-        hcls->gc_visit(&visitor);
+    } else if (kind_id == GCKind::RUNTIME) {
+        GCAllocatedRuntime* runtime_obj = reinterpret_cast<GCAllocatedRuntime*>(p);
+        runtime_obj->gc_visit(&visitor);
     } else {
         RELEASE_ASSERT(0, "Unhandled kind: %d", (int)kind_id);
     }

src/gc/gc.h

+// Copyright (c) 2014-2015 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef PYSTON_GC_GC_H
+#define PYSTON_GC_GC_H
+
+namespace pyston {
+namespace gc {
+
+// GOAL: Eventually, move any declaration that needs to be visible outside the gc/ folder
+// to this file and only expose this header.
+
+class TraceStack;
+class GCVisitor {
+private:
+    bool isValid(void* p);
+
+public:
+    TraceStack* stack;
+    GCVisitor(TraceStack* stack) : stack(stack) {}
+
+    // These all work on *user* pointers, ie pointers to the user_data section of GCAllocations
+    void visitIf(void* p) {
+        if (p)
+            visit(p);
+    }
+    void visit(void* p);
+    void visitRange(void* const* start, void* const* end);
+    void visitPotential(void* p);
+    void visitPotentialRange(void* const* start, void* const* end);
+};
+
+enum class GCKind : uint8_t {
+    // Any Python object (e.g. any Box) that can be visited precisely, using
+    // a GC handler function.
+    PYTHON = 1,
+
+    // An arbitrary block of memory that may contain pointers.
+    CONSERVATIVE = 2,
+
+    // An arbitrary block of memory with contiguous pointers.
+    PRECISE = 3,
+
+    // An arbitrary block of memory that does not contain pointers.
+    UNTRACKED = 4,
+
+    // C++ objects that we need to manage with our own heap and GC, either
+    // because it contains pointers into our heap or our heap points to these
+    // objects. These objects inherit from GCAllocatedRuntime.
+    RUNTIME = 5,
+
+    // A Python object where we don't have a way to visit precisely with a GC
+    // handler function. These are usually Python objects allocated in C extensions.
+    CONSERVATIVE_PYTHON = 6,
+};
+
+extern "C" void* gc_alloc(size_t nbytes, GCKind kind);
+extern "C" void* gc_realloc(void* ptr, size_t bytes);
+extern "C" void gc_free(void* ptr);
+
+// Use this if a C++ object needs to be allocated in our heap.
+class GCAllocatedRuntime {
+public:
+    virtual ~GCAllocatedRuntime() {}
+
+    void* operator new(size_t size) __attribute__((visibility("default"))) { return gc_alloc(size, GCKind::RUNTIME); }
+    void operator delete(void* ptr) __attribute__((visibility("default"))) { gc_free(ptr); }
+
+    virtual void gc_visit(GCVisitor* visitor) = 0;
+};
+} // namespace gc
+}
+
+#endif

src/gc/heap.cpp

@@ -234,7 +234,7 @@ struct HeapStatistics {
     int num_hcls_by_attrs[HCLS_ATTRS_STAT_MAX + 1];
     int num_hcls_by_attrs_exceed;
 
-    TypeStats python, conservative, conservative_python, untracked, hcls, precise;
+    TypeStats python, conservative, conservative_python, untracked, runtime, precise;
     TypeStats total;
 
     HeapStatistics(bool collect_cls_stats, bool collect_hcls_stats)
@@ -287,18 +287,9 @@ void addStatistic(HeapStatistics* stats, GCAllocation* al, int nbytes) {
     } else if (al->kind_id == GCKind::UNTRACKED) {
         stats->untracked.nallocs++;
         stats->untracked.nbytes += nbytes;
-    } else if (al->kind_id == GCKind::HIDDEN_CLASS) {
-        stats->hcls.nallocs++;
-        stats->hcls.nbytes += nbytes;
-
-        if (stats->collect_hcls_stats) {
-            HiddenClass* hcls = (HiddenClass*)al->user_data;
-            int numattrs = hcls->attributeArraySize();
-            if (numattrs <= HCLS_ATTRS_STAT_MAX)
-                stats->num_hcls_by_attrs[numattrs]++;
-            else
-                stats->num_hcls_by_attrs_exceed++;
-        }
+    } else if (al->kind_id == GCKind::RUNTIME) {
+        stats->runtime.nallocs++;
+        stats->runtime.nbytes += nbytes;
     } else if (al->kind_id == GCKind::PRECISE) {
         stats->precise.nallocs++;
         stats->precise.nbytes += nbytes;
@@ -327,7 +318,7 @@ void Heap::dumpHeapStatistics(int level) {
     stats.conservative.print("conservative");
     stats.conservative_python.print("conservative_python");
     stats.untracked.print("untracked");
-    stats.hcls.print("hcls");
+    stats.runtime.print("runtime");
     stats.precise.print("precise");
 
     if (collect_cls_stats) {
@@ -338,16 +329,6 @@ void Heap::dumpHeapStatistics(int level) {
 
     stats.total.print("Total");
 
-    if (collect_hcls_stats) {
-        fprintf(stderr, "%ld hidden classes currently alive\n", stats.hcls.nallocs);
-        fprintf(stderr, "%ld have at least one Box that uses them\n", stats.hcls_uses.size());
-
-        for (int i = 0; i <= HCLS_ATTRS_STAT_MAX; i++) {
-            fprintf(stderr, "With % 3d attributes: %d\n", i, stats.num_hcls_by_attrs[i]);
-        }
-        fprintf(stderr, "With >% 2d attributes: %d\n", HCLS_ATTRS_STAT_MAX, stats.num_hcls_by_attrs_exceed);
-    }
-
     fprintf(stderr, "\n");
 }
 

src/runtime/hiddenclass.cpp

@@ -27,6 +27,20 @@
 
 namespace pyston {
 
+void HiddenClass::gc_visit(GCVisitor* visitor) {
+    // Visit children even for the dict-backed case, since children will just be empty
+    visitor->visitRange((void* const*)&children.vector()[0], (void* const*)&children.vector()[children.size()]);
+    visitor->visit(attrwrapper_child);
+
+    // We don't need to visit the keys of the 'children' map, since the children should have those as entries
+    // in the attr_offssets map.
+    // Also, if we have any children, we can skip scanning our attr_offsets map, since it will be a subset
+    // of our child's map.
+    if (children.empty())
+        for (auto p : attr_offsets)
+            visitor->visit(p.first);
+}
+
 void HiddenClass::appendAttribute(BoxedString* attr) {
     assert(attr->interned_state != SSTATE_NOT_INTERNED);
     assert(type == SINGLETON);

src/runtime/hiddenclass.h

@@ -25,7 +25,7 @@
 
 namespace pyston {
 
-class HiddenClass : public GCAllocated<gc::GCKind::HIDDEN_CLASS> {
+class HiddenClass : public gc::GCAllocatedRuntime {
 public:
     // We have a couple different storage strategies for attributes, which
     // are distinguished by having a different hidden class type.
@@ -76,19 +76,7 @@ public:
         return new HiddenClass(DICT_BACKED);
     }
 
-    void gc_visit(GCVisitor* visitor) {
-        // Visit children even for the dict-backed case, since children will just be empty
-        visitor->visitRange((void* const*)&children.vector()[0], (void* const*)&children.vector()[children.size()]);
-        visitor->visit(attrwrapper_child);
-
-        // We don't need to visit the keys of the 'children' map, since the children should have those as entries
-        // in the attr_offssets map.
-        // Also, if we have any children, we can skip scanning our attr_offsets map, since it will be a subset
-        // of our child's map.
-        if (children.empty())
-            for (auto p : attr_offsets)
-                visitor->visit(p.first);
-    }
+    void gc_visit(GCVisitor* visitor);
 
     // The total size of the attribute array.  The slots in the attribute array may not correspond 1:1 to Python
     // attributes.

src/runtime/iterators.cpp

@@ -62,6 +62,13 @@ public:
         static BoxIteratorGeneric _end(nullptr);
         return &_end;
     }
+
+    void gc_visit(GCVisitor* v) {
+        if (iterator)
+            v->visit(iterator);
+        if (value)
+            v->visit(value);
+    }
 };
 
 template <typename T> class BoxIteratorIndex : public BoxIteratorImpl {
@@ -107,6 +114,11 @@ public:
         static BoxIteratorIndex _end(nullptr);
         return &_end;
     }
+
+    void gc_visit(GCVisitor* v) {
+        if (obj)
+            v->visit(obj);
+    }
 };
 }
 

src/runtime/util.cpp

@@ -154,7 +154,12 @@ extern "C" void dumpEx(void* p, int levels) {
     }
 
     if (al->kind_id == gc::GCKind::PRECISE) {
-        printf("precise gc array\n");
+        printf("pyston precise object\n");
+        return;
+    }
+
+    if (al->kind_id == gc::GCKind::RUNTIME) {
+        printf("pyston runtime object\n");
         return;
     }
 
@@ -291,11 +296,6 @@ extern "C" void dumpEx(void* p, int levels) {
         return;
     }
 
-    if (al->kind_id == gc::GCKind::HIDDEN_CLASS) {
-        printf("Hidden class object\n");
-        return;
-    }
-
     RELEASE_ASSERT(0, "%d", (int)al->kind_id);
 }
 }