Merge pull request #274 from toshok/more-gc-perf-changes

More gc perf changes

src/core/types.h

@@ -349,17 +349,17 @@ namespace gc {
 enum class GCKind : uint8_t {
     PYTHON = 1,
     CONSERVATIVE = 2,
-    UNTRACKED = 3,
+    PRECISE = 3,
+    UNTRACKED = 4,
+    HIDDEN_CLASS = 5,
 };
 
 extern "C" void* gc_alloc(size_t nbytes, GCKind kind);
 }
 
-class ConservativeGCObject {
+template <gc::GCKind gc_kind> class GCAllocated {
 public:
-    void* operator new(size_t size) __attribute__((visibility("default"))) {
-        return gc_alloc(size, gc::GCKind::CONSERVATIVE);
-    }
+    void* operator new(size_t size) __attribute__((visibility("default"))) { return gc_alloc(size, gc_kind); }
     void operator delete(void* ptr) __attribute__((visibility("default"))) { abort(); }
 };
 
@@ -372,7 +372,7 @@ struct DelattrRewriteArgs;
 
 struct HCAttrs {
 public:
-    struct AttrList : ConservativeGCObject {
+    struct AttrList : public GCAllocated<gc::GCKind::PRECISE> {
         Box* attrs[0];
     };
 

src/gc/collector.cpp

@@ -256,6 +256,9 @@ static void markPhase() {
         } else if (kind_id == GCKind::CONSERVATIVE) {
             uint32_t bytes = al->kind_data;
             visitor.visitPotentialRange((void**)p, (void**)((char*)p + bytes));
+        } else if (kind_id == GCKind::PRECISE) {
+            uint32_t bytes = al->kind_data;
+            visitor.visitRange((void**)p, (void**)((char*)p + bytes));
         } else if (kind_id == GCKind::PYTHON) {
             Box* b = reinterpret_cast<Box*>(p);
             BoxedClass* cls = b->cls;
@@ -268,6 +271,9 @@ static void markPhase() {
                 ASSERT(cls->gc_visit, "%s", getTypeName(b)->c_str());
                 cls->gc_visit(&visitor, b);
             }
+        } else if (kind_id == GCKind::HIDDEN_CLASS) {
+            HiddenClass* hcls = reinterpret_cast<HiddenClass*>(p);
+            hcls->gc_visit(&visitor);
         } else {
             RELEASE_ASSERT(0, "Unhandled kind: %d", (int)kind_id);
         }

src/gc/heap.cpp

@@ -130,6 +130,9 @@ static Block* alloc_block(uint64_t size, Block** prev) {
     Block* rtn = (Block*)small_arena.doMmap(sizeof(Block));
     assert(rtn);
     rtn->size = size;
+    rtn->num_obj = BLOCK_SIZE / size;
+    rtn->min_obj_index = (BLOCK_HEADER_SIZE + size - 1) / size;
+    rtn->atoms_per_obj = size / ATOM_SIZE;
     rtn->prev = prev;
     rtn->next = NULL;
 
@@ -371,7 +374,7 @@ GCAllocation* Heap::getAllocationFromInteriorPointer(void* ptr) {
     if (obj_idx < b->minObjIndex() || obj_idx >= b->numObjects())
         return NULL;
 
-    int atom_idx = obj_idx * (size / ATOM_SIZE);
+    int atom_idx = obj_idx * b->atomsPerObj();
 
     if (b->isfree.isSet(atom_idx))
         return NULL;

src/gc/heap.h

@@ -87,19 +87,20 @@ public:
     void clear(int idx) { data[idx / 64] &= ~(1UL << (idx % 64)); }
 
     int scanForNext(Scanner& sc) {
-        uint64_t mask = 0;
+        uint64_t mask = data[sc.next_to_check];
 
+        if (unlikely(mask == 0L)) {
             while (true) {
-            mask = data[sc.next_to_check];
-            if (likely(mask != 0L)) {
-                break;
-            }
-
                 sc.next_to_check++;
                 if (sc.next_to_check == N / 64) {
                     sc.next_to_check = 0;
                     return -1;
                 }
+                mask = data[sc.next_to_check];
+                if (likely(mask != 0L)) {
+                    break;
+                }
+            }
         }
 
         int i = sc.next_to_check;
@@ -134,7 +135,10 @@ struct Block {
     union {
         struct {
             Block* next, **prev;
-            uint64_t size;
+            uint32_t size;
+            uint16_t num_obj;
+            uint8_t min_obj_index;
+            uint8_t atoms_per_obj;
             Bitmap<ATOMS_PER_BLOCK> isfree;
             Bitmap<ATOMS_PER_BLOCK>::Scanner next_to_check;
             void* _header_end[0];
@@ -142,11 +146,11 @@ struct Block {
         Atoms atoms[ATOMS_PER_BLOCK];
     };
 
-    inline int minObjIndex() { return (BLOCK_HEADER_SIZE + size - 1) / size; }
+    inline int minObjIndex() const { return min_obj_index; }
 
-    inline int numObjects() { return BLOCK_SIZE / size; }
+    inline int numObjects() const { return num_obj; }
 
-    inline int atomsPerObj() { return size / ATOM_SIZE; }
+    inline int atomsPerObj() const { return atoms_per_obj; }
 
     static Block* forPointer(void* ptr) { return (Block*)((uintptr_t)ptr & ~(BLOCK_SIZE - 1)); }
 };

src/runtime/types.h

@@ -255,7 +255,7 @@ static_assert(offsetof(pyston::BoxedHeapClass, as_buffer) == offsetof(PyHeapType
 static_assert(sizeof(pyston::BoxedHeapClass) == sizeof(PyHeapTypeObject), "");
 
 
-class HiddenClass : public ConservativeGCObject {
+class HiddenClass : public GCAllocated<gc::GCKind::HIDDEN_CLASS> {
 private:
     HiddenClass() {}
     HiddenClass(const HiddenClass* parent) : attr_offsets(parent->attr_offsets) {}
@@ -270,8 +270,8 @@ public:
         return new HiddenClass();
     }
 
-    conservative_unordered_map<std::string, int> attr_offsets;
-    conservative_unordered_map<std::string, HiddenClass*> children;
+    std::unordered_map<std::string, int> attr_offsets;
+    std::unordered_map<std::string, HiddenClass*> children;
 
     HiddenClass* getOrMakeChild(const std::string& attr);
 
@@ -282,6 +282,12 @@ public:
         return it->second;
     }
     HiddenClass* delAttrToMakeHC(const std::string& attr);
+
+    void gc_visit(GCVisitor* visitor) {
+        for (const auto& p : children) {
+            visitor->visit(p.second);
+        }
+    }
 };
 
 class BoxedInt : public Box {