generator support wip

Some basic support, but leaks references if you abandon a generator
while it is in a "yield" expression.

src/runtime/frame.cpp

@@ -234,6 +234,26 @@ extern "C" void deinitFrame(FrameInfo* frame_info) {
     }
 }
 
+int frameinfo_traverse(FrameInfo* frame_info, visitproc visit, void* arg) noexcept {
+    Py_VISIT(frame_info->frame_obj);
+
+    if (frame_info->vregs) {
+        int num_user_visible_vregs = frame_info->md->calculateNumUserVisibleVRegs();
+        for (int i = 0; i < num_user_visible_vregs; i++) {
+            Py_VISIT(frame_info->vregs[i]);
+        }
+    }
+    Py_VISIT(frame_info->boxedLocals);
+
+    if (frame_info->exc.type) {
+        Py_VISIT(frame_info->exc.type);
+        Py_VISIT(frame_info->exc.value);
+        Py_VISIT(frame_info->exc.traceback);
+    }
+
+    return 0;
+}
+
 extern "C" void setFrameExcInfo(FrameInfo* frame_info, STOLEN(Box*) type, STOLEN(Box*) value, STOLEN(Box*) tb) {
     Box* old_type = frame_info->exc.type;
     Box* old_value = frame_info->exc.value;

src/runtime/generator.cpp

@@ -347,8 +347,10 @@ extern "C" Box* yield(BoxedGenerator* obj, Box* value) {
 
     // reset current frame to the caller tops frame --> removes the frame the generator added
     cur_thread_state.frame_info = self->top_caller_frame_info;
+    obj->paused_frame_info = generator_frame_info;
     swapContext(&self->context, self->returnContext, 0);
     FrameInfo* top_new_caller_frame_info = (FrameInfo*)cur_thread_state.frame_info;
+    obj->paused_frame_info = NULL;
 
     // the caller of the generator can change between yield statements that means we can't just restore the top of the
     // frame to the point before the yield instead we have to update it.
@@ -397,7 +399,8 @@ extern "C" BoxedGenerator::BoxedGenerator(BoxedFunctionBase* function, Box* arg1
       exception(nullptr, nullptr, nullptr),
       context(nullptr),
       returnContext(nullptr),
-      top_caller_frame_info(nullptr)
+      top_caller_frame_info(nullptr),
+      paused_frame_info(nullptr)
 #if STAT_TIMERS
       ,
       prev_stack(NULL),
@@ -481,7 +484,13 @@ Box* generatorName(Box* _self, void* context) {
 }
 
 extern "C" int PyGen_NeedsFinalizing(PyGenObject* gen) noexcept {
-    Py_FatalError("unimplemented");
+    auto self = (BoxedGenerator*)gen;
+
+    // CPython has some optimizations for not needing to finalize generators that haven't exited, but
+    // which are guaranteed to not need any special cleanups.
+    // For now just say anything still in-progress needs finalizing.
+    return (bool)self->paused_frame_info;
+
 #if 0
     int i;
     PyFrameObject* f = gen->gi_frame;
@@ -504,6 +513,13 @@ extern "C" int PyGen_NeedsFinalizing(PyGenObject* gen) noexcept {
 static void generator_dealloc(BoxedGenerator* self) noexcept {
     assert(isSubclass(self->cls, generator_cls));
 
+    // Hopefully this never happens:
+    assert(!self->running);
+
+    // I don't think this should get hit currently because we don't properly collect the cycle that this would
+    // represent:
+    ASSERT(!self->paused_frame_info, "Can't clean up a generator that is currently paused");
+
     PyObject_GC_UnTrack(self);
 
     freeGeneratorStack(self);
@@ -534,7 +550,35 @@ static void generator_dealloc(BoxedGenerator* self) noexcept {
 }
 
 static int generator_traverse(BoxedGenerator* self, visitproc visit, void *arg) noexcept {
-    Py_FatalError("unimplemented");
+    assert(isSubclass(self->cls, generator_cls));
+
+    if (self->paused_frame_info) {
+        int r = frameinfo_traverse(self->paused_frame_info, visit, arg);
+        if (r)
+            return r;
+    }
+
+    int numArgs = self->function->md->numReceivedArgs();
+    if (numArgs > 3) {
+        numArgs -= 3;
+        for (int i= 0; i < numArgs; i++) {
+            Py_VISIT(self->args->elts[i]);
+        }
+    }
+
+    Py_VISIT(self->arg1);
+    Py_VISIT(self->arg2);
+    Py_VISIT(self->arg3);
+
+    Py_VISIT(self->function);
+
+    Py_VISIT(self->returnValue);
+
+    Py_VISIT(self->exception.type);
+    Py_VISIT(self->exception.value);
+    Py_VISIT(self->exception.traceback);
+
+    return 0;
 }
 
 void setupGenerator() {

src/runtime/types.h

@@ -1202,7 +1202,11 @@ public:
 
     struct Context* context, *returnContext;
     void* stack_begin;
-    FrameInfo* top_caller_frame_info;
+    FrameInfo* top_caller_frame_info; // The FrameInfo that called into this generator.
+
+    // For abandoned-generator collection -- WIP
+    FrameInfo* paused_frame_info; // The FrameInfo the generator was on when it called yield (or NULL if the generator
+                                  // hasn't started or has exited).
 
 #if STAT_TIMERS
     StatTimer* prev_stack;
@@ -1316,6 +1320,7 @@ Box* getFrame(FrameInfo* frame_info);
 Box* getFrame(int depth);
 void frameInvalidateBack(BoxedFrame* frame);
 extern "C" void deinitFrame(FrameInfo* frame_info);
+int frameinfo_traverse(FrameInfo* frame_info, visitproc visit, void* arg) noexcept;
 extern "C" void initFrame(FrameInfo* frame_info);
 extern "C" void setFrameExcInfo(FrameInfo* frame_info, STOLEN(Box*) type, STOLEN(Box*) value, STOLEN(Box*) tb);
 

test/tests/generator_cycle.py

+# Test a generator cycle involving an unfinished generator.
+
+def f():
+    g = (i in (None, g) for i in xrange(2))
+    print g.next()
+print f()