Merge pull request #108 from undingen/generators

Implement Python Generators

src/analysis/scoping_analysis.cpp

@@ -20,6 +20,35 @@
 
 namespace pyston {
 
+class YieldVisitor : public NoopASTVisitor {
+public:
+    YieldVisitor() : containsYield(false) {}
+
+    virtual bool visit_functiondef(AST_FunctionDef*) { return true; }
+
+    virtual bool visit_yield(AST_Yield*) {
+        containsYield = true;
+        return true;
+    }
+
+    bool containsYield;
+};
+
+bool containsYield(AST* ast) {
+    YieldVisitor visitor;
+    if (ast->type == AST_TYPE::FunctionDef) {
+        AST_FunctionDef* funcDef = static_cast<AST_FunctionDef*>(ast);
+        for (auto& e : funcDef->body) {
+            e->accept(&visitor);
+            if (visitor.containsYield)
+                return true;
+        }
+    } else {
+        ast->accept(&visitor);
+    }
+    return visitor.containsYield;
+}
+
 static bool isCompilerCreatedName(const std::string& name) {
     return name[0] == '!' || name[0] == '#';
 }
@@ -206,6 +235,7 @@ public:
     virtual bool visit_unaryop(AST_UnaryOp* node) { return false; }
     virtual bool visit_while(AST_While* node) { return false; }
     virtual bool visit_with(AST_With* node) { return false; }
+    virtual bool visit_yield(AST_Yield* node) { return false; }
 
     virtual bool visit_branch(AST_Branch* node) { return false; }
     virtual bool visit_jump(AST_Jump* node) { return false; }
@@ -394,9 +424,12 @@ void ScopingAnalysis::processNameUsages(ScopingAnalysis::NameUsageMap* usages) {
         switch (node->type) {
             case AST_TYPE::ClassDef:
             case AST_TYPE::FunctionDef:
-            case AST_TYPE::Lambda:
-                this->scopes[node] = new ScopeInfoBase(parent_info, usage);
+            case AST_TYPE::Lambda: {
+                ScopeInfoBase* scopInfo = new ScopeInfoBase(parent_info, usage);
+                scopInfo->setTakesGenerator(containsYield(node));
+                this->scopes[node] = scopInfo;
                 break;
+            }
             default:
                 RELEASE_ASSERT(0, "%d", usage->node->type);
                 break;
@@ -413,6 +446,9 @@ ScopeInfo* ScopingAnalysis::analyzeSubtree(AST* node) {
 
     ScopeInfo* rtn = scopes[node];
     assert(rtn);
+
+    rtn->setTakesGenerator(containsYield(node));
+
     return rtn;
 }
 

src/analysis/scoping_analysis.h

@@ -24,6 +24,7 @@ class AST_Module;
 
 class ScopeInfo {
 public:
+    ScopeInfo() : isGeneratorValue(false) {}
     virtual ~ScopeInfo() {}
     virtual ScopeInfo* getParent() = 0;
 
@@ -31,6 +32,9 @@ public:
     virtual bool takesClosure() = 0;
     virtual bool passesThroughClosure() = 0;
 
+    virtual bool takesGenerator() { return isGeneratorValue; }
+    virtual void setTakesGenerator(bool b = true) { isGeneratorValue = b; }
+
     virtual bool refersToGlobal(const std::string& name) = 0;
     virtual bool refersToClosure(const std::string name) = 0;
     virtual bool saveInClosure(const std::string name) = 0;
@@ -39,6 +43,9 @@ public:
     // the metaclass constructor.
     // An error to call this on a non-classdef node.
     virtual const std::unordered_set<std::string>& getClassDefLocalNames() = 0;
+
+protected:
+    bool isGeneratorValue;
 };
 
 class ScopingAnalysis {
@@ -59,6 +66,8 @@ public:
 };
 
 ScopingAnalysis* runScopingAnalysis(AST_Module* m);
+
+bool containsYield(AST* ast);
 }
 
 #endif

src/analysis/type_analysis.cpp

@@ -445,6 +445,7 @@ private:
         return attr_type->callType(ArgPassSpec(0), arg_types, NULL);
     }
 
+    virtual void* visit_yield(AST_Yield*) { return UNKNOWN; }
 
 
     virtual void visit_assert(AST_Assert* node) {

src/codegen/codegen.h

@@ -70,7 +70,7 @@ struct GlobalState {
     llvm::Type* llvm_flavor_type, *llvm_flavor_type_ptr;
     llvm::Type* llvm_opaque_type;
     llvm::Type* llvm_str_type_ptr;
-    llvm::Type* llvm_clfunction_type_ptr, *llvm_closure_type_ptr;
+    llvm::Type* llvm_clfunction_type_ptr, *llvm_closure_type_ptr, *llvm_generator_type_ptr;
     llvm::Type* llvm_module_type_ptr, *llvm_bool_type_ptr;
     llvm::Type* i1, *i8, *i8_ptr, *i32, *i64, *void_, *double_;
     llvm::Type* vector_ptr;

src/codegen/compvars.cpp

@@ -589,16 +589,16 @@ ConcreteCompilerVariable* UnknownType::nonzero(IREmitter& emitter, const OpInfo&
     return new ConcreteCompilerVariable(BOOL, rtn_val, true);
 }
 
-CompilerVariable* makeFunction(IREmitter& emitter, CLFunction* f, CompilerVariable* closure,
+CompilerVariable* makeFunction(IREmitter& emitter, CLFunction* f, CompilerVariable* closure, bool isGenerator,
                                const std::vector<ConcreteCompilerVariable*>& defaults) {
     // Unlike the CLFunction*, which can be shared between recompilations, the Box* around it
     // should be created anew every time the functiondef is encountered
 
     llvm::Value* closure_v;
-    ConcreteCompilerVariable* converted = NULL;
+    ConcreteCompilerVariable* convertedClosure = NULL;
     if (closure) {
-        converted = closure->makeConverted(emitter, closure->getConcreteType());
-        closure_v = converted->getValue();
+        convertedClosure = closure->makeConverted(emitter, closure->getConcreteType());
+        closure_v = convertedClosure->getValue();
     } else {
         closure_v = embedConstantPtr(nullptr, g.llvm_closure_type_ptr);
     }
@@ -618,12 +618,15 @@ CompilerVariable* makeFunction(IREmitter& emitter, CLFunction* f, CompilerVariab
         scratch = embedConstantPtr(nullptr, g.llvm_value_type_ptr_ptr);
     }
 
+    llvm::Value* isGenerator_v = llvm::ConstantInt::get(g.i1, isGenerator, false);
+
     llvm::Value* boxed = emitter.getBuilder()->CreateCall(
-        g.funcs.boxCLFunction, std::vector<llvm::Value*>{ embedConstantPtr(f, g.llvm_clfunction_type_ptr), closure_v,
-                                                          scratch, getConstantInt(defaults.size(), g.i64) });
+        g.funcs.boxCLFunction,
+        std::vector<llvm::Value*>{ embedConstantPtr(f, g.llvm_clfunction_type_ptr), closure_v, isGenerator_v, scratch,
+                                   getConstantInt(defaults.size(), g.i64) });
 
-    if (converted)
-        converted->decvref(emitter);
+    if (convertedClosure)
+        convertedClosure->decvref(emitter);
     return new ConcreteCompilerVariable(typeFromClass(function_cls), boxed, true);
 }
 
@@ -1507,6 +1510,23 @@ public:
 } _CLOSURE;
 ConcreteCompilerType* CLOSURE = &_CLOSURE;
 
+class GeneratorType : public ConcreteCompilerType {
+public:
+    llvm::Type* llvmType() { return g.llvm_generator_type_ptr; }
+    std::string debugName() { return "generator"; }
+
+    virtual ConcreteCompilerType* getConcreteType() { return this; }
+    virtual ConcreteCompilerType* getBoxType() { return GENERATOR; }
+
+    virtual void drop(IREmitter& emitter, VAR* var) {
+        // pass
+    }
+    virtual void grab(IREmitter& emitter, VAR* var) {
+        // pass
+    }
+} _GENERATOR;
+ConcreteCompilerType* GENERATOR = &_GENERATOR;
+
 class StrConstantType : public ValuedCompilerType<std::string*> {
 public:
     std::string debugName() { return "str_constant"; }

src/codegen/compvars.h

@@ -30,7 +30,7 @@ class CompilerType;
 class IREmitter;
 
 extern ConcreteCompilerType* INT, *BOXED_INT, *FLOAT, *BOXED_FLOAT, *VOID, *UNKNOWN, *BOOL, *STR, *NONE, *LIST, *SLICE,
-    *MODULE, *DICT, *BOOL, *BOXED_BOOL, *BOXED_TUPLE, *SET, *FROZENSET, *CLOSURE;
+    *MODULE, *DICT, *BOOL, *BOXED_BOOL, *BOXED_TUPLE, *SET, *FROZENSET, *CLOSURE, *GENERATOR;
 extern CompilerType* UNDEF;
 
 class CompilerType {
@@ -348,7 +348,7 @@ ConcreteCompilerVariable* makeInt(int64_t);
 ConcreteCompilerVariable* makeFloat(double);
 ConcreteCompilerVariable* makeBool(bool);
 CompilerVariable* makeStr(std::string*);
-CompilerVariable* makeFunction(IREmitter& emitter, CLFunction*, CompilerVariable* closure,
+CompilerVariable* makeFunction(IREmitter& emitter, CLFunction*, CompilerVariable* closure, bool isGenerator,
                                const std::vector<ConcreteCompilerVariable*>& defaults);
 ConcreteCompilerVariable* undefVariable();
 CompilerVariable* makeTuple(const std::vector<CompilerVariable*>& elts);

src/codegen/irgen.cpp

@@ -914,10 +914,14 @@ CompiledFunction* doCompile(SourceInfo* source, const OSREntryDescriptor* entry_
     int nargs = source->arg_names.totalParameters();
     ASSERT(nargs == spec->arg_types.size(), "%d %ld", nargs, spec->arg_types.size());
 
+
     std::vector<llvm::Type*> llvm_arg_types;
     if (source->scoping->getScopeInfoForNode(source->ast)->takesClosure())
         llvm_arg_types.push_back(g.llvm_closure_type_ptr);
 
+    if (source->scoping->getScopeInfoForNode(source->ast)->takesGenerator())
+        llvm_arg_types.push_back(g.llvm_generator_type_ptr);
+
     if (entry_descriptor == NULL) {
         for (int i = 0; i < nargs; i++) {
             if (i == 3) {

src/codegen/irgen/hooks.cpp

@@ -238,7 +238,7 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm) {
     }
 
     if (cf->is_interpreted)
-        interpretFunction(cf->func, 0, NULL, NULL, NULL, NULL, NULL);
+        interpretFunction(cf->func, 0, NULL, NULL, NULL, NULL, NULL, NULL);
     else
         ((void (*)())cf->code)();
 }

src/codegen/irgen/irgenerator.cpp

@@ -27,7 +27,9 @@
 #include "codegen/type_recording.h"
 #include "core/ast.h"
 #include "core/cfg.h"
+#include "core/types.h"
 #include "core/util.h"
+#include "runtime/generator.h"
 #include "runtime/objmodel.h"
 #include "runtime/types.h"
 
@@ -197,6 +199,7 @@ static std::vector<const std::string*>* getKeywordNameStorage(AST_Call* node) {
 
 static const std::string CREATED_CLOSURE_NAME = "!created_closure";
 static const std::string PASSED_CLOSURE_NAME = "!passed_closure";
+static const std::string PASSED_GENERATOR_NAME = "!passed_generator";
 
 class IRGeneratorImpl : public IRGenerator {
 private:
@@ -841,6 +844,25 @@ private:
         }
     }
 
+    CompilerVariable* evalYield(AST_Yield* node, ExcInfo exc_info) {
+        assert(state != PARTIAL);
+
+        CompilerVariable* generator = _getFake(PASSED_GENERATOR_NAME, false);
+        ConcreteCompilerVariable* convertedGenerator = generator->makeConverted(emitter, generator->getBoxType());
+
+
+        CompilerVariable* value = node->value ? evalExpr(node->value, exc_info) : getNone();
+        ConcreteCompilerVariable* convertedValue = value->makeConverted(emitter, value->getBoxType());
+        value->decvref(emitter);
+
+        llvm::Value* rtn = emitter.createCall2(exc_info, g.funcs.yield, convertedGenerator->getValue(),
+                                               convertedValue->getValue()).getInstruction();
+        convertedGenerator->decvref(emitter);
+        convertedValue->decvref(emitter);
+
+        return new ConcreteCompilerVariable(UNKNOWN, rtn, true);
+    }
+
     ConcreteCompilerVariable* unboxVar(ConcreteCompilerType* t, llvm::Value* v, bool grabbed) {
         assert(state != PARTIAL);
 
@@ -923,6 +945,10 @@ private:
                 case AST_TYPE::UnaryOp:
                     rtn = evalUnaryOp(ast_cast<AST_UnaryOp>(node), exc_info);
                     break;
+                case AST_TYPE::Yield:
+                    rtn = evalYield(ast_cast<AST_Yield>(node), exc_info);
+                    break;
+
                 case AST_TYPE::ClsAttribute:
                     rtn = evalClsAttribute(ast_cast<AST_ClsAttribute>(node), exc_info);
                     break;
@@ -1283,7 +1309,7 @@ private:
         // one reason to do this is to pass the closure through if necessary,
         // but since the classdef can't create its own closure, shouldn't need to explicitly
         // create that scope to pass the closure through.
-        CompilerVariable* func = makeFunction(emitter, cl, created_closure, {});
+        CompilerVariable* func = makeFunction(emitter, cl, created_closure, false, {});
 
         CompilerVariable* attr_dict = func->call(emitter, getEmptyOpInfo(exc_info), ArgPassSpec(0), {}, NULL);
 
@@ -1438,8 +1464,12 @@ private:
         // Top level functions never take a closure, so we can skip the analysis.
         if (irstate->getSourceInfo()->ast->type == AST_TYPE::Module)
             takes_closure = false;
-        else
+        else {
             takes_closure = irstate->getSourceInfo()->scoping->getScopeInfoForNode(node)->takesClosure();
+        }
+
+        // TODO: this lines disables the optimization mentioned above...
+        bool is_generator = irstate->getSourceInfo()->scoping->getScopeInfoForNode(node)->takesGenerator();
 
         if (takes_closure) {
             if (irstate->getScopeInfo()->createsClosure()) {
@@ -1451,16 +1481,12 @@ private:
             assert(created_closure);
         }
 
-        CompilerVariable* func = makeFunction(emitter, cl, created_closure, defaults);
+        CompilerVariable* func = makeFunction(emitter, cl, created_closure, is_generator, defaults);
 
         for (auto d : defaults) {
             d->decvref(emitter);
         }
 
-        // llvm::Type* boxCLFuncArgType = g.funcs.boxCLFunction->arg_begin()->getType();
-        // llvm::Value *boxed = emitter.getBuilder()->CreateCall(g.funcs.boxCLFunction, embedConstantPtr(cl,
-        // boxCLFuncArgType));
-        // CompilerVariable *func = new ConcreteCompilerVariable(typeFromClass(function_cls), boxed, true);
         return func;
     }
 
@@ -1920,7 +1946,8 @@ private:
     }
 
     bool allowableFakeEndingSymbol(const std::string& name) {
-        return startswith(name, "!is_defined") || name == PASSED_CLOSURE_NAME || name == CREATED_CLOSURE_NAME;
+        return startswith(name, "!is_defined") || name == PASSED_CLOSURE_NAME || name == CREATED_CLOSURE_NAME
+               || name == PASSED_GENERATOR_NAME;
     }
 
     void endBlock(State new_state) {
@@ -2027,7 +2054,6 @@ public:
             }
             st->clear();
             symbol_table.clear();
-
             return EndingState(st, phi_st, curblock);
         } else if (myblock->successors.size() > 1) {
             // Since there are no critical edges, all successors come directly from this node,
@@ -2058,6 +2084,8 @@ public:
                     ending_type = getPassedClosureType();
                 } else if (it->first == CREATED_CLOSURE_NAME) {
                     ending_type = getCreatedClosureType();
+                } else if (it->first == PASSED_GENERATOR_NAME) {
+                    ending_type = GENERATOR;
                 } else {
                     ending_type = types->getTypeAtBlockEnd(it->first, myblock);
                 }
@@ -2110,6 +2138,7 @@ public:
 
         llvm::Value* passed_closure = NULL;
         llvm::Function::arg_iterator AI = irstate->getLLVMFunction()->arg_begin();
+
         if (scope_info->takesClosure()) {
             passed_closure = AI;
             _setFake(PASSED_CLOSURE_NAME, new ConcreteCompilerVariable(getPassedClosureType(), AI, true));
@@ -2124,6 +2153,11 @@ public:
             _setFake(CREATED_CLOSURE_NAME, new ConcreteCompilerVariable(getCreatedClosureType(), new_closure, true));
         }
 
+        if (scope_info->takesGenerator()) {
+            _setFake(PASSED_GENERATOR_NAME, new ConcreteCompilerVariable(GENERATOR, AI, true));
+            ++AI;
+        }
+
 
         std::vector<llvm::Value*> python_parameters;
         for (int i = 0; i < arg_types.size(); i++) {

src/codegen/llvm_interpreter.cpp

@@ -263,7 +263,8 @@ const LineInfo* getLineInfoForInterpretedFrame(void* frame_ptr) {
     }
 }
 
-Box* interpretFunction(llvm::Function* f, int nargs, Box* closure, Box* arg1, Box* arg2, Box* arg3, Box** args) {
+Box* interpretFunction(llvm::Function* f, int nargs, Box* closure, Box* generator, Box* arg1, Box* arg2, Box* arg3,
+                       Box** args) {
     assert(f);
 
 #ifdef TIME_INTERPRETS
@@ -287,20 +288,24 @@ Box* interpretFunction(llvm::Function* f, int nargs, Box* closure, Box* arg1, Bo
 
     int arg_num = -1;
     int closure_indicator = closure ? 1 : 0;
+    int generator_indicator = generator ? 1 : 0;
+    int arg_offset = closure_indicator + generator_indicator;
     for (llvm::Argument& arg : f->args()) {
         arg_num++;
 
         if (arg_num == 0 && closure)
             symbols.insert(std::make_pair(static_cast<llvm::Value*>(&arg), Val(closure)));
-        else if (arg_num == 0 + closure_indicator)
+        else if ((arg_num == 0 || (arg_num == 1 && closure)) && generator)
+            symbols.insert(std::make_pair(static_cast<llvm::Value*>(&arg), Val(generator)));
+        else if (arg_num == 0 + arg_offset)
             symbols.insert(std::make_pair(static_cast<llvm::Value*>(&arg), Val(arg1)));
-        else if (arg_num == 1 + closure_indicator)
+        else if (arg_num == 1 + arg_offset)
             symbols.insert(std::make_pair(static_cast<llvm::Value*>(&arg), Val(arg2)));
-        else if (arg_num == 2 + closure_indicator)
+        else if (arg_num == 2 + arg_offset)
             symbols.insert(std::make_pair(static_cast<llvm::Value*>(&arg), Val(arg3)));
         else {
-            assert(arg_num == 3 + closure_indicator);
-            assert(f->getArgumentList().size() == 4 + closure_indicator);
+            assert(arg_num == 3 + arg_offset);
+            assert(f->getArgumentList().size() == 4 + arg_offset);
             assert(f->getArgumentList().back().getType() == g.llvm_value_type_ptr->getPointerTo());
             symbols.insert(std::make_pair(static_cast<llvm::Value*>(&arg), Val((int64_t)args)));
             // printf("loading %%4 with %p\n", (void*)args);

src/codegen/llvm_interpreter.h

@@ -25,7 +25,8 @@ class Box;
 class GCVisitor;
 class LineInfo;
 
-Box* interpretFunction(llvm::Function* f, int nargs, Box* closure, Box* arg1, Box* arg2, Box* arg3, Box** args);
+Box* interpretFunction(llvm::Function* f, int nargs, Box* closure, Box* generator, Box* arg1, Box* arg2, Box* arg3,
+                       Box** args);
 
 void gatherInterpreterRoots(GCVisitor* visitor);
 const LineInfo* getLineInfoForInterpretedFrame(void* frame_ptr);

src/codegen/parser.cpp

@@ -677,6 +677,16 @@ AST_With* read_with(BufferedReader* reader) {
     return rtn;
 }
 
+AST_Yield* read_yield(BufferedReader* reader) {
+    AST_Yield* rtn = new AST_Yield();
+
+    rtn->col_offset = readColOffset(reader);
+    rtn->lineno = reader->readULL();
+    rtn->value = readASTExpr(reader);
+
+    return rtn;
+}
+
 AST_expr* readASTExpr(BufferedReader* reader) {
     uint8_t type = reader->readByte();
     if (VERBOSITY("parsing") >= 2)
@@ -728,6 +738,8 @@ AST_expr* readASTExpr(BufferedReader* reader) {
             return read_tuple(reader);
         case AST_TYPE::UnaryOp:
             return read_unaryop(reader);
+        case AST_TYPE::Yield:
+            return read_yield(reader);
         default:
             fprintf(stderr, "Unknown expr node type (parser.cpp:" STRINGIFY(__LINE__) "): %d\n", type);
             abort();

src/codegen/runtime_hooks.cpp

@@ -34,6 +34,7 @@
 #include "core/types.h"
 #include "runtime/float.h"
 #include "runtime/gc_runtime.h"
+#include "runtime/generator.h"
 #include "runtime/inline/boxing.h"
 #include "runtime/int.h"
 #include "runtime/objmodel.h"
@@ -141,6 +142,9 @@ void initGlobalFuncs(GlobalState& g) {
     g.llvm_closure_type_ptr = g.stdlib_module->getTypeByName("class.pyston::BoxedClosure")->getPointerTo();
     assert(g.llvm_closure_type_ptr);
 
+    g.llvm_generator_type_ptr = g.stdlib_module->getTypeByName("class.pyston::BoxedGenerator")->getPointerTo();
+    assert(g.llvm_generator_type_ptr);
+
 #define GET(N) g.funcs.N = getFunc((void*)N, STRINGIFY(N))
 
     g.funcs.printf = addFunc((void*)printf, g.i8_ptr, true);
@@ -166,6 +170,7 @@ void initGlobalFuncs(GlobalState& g) {
     GET(createDict);
     GET(createSlice);
     GET(createClosure);
+    GET(createGenerator);
 
     GET(getattr);
     GET(setattr);
@@ -188,6 +193,7 @@ void initGlobalFuncs(GlobalState& g) {
     GET(importStar);
     GET(repr);
     GET(isinstance);
+    GET(yield);
 
     GET(checkUnpackingLength);
     GET(raiseAttributeError);

src/codegen/runtime_hooks.h

@@ -32,10 +32,10 @@ struct GlobalFuncs {
 
     llvm::Value* boxInt, *unboxInt, *boxFloat, *unboxFloat, *boxStringPtr, *boxCLFunction, *unboxCLFunction,
         *boxInstanceMethod, *boxBool, *unboxBool, *createTuple, *createDict, *createList, *createSlice,
-        *createUserClass, *createClosure;
+        *createUserClass, *createClosure, *createGenerator;
     llvm::Value* getattr, *setattr, *delattr, *delitem, *delGlobal, *print, *nonzero, *binop, *compare, *augbinop,
         *unboxedLen, *getitem, *getclsattr, *getGlobal, *setitem, *unaryop, *import, *importFrom, *importStar, *repr,
-        *isinstance;
+        *isinstance, *yield;
 
     llvm::Value* checkUnpackingLength, *raiseAttributeError, *raiseAttributeErrorStr, *raiseNotIterableError,
         *assertNameDefined, *assertFail;

src/core/ast.cpp

@@ -836,6 +836,18 @@ void AST_With::accept_stmt(StmtVisitor* v) {
     v->visit_with(this);
 }
 
+void AST_Yield::accept(ASTVisitor* v) {
+    bool skip = v->visit_yield(this);
+    if (skip)
+        return;
+
+    if (value)
+        value->accept(v);
+}
+
+void* AST_Yield::accept_expr(ExprVisitor* v) {
+    return v->visit_yield(this);
+}
 
 void AST_Branch::accept(ASTVisitor* v) {
     bool skip = v->visit_branch(this);
@@ -1605,6 +1617,13 @@ bool PrintVisitor::visit_with(AST_With* node) {
     return true;
 }
 
+bool PrintVisitor::visit_yield(AST_Yield* node) {
+    printf("yield ");
+    if (node->value)
+        node->value->accept(this);
+    return true;
+}
+
 bool PrintVisitor::visit_branch(AST_Branch* node) {
     printf("if ");
     node->test->accept(this);
@@ -1838,6 +1857,10 @@ public:
         output->push_back(node);
         return false;
     }
+    virtual bool visit_yield(AST_Yield* node) {
+        output->push_back(node);
+        return false;
+    }
 
     virtual bool visit_branch(AST_Branch* node) {
         output->push_back(node);

src/core/ast.h

@@ -797,6 +797,18 @@ public:
     static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::With;
 };
 
+class AST_Yield : public AST_expr {
+public:
+    AST_expr* value;
+
+    virtual void accept(ASTVisitor* v);
+    virtual void* accept_expr(ExprVisitor* v);
+
+    AST_Yield() : AST_expr(AST_TYPE::Yield) {}
+
+    static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Yield;
+};
+
 
 // AST pseudo-nodes that will get added during CFG-construction.  These don't exist in the input AST, but adding them in
 // lets us avoid creating a completely new IR for this phase
@@ -954,6 +966,7 @@ public:
     virtual bool visit_unreachable(AST_Unreachable* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_while(AST_While* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_with(AST_With* node) { RELEASE_ASSERT(0, ""); }
+    virtual bool visit_yield(AST_Yield* node) { RELEASE_ASSERT(0, ""); }
 
     virtual bool visit_branch(AST_Branch* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_jump(AST_Jump* node) { RELEASE_ASSERT(0, ""); }
@@ -1017,6 +1030,7 @@ public:
     virtual bool visit_unreachable(AST_Unreachable* node) { return false; }
     virtual bool visit_while(AST_While* node) { return false; }
     virtual bool visit_with(AST_With* node) { return false; }
+    virtual bool visit_yield(AST_Yield* node) { return false; }
 
     virtual bool visit_branch(AST_Branch* node) { return false; }
     virtual bool visit_jump(AST_Jump* node) { return false; }
@@ -1050,6 +1064,7 @@ public:
     virtual void* visit_subscript(AST_Subscript* node) { RELEASE_ASSERT(0, ""); }
     virtual void* visit_tuple(AST_Tuple* node) { RELEASE_ASSERT(0, ""); }
     virtual void* visit_unaryop(AST_UnaryOp* node) { RELEASE_ASSERT(0, ""); }
+    virtual void* visit_yield(AST_Yield* node) { RELEASE_ASSERT(0, ""); }
 };
 
 class StmtVisitor {
@@ -1149,6 +1164,7 @@ public:
     virtual bool visit_unreachable(AST_Unreachable* node);
     virtual bool visit_while(AST_While* node);
     virtual bool visit_with(AST_With* node);
+    virtual bool visit_yield(AST_Yield* node);
 
     virtual bool visit_branch(AST_Branch* node);
     virtual bool visit_jump(AST_Jump* node);

src/core/cfg.cpp

@@ -713,6 +713,14 @@ private:
         return rtn;
     }
 
+    AST_expr* remapYield(AST_Yield* node) {
+        AST_Yield* rtn = new AST_Yield();
+        rtn->lineno = node->lineno;
+        rtn->col_offset = node->col_offset;
+        rtn->value = remapExpr(node->value);
+        return rtn;
+    }
+
     AST_expr* remapExpr(AST_expr* node, bool wrap_with_assign = true) {
         if (node == NULL)
             return NULL;
@@ -783,6 +791,9 @@ private:
             case AST_TYPE::UnaryOp:
                 rtn = remapUnaryOp(ast_cast<AST_UnaryOp>(node));
                 break;
+            case AST_TYPE::Yield:
+                rtn = remapYield(ast_cast<AST_Yield>(node));
+                break;
             default:
                 RELEASE_ASSERT(0, "%d", node->type);
         }

src/core/types.h

@@ -166,6 +166,7 @@ struct FunctionSpecialization {
 };
 
 class BoxedClosure;
+class BoxedGenerator;
 struct CompiledFunction {
 private:
 public:
@@ -178,6 +179,8 @@ public:
     union {
         Box* (*call)(Box*, Box*, Box*, Box**);
         Box* (*closure_call)(BoxedClosure*, Box*, Box*, Box*, Box**);
+        Box* (*closure_generator_call)(BoxedClosure*, BoxedGenerator*, Box*, Box*, Box*, Box**);
+        Box* (*generator_call)(BoxedGenerator*, Box*, Box*, Box*, Box**);
         void* code;
     };
     llvm::Value* llvm_code; // the llvm callable.

src/runtime/builtin_modules/builtins.cpp

@@ -378,9 +378,9 @@ BoxedClass* notimplemented_cls;
 BoxedModule* builtins_module;
 
 // TODO looks like CPython and pypy put this into an "exceptions" module:
-BoxedClass* Exception, *AssertionError, *AttributeError, *TypeError, *NameError, *KeyError, *IndexError, *IOError,
-    *OSError, *ZeroDivisionError, *ValueError, *UnboundLocalError, *RuntimeError, *ImportError, *StopIteration,
-    *Warning;
+BoxedClass* Exception, *AssertionError, *AttributeError, *GeneratorExit, *TypeError, *NameError, *KeyError, *IndexError,
+    *IOError, *OSError, *ZeroDivisionError, *ValueError, *UnboundLocalError, *RuntimeError, *ImportError,
+    *StopIteration, *Warning;
 
 const ObjectFlavor exception_flavor(&boxGCHandler, NULL);
 Box* exceptionNew1(BoxedClass* cls) {
@@ -456,6 +456,7 @@ void setupBuiltins() {
     Exception = makeBuiltinException(object_cls, "Exception");
     AssertionError = makeBuiltinException(Exception, "AssertionError");
     AttributeError = makeBuiltinException(Exception, "AttributeError");
+    GeneratorExit = makeBuiltinException(Exception, "GeneratorExit");
     TypeError = makeBuiltinException(Exception, "TypeError");
     NameError = makeBuiltinException(Exception, "NameError");
     KeyError = makeBuiltinException(Exception, "KeyError");

src/runtime/generator.cpp

+// Copyright (c) 2014 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "runtime/generator.h"
+
+#include <algorithm>
+#include <cstddef>
+#include <cstring>
+#include <ucontext.h>
+
+#include "codegen/compvars.h"
+#include "codegen/llvm_interpreter.h"
+#include "core/ast.h"
+#include "core/common.h"
+#include "core/stats.h"
+#include "core/types.h"
+#include "gc/collector.h"
+#include "runtime/gc_runtime.h"
+#include "runtime/objmodel.h"
+#include "runtime/types.h"
+#include "runtime/util.h"
+
+namespace pyston {
+
+
+static void generatorEntry(BoxedGenerator* g) {
+    assert(g->cls == generator_cls);
+    assert(g->function->cls == function_cls);
+
+    try {
+        // call body of the generator
+        BoxedFunction* func = g->function;
+
+        Box** args = g->args ? &g->args->elts[0] : nullptr;
+        callCLFunc(func->f, nullptr, func->f->numReceivedArgs(), func->closure, g, g->arg1, g->arg2, g->arg3, args);
+    } catch (Box* e) {
+        // unhandled exception: propagate the exception to the caller
+        g->exception = e;
+    }
+
+    // we returned from the body of the generator. next/send/throw will notify the caller
+    g->entryExited = true;
+    swapcontext(&g->context, &g->returnContext);
+}
+
+Box* generatorIter(Box* s) {
+    return s;
+}
+
+Box* generatorSend(Box* s, Box* v) {
+    assert(s->cls == generator_cls);
+    BoxedGenerator* self = static_cast<BoxedGenerator*>(s);
+
+    // check if the generator already exited
+    if (self->entryExited)
+        raiseExcHelper(StopIteration, "");
+
+    self->returnValue = v;
+    swapcontext(&self->returnContext, &self->context);
+
+    // propagate exception to the caller
+    if (self->exception)
+        raiseExc(self->exception);
+
+    // throw StopIteration if the generator exited
+    if (self->entryExited)
+        raiseExcHelper(StopIteration, "");
+
+    return self->returnValue;
+}
+
+Box* generatorThrow(Box* s, BoxedClass* e) {
+    assert(s->cls == generator_cls);
+    assert(isSubclass(e, Exception));
+    BoxedGenerator* self = static_cast<BoxedGenerator*>(s);
+    self->exception = exceptionNew1(e);
+    return generatorSend(self, None);
+}
+
+Box* generatorClose(Box* s) {
+    assert(s->cls == generator_cls);
+    BoxedGenerator* self = static_cast<BoxedGenerator*>(s);
+
+    // check if the generator already exited
+    if (self->entryExited)
+        return None;
+
+    return generatorThrow(self, GeneratorExit);
+}
+
+Box* generatorNext(Box* s) {
+    return generatorSend(s, None);
+}
+
+extern "C" Box* yield(BoxedGenerator* obj, Box* value) {
+    assert(obj->cls == generator_cls);
+    BoxedGenerator* self = static_cast<BoxedGenerator*>(obj);
+    self->returnValue = value;
+
+    swapcontext(&self->context, &self->returnContext);
+
+    // if the generator receives a exception from the caller we have to throw it
+    if (self->exception) {
+        Box* exception = self->exception;
+        self->exception = nullptr;
+        raiseExc(exception);
+    }
+    return self->returnValue;
+}
+
+
+extern "C" BoxedGenerator* createGenerator(BoxedFunction* function, Box* arg1, Box* arg2, Box* arg3, Box** args) {
+    assert(function);
+    assert(function->cls == function_cls);
+    return new BoxedGenerator(function, arg1, arg2, arg3, args);
+}
+
+
+extern "C" BoxedGenerator::BoxedGenerator(BoxedFunction* function, Box* arg1, Box* arg2, Box* arg3, Box** args)
+    : Box(&generator_flavor, generator_cls), function(function), arg1(arg1), arg2(arg2), arg3(arg3), args(nullptr),
+      entryExited(false), returnValue(nullptr), exception(nullptr) {
+
+    giveAttr("__name__", boxString(function->f->source->getName()));
+
+    int numArgs = function->f->num_args;
+    if (numArgs > 3) {
+        numArgs -= 3;
+        this->args = new (numArgs) GCdArray();
+        memcpy(&this->args->elts[0], args, numArgs * sizeof(Box*));
+    }
+
+    getcontext(&context);
+    context.uc_link = 0;
+    context.uc_stack.ss_sp = stack;
+    context.uc_stack.ss_size = STACK_SIZE;
+    makecontext(&context, (void (*)(void))generatorEntry, 1, this);
+}
+
+
+void setupGenerator() {
+    generator_cls = new BoxedClass(object_cls, offsetof(BoxedGenerator, attrs), sizeof(BoxedGenerator), false);
+    generator_cls->giveAttr("__name__", boxStrConstant("generator"));
+    generator_cls->giveAttr("__iter__",
+                            new BoxedFunction(boxRTFunction((void*)generatorIter, typeFromClass(generator_cls), 1)));
+
+    generator_cls->giveAttr("close", new BoxedFunction(boxRTFunction((void*)generatorClose, UNKNOWN, 1)));
+    generator_cls->giveAttr("next", new BoxedFunction(boxRTFunction((void*)generatorNext, UNKNOWN, 1)));
+    generator_cls->giveAttr("send", new BoxedFunction(boxRTFunction((void*)generatorSend, UNKNOWN, 2)));
+    generator_cls->giveAttr("throw", new BoxedFunction(boxRTFunction((void*)generatorThrow, UNKNOWN, 2)));
+
+    gc::registerStaticRootObj(generator_cls);
+    generator_cls->freeze();
+}
+}

src/runtime/generator.h

+// Copyright (c) 2014 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef PYSTON_RUNTIME_GENERATOR_H
+#define PYSTON_RUNTIME_GENERATOR_H
+
+#include "core/types.h"
+#include "runtime/types.h"
+
+namespace pyston {
+
+extern BoxedClass* generator_cls;
+extern "C" const ObjectFlavor generator_flavor;
+
+void setupGenerator();
+
+extern "C" Box* yield(BoxedGenerator* obj, Box* value);
+extern "C" BoxedGenerator* createGenerator(BoxedFunction* function, Box* arg1, Box* arg2, Box* arg3, Box** args);
+}
+
+#endif

src/runtime/inline/link_forcer.cpp

@@ -19,6 +19,7 @@
 #include "gc/heap.h"
 #include "runtime/float.h"
 #include "runtime/gc_runtime.h"
+#include "runtime/generator.h"
 #include "runtime/inline/boxing.h"
 #include "runtime/int.h"
 #include "runtime/list.h"
@@ -58,6 +59,7 @@ void force() {
     FORCE(createSlice);
     FORCE(createUserClass);
     FORCE(createClosure);
+    FORCE(createGenerator);
 
     FORCE(getattr);
     FORCE(setattr);
@@ -80,6 +82,7 @@ void force() {
     FORCE(importStar);
     FORCE(repr);
     FORCE(isinstance);
+    FORCE(yield);
 
     FORCE(checkUnpackingLength);
     FORCE(raiseAttributeError);

src/runtime/objmodel.cpp

@@ -41,6 +41,7 @@
 #include "runtime/capi.h"
 #include "runtime/float.h"
 #include "runtime/gc_runtime.h"
+#include "runtime/generator.h"
 #include "runtime/types.h"
 #include "runtime/util.h"
 
@@ -1755,7 +1756,7 @@ Box* callFunc(BoxedFunction* func, CallRewriteArgs* rewrite_args, ArgPassSpec ar
 
     BoxedClosure* closure = func->closure;
 
-    if (argspec.has_starargs || argspec.has_kwargs || f->takes_kwargs)
+    if (argspec.has_starargs || argspec.has_kwargs || f->takes_kwargs || func->isGenerator)
         rewrite_args = NULL;
 
     // These could be handled:
@@ -1988,14 +1989,24 @@ Box* callFunc(BoxedFunction* func, CallRewriteArgs* rewrite_args, ArgPassSpec ar
         getArg(i, oarg1, oarg2, oarg3, oargs) = default_obj;
     }
 
+    // special handling for generators:
+    // the call to function containing a yield should just create a new generator object.
+    if (func->isGenerator) {
+        return createGenerator(func, oarg1, oarg2, oarg3, oargs);
+    }
 
+    return callCLFunc(f, rewrite_args, num_output_args, closure, NULL, oarg1, oarg2, oarg3, oargs);
+}
 
+Box* callCLFunc(CLFunction* f, CallRewriteArgs* rewrite_args, int num_output_args, BoxedClosure* closure,
+                BoxedGenerator* generator, Box* oarg1, Box* oarg2, Box* oarg3, Box** oargs) {
     CompiledFunction* chosen_cf = pickVersion(f, num_output_args, oarg1, oarg2, oarg3, oargs);
 
     assert(chosen_cf->is_interpreted == (chosen_cf->code == NULL));
     if (chosen_cf->is_interpreted) {
-        return interpretFunction(chosen_cf->func, num_output_args, func->closure, oarg1, oarg2, oarg3, oargs);
-    } else {
+        return interpretFunction(chosen_cf->func, num_output_args, closure, generator, oarg1, oarg2, oarg3, oargs);
+    }
+
     if (rewrite_args) {
         rewrite_args->rewriter->addDependenceOn(chosen_cf->dependent_callsites);
 
@@ -2005,11 +2016,14 @@ Box* callFunc(BoxedFunction* func, CallRewriteArgs* rewrite_args, ArgPassSpec ar
         rewrite_args->out_success = true;
     }
 
-        if (closure)
+    if (closure && generator)
+        return chosen_cf->closure_generator_call(closure, generator, oarg1, oarg2, oarg3, oargs);
+    else if (closure)
         return chosen_cf->closure_call(closure, oarg1, oarg2, oarg3, oargs);
+    else if (generator)
+        return chosen_cf->generator_call(generator, oarg1, oarg2, oarg3, oargs);
     else
         return chosen_cf->call(oarg1, oarg2, oarg3, oargs);
-    }
 }
 
 

src/runtime/objmodel.h

@@ -27,10 +27,12 @@ class BoxedClass;
 class BoxedInt;
 class BoxedList;
 class BoxedString;
+class BoxedGenerator;
 
 // user-level raise functions that implement python-level semantics
 extern "C" void raise0() __attribute__((__noreturn__));
 extern "C" void raise3(Box*, Box*, Box*) __attribute__((__noreturn__));
+void raiseExc(Box* exc_obj) __attribute__((__noreturn__));
 
 // helper function for raising from the runtime:
 void raiseExcHelper(BoxedClass*, const char* fmt, ...) __attribute__((__noreturn__));
@@ -111,5 +113,8 @@ extern "C" void raiseNotIterableError(const char* typeName) __attribute__((__nor
 Box* typeCall(Box*, BoxedList*);
 Box* typeNew(Box*, Box*);
 bool isUserDefined(BoxedClass* cls);
+
+Box* callCLFunc(CLFunction* f, CallRewriteArgs* rewrite_args, int num_output_args, BoxedClosure* closure,
+                BoxedGenerator* generator, Box* oarg1, Box* oarg2, Box* oarg3, Box** oargs);
 }
 #endif

src/runtime/stacktrace.cpp

@@ -113,7 +113,6 @@ void raiseRaw(Box* exc_obj) {
     // unwindExc(exc_obj);
 }
 
-void raiseExc(Box* exc_obj) __attribute__((__noreturn__));
 void raiseExc(Box* exc_obj) {
     auto entries = getTracebackEntries();
     last_tb = std::move(entries);

src/runtime/types.cpp

@@ -44,12 +44,24 @@ BoxIterator& BoxIterator::operator++() {
     static std::string next_str("next");
 
     Box* hasnext = callattrInternal(iter, &hasnext_str, CLASS_ONLY, NULL, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
+    if (hasnext) {
         if (nonzero(hasnext)) {
             value = callattrInternal(iter, &next_str, CLASS_ONLY, NULL, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
         } else {
             iter = nullptr;
             value = nullptr;
         }
+    } else {
+        try {
+            value = callattrInternal(iter, &next_str, CLASS_ONLY, NULL, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
+        } catch (Box* e) {
+            if ((e == StopIteration) || isSubclass(e->cls, StopIteration)) {
+                iter = nullptr;
+                value = nullptr;
+            } else
+                throw;
+        }
+    }
     return *this;
 }
 
@@ -65,7 +77,7 @@ llvm::iterator_range<BoxIterator> Box::pyElements() {
 }
 
 extern "C" BoxedFunction::BoxedFunction(CLFunction* f)
-    : Box(&function_flavor, function_cls), f(f), closure(NULL), ndefaults(0), defaults(NULL) {
+    : Box(&function_flavor, function_cls), f(f), closure(NULL), isGenerator(false), ndefaults(0), defaults(NULL) {
     if (f->source) {
         assert(f->source->ast);
         // this->giveAttr("__name__", boxString(&f->source->ast->name));
@@ -80,8 +92,10 @@ extern "C" BoxedFunction::BoxedFunction(CLFunction* f)
     assert(f->num_defaults == ndefaults);
 }
 
-extern "C" BoxedFunction::BoxedFunction(CLFunction* f, std::initializer_list<Box*> defaults, BoxedClosure* closure)
-    : Box(&function_flavor, function_cls), f(f), closure(closure), ndefaults(0), defaults(NULL) {
+extern "C" BoxedFunction::BoxedFunction(CLFunction* f, std::initializer_list<Box*> defaults, BoxedClosure* closure,
+                                        bool isGenerator)
+    : Box(&function_flavor, function_cls), f(f), closure(closure), isGenerator(isGenerator), ndefaults(0),
+      defaults(NULL) {
     if (defaults.size()) {
         // make sure to initialize defaults first, since the GC behavior is triggered by ndefaults,
         // and a GC can happen within this constructor:
@@ -139,11 +153,12 @@ std::string BoxedModule::name() {
     }
 }
 
-extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, std::initializer_list<Box*> defaults) {
+extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, bool isGenerator,
+                              std::initializer_list<Box*> defaults) {
     if (closure)
         assert(closure->cls == closure_cls);
 
-    return new BoxedFunction(f, defaults, closure);
+    return new BoxedFunction(f, defaults, closure, isGenerator);
 }
 
 extern "C" CLFunction* unboxCLFunction(Box* b) {
@@ -265,10 +280,37 @@ extern "C" void closureGCHandler(GCVisitor* v, void* p) {
         v->visit(c->parent);
 }
 
+extern "C" void generatorGCHandler(GCVisitor* v, void* p) {
+    boxGCHandler(v, p);
+
+    BoxedGenerator* g = (BoxedGenerator*)p;
+
+    v->visit(g->function);
+    int num_args = g->function->f->num_args;
+    if (num_args >= 1)
+        v->visit(g->arg1);
+    if (num_args >= 2)
+        v->visit(g->arg2);
+    if (num_args >= 3)
+        v->visit(g->arg3);
+    if (num_args > 3)
+        v->visitPotentialRange(reinterpret_cast<void* const*>(&g->args->elts[0]),
+                               reinterpret_cast<void* const*>(&g->args->elts[num_args - 3]));
+    if (g->returnValue)
+        v->visit(g->returnValue);
+    if (g->exception)
+        v->visit(g->exception);
+
+    v->visitPotentialRange((void**)&g->context, ((void**)&g->context) + sizeof(g->context) / sizeof(void*));
+    v->visitPotentialRange((void**)&g->returnContext,
+                           ((void**)&g->returnContext) + sizeof(g->returnContext) / sizeof(void*));
+    v->visitPotentialRange((void**)&g->stack[0], (void**)&g->stack[BoxedGenerator::STACK_SIZE]);
+}
+
 extern "C" {
 BoxedClass* object_cls, *type_cls, *none_cls, *bool_cls, *int_cls, *float_cls, *str_cls, *function_cls,
     *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls, *file_cls, *member_cls,
-    *closure_cls;
+    *closure_cls, *generator_cls;
 
 const ObjectFlavor object_flavor(&boxGCHandler, NULL);
 const ObjectFlavor type_flavor(&typeGCHandler, NULL);
@@ -287,6 +329,7 @@ const ObjectFlavor tuple_flavor(&tupleGCHandler, NULL);
 const ObjectFlavor file_flavor(&boxGCHandler, NULL);
 const ObjectFlavor member_flavor(&boxGCHandler, NULL);
 const ObjectFlavor closure_flavor(&closureGCHandler, NULL);
+const ObjectFlavor generator_flavor(&generatorGCHandler, NULL);
 
 const AllocationKind untracked_kind(NULL, NULL);
 const AllocationKind hc_kind(&hcGCHandler, NULL);
@@ -585,6 +628,7 @@ void setupRuntime() {
     setupSet();
     setupTuple();
     setupFile();
+    setupGenerator();
 
     function_cls->giveAttr("__name__", boxStrConstant("function"));
     function_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)functionRepr, STR, 1)));

src/runtime/types.h

@@ -15,6 +15,8 @@
 #ifndef PYSTON_RUNTIME_TYPES_H
 #define PYSTON_RUNTIME_TYPES_H
 
+#include <ucontext.h>
+
 #include "core/threading.h"
 #include "core/types.h"
 
@@ -28,6 +30,7 @@ class BoxedDict;
 class BoxedTuple;
 class BoxedFile;
 class BoxedClosure;
+class BoxedGenerator;
 
 void setupInt();
 void teardownInt();
@@ -51,6 +54,7 @@ void setupFile();
 void teardownFile();
 void setupCAPI();
 void teardownCAPI();
+void setupGenerator();
 
 void setupSys();
 void setupBuiltins();
@@ -68,12 +72,12 @@ BoxedList* getSysPath();
 extern "C" {
 extern BoxedClass* object_cls, *type_cls, *bool_cls, *int_cls, *float_cls, *str_cls, *function_cls, *none_cls,
     *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls, *file_cls, *xrange_cls, *member_cls,
-    *closure_cls;
+    *closure_cls, *generator_cls;
 }
 extern "C" {
 extern const ObjectFlavor object_flavor, type_flavor, bool_flavor, int_flavor, float_flavor, str_flavor,
     function_flavor, none_flavor, instancemethod_flavor, list_flavor, slice_flavor, module_flavor, dict_flavor,
-    tuple_flavor, file_flavor, xrange_flavor, member_flavor, closure_flavor;
+    tuple_flavor, file_flavor, xrange_flavor, member_flavor, closure_flavor, generator_flavor;
 }
 extern "C" { extern Box* None, *NotImplemented, *True, *False; }
 extern "C" {
@@ -92,7 +96,8 @@ Box* boxString(const std::string& s);
 extern "C" BoxedString* boxStrConstant(const char* chars);
 extern "C" void listAppendInternal(Box* self, Box* v);
 extern "C" void listAppendArrayInternal(Box* self, Box** v, int nelts);
-extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, std::initializer_list<Box*> defaults);
+extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, bool isGenerator,
+                              std::initializer_list<Box*> defaults);
 extern "C" CLFunction* unboxCLFunction(Box* b);
 extern "C" Box* createUserClass(std::string* name, Box* base, Box* attr_dict);
 extern "C" double unboxFloat(Box* b);
@@ -280,11 +285,13 @@ public:
     CLFunction* f;
     BoxedClosure* closure;
 
+    bool isGenerator;
     int ndefaults;
     GCdArray* defaults;
 
     BoxedFunction(CLFunction* f);
-    BoxedFunction(CLFunction* f, std::initializer_list<Box*> defaults, BoxedClosure* closure = NULL);
+    BoxedFunction(CLFunction* f, std::initializer_list<Box*> defaults, BoxedClosure* closure = NULL,
+                  bool isGenerator = false);
 };
 
 class BoxedModule : public Box {
@@ -323,6 +330,25 @@ public:
     BoxedClosure(BoxedClosure* parent) : Box(&closure_flavor, closure_cls), parent(parent) {}
 };
 
+class BoxedGenerator : public Box {
+public:
+    enum { STACK_SIZE = SIGSTKSZ * 5 };
+
+    HCAttrs attrs;
+    BoxedFunction* function;
+    Box* arg1, *arg2, *arg3;
+    GCdArray* args;
+
+    bool entryExited;
+    Box* returnValue;
+    Box* exception;
+
+    ucontext_t context, returnContext;
+    char stack[STACK_SIZE];
+
+    BoxedGenerator(BoxedFunction* function, Box* arg1, Box* arg2, Box* arg3, Box** args);
+};
+
 extern "C" void boxGCHandler(GCVisitor* v, void* p);
 
 Box* exceptionNew1(BoxedClass* cls);
@@ -330,7 +356,7 @@ Box* exceptionNew2(BoxedClass* cls, Box* message);
 
 extern BoxedClass* Exception, *AssertionError, *AttributeError, *TypeError, *NameError, *KeyError, *IndexError,
     *IOError, *OSError, *ZeroDivisionError, *ValueError, *UnboundLocalError, *RuntimeError, *ImportError,
-    *StopIteration;
+    *StopIteration, *GeneratorExit;
 
 // cls should be obj->cls.
 // Added as parameter because it should typically be available

test/tests/generators.py

+def G1(i=0):
+    while True:
+        yield i
+        i += i
+
+g1 = G1();
+for i in range(5):
+    print g1.next()
+
+
+
+def G2():
+    yield 1
+    yield 2
+    yield 3
+g2a = G2()
+g2b = G2()
+print g2b.next()
+print list(g2a)
+print list(g2b)
+print list(g2a)
+print list(G2())
+
+
+
+def G3(i=0):
+    while True:
+        got = (yield i**2)
+        print "i=", i, "got=", got
+        i += 1
+
+g3 = G3();
+g3.send(None)
+for i in range(5):
+    r = g3.send(i)
+    print "received= ", r
+
+
+
+def G4(i=1):
+    1/0
+    while True:
+        print "unreachable"
+
+try:
+    print list(G4(0))
+except ZeroDivisionError:
+    print "catched a ZeroDivisionError"
+    
+
+def G5():
+    i = 0
+    try:
+        while True:
+            yield i
+            i += 1
+    except:
+        print "catched a ZeroDivisionError inside G5"
+    yield 42
+
+g5 = G5()
+for i in range(5):
+    print g5.next()
+print g5.throw(ZeroDivisionError)
+
+def G6(a=[]):
+    for i in range(2):
+        a.append(i)
+        yield a
+print list(G6())
+print list(G6())
+
+def G7(p):
+    a = p
+    b = 2
+    def G():
+        yield a+b
+    return G()
+print list(G7(1))
+
+def G8(*args):
+    for a in args:
+        yield a
+print list(G8(1, 2, 3, 4, 5))
+
+def G9(**kwargs):
+    for a in sorted(kwargs.keys()):
+        yield a, kwargs[a]
+print list(G9(a="1", b="2", c="3", d="4", e="5"))

test/tests/stop_iterable.py

+class C(object):
+  i = 0
+  def next(self):
+    if self.i<5:
+        self.i += 1
+        return self.i
+    else:
+        raise StopIteration
+  def __iter__(self):
+    return self
+
+print list(C())
+
+