basic eval works (without name resolution really working)

src/analysis/scoping_analysis.cpp

@@ -182,7 +182,8 @@ private:
 public:
     ScopeInfoBase(ScopeInfo* parent, ScopingAnalysis::ScopeNameUsage* usage, AST* ast, bool usesNameLookup)
         : parent(parent), usage(usage), ast(ast), usesNameLookup(usesNameLookup) {
-        assert(parent);
+        // not true anymore: Expression
+        // assert(parent);
         assert(usage);
         assert(ast);
     }
@@ -314,6 +315,7 @@ public:
     bool visit_keyword(AST_keyword* node) override { return false; }
     bool visit_list(AST_List* node) override { return false; }
     bool visit_listcomp(AST_ListComp* node) override { return false; }
+    bool visit_expression(AST_Expression* node) override { return false; }
     // bool visit_module(AST_Module *node) override { return false; }
     // bool visit_name(AST_Name *node) override { return false; }
     bool visit_num(AST_Num* node) override { return false; }
@@ -550,6 +552,7 @@ void ScopingAnalysis::processNameUsages(ScopingAnalysis::NameUsageMap* usages) {
         ScopeInfo* parent_info = this->scopes[(usage->parent == NULL) ? this->parent_module : usage->parent->node];
 
         switch (node->type) {
+            case AST_TYPE::Expression:
             case AST_TYPE::ClassDef: {
                 ScopeInfoBase* scopeInfo
                     = new ScopeInfoBase(parent_info, usage, usage->node, true /* usesNameLookup */);
@@ -572,8 +575,7 @@ void ScopingAnalysis::processNameUsages(ScopingAnalysis::NameUsageMap* usages) {
 }
 
 InternedStringPool& ScopingAnalysis::getInternedStrings() {
-    assert(parent_module);
-    return *parent_module->interned_strings.get();
+    return interned_strings;
 }
 
 ScopeInfo* ScopingAnalysis::analyzeSubtree(AST* node) {
@@ -624,4 +626,8 @@ ScopingAnalysis::ScopingAnalysis(AST_Module* m) : parent_module(m), interned_str
 ScopingAnalysis* runScopingAnalysis(AST_Module* m) {
     return new ScopingAnalysis(m);
 }
+
+ScopingAnalysis::ScopingAnalysis(AST_Expression* e) : interned_strings(*e->interned_strings.get()) {
+    scopes[e] = getScopeInfoForNode(e);
+}
 }

src/analysis/scoping_analysis.h

@@ -22,6 +22,7 @@ namespace pyston {
 
 class AST;
 class AST_Module;
+class AST_Expression;
 
 class ScopeInfo {
 public:
@@ -97,6 +98,7 @@ public:
     void registerScopeReplacement(AST* original_node, AST* new_node);
 
     ScopingAnalysis(AST_Module* m);
+    ScopingAnalysis(AST_Expression* e);
     ScopeInfo* getScopeInfoForNode(AST* node);
 
     InternedStringPool& getInternedStrings();

src/codegen/codegen.cpp

@@ -39,6 +39,7 @@ SourceInfo::SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, AST* ast, const
         case AST_TYPE::ClassDef:
         case AST_TYPE::Lambda:
         case AST_TYPE::Module:
+        case AST_TYPE::Expression:
             is_generator = false;
             break;
         case AST_TYPE::FunctionDef:

src/codegen/irgen/hooks.cpp

@@ -27,6 +27,7 @@
 #include "codegen/irgen/future.h"
 #include "codegen/irgen/util.h"
 #include "codegen/osrentry.h"
+#include "codegen/parser.h"
 #include "codegen/patchpoints.h"
 #include "codegen/stackmaps.h"
 #include "core/ast.h"
@@ -43,7 +44,7 @@ namespace pyston {
 
 // TODO terrible place for these!
 ParamNames::ParamNames(AST* ast) : takes_param_names(true) {
-    if (ast->type == AST_TYPE::Module || ast->type == AST_TYPE::ClassDef) {
+    if (ast->type == AST_TYPE::Module || ast->type == AST_TYPE::ClassDef || ast->type == AST_TYPE::Expression) {
         kwarg = "";
         vararg = "";
     } else if (ast->type == AST_TYPE::FunctionDef || ast->type == AST_TYPE::Lambda) {
@@ -93,6 +94,7 @@ const std::string SourceInfo::getName() {
         case AST_TYPE::Lambda:
             return "<lambda>";
         case AST_TYPE::Module:
+        case AST_TYPE::Expression:
             return "<module>";
         default:
             RELEASE_ASSERT(0, "%d", ast->type);
@@ -300,19 +302,44 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm) {
         ((void (*)())cf->code)();
 }
 
-/*
-void compileAndRunExpression(AST_expr* expr, BoxedModule* bm) {
+static Box* compileAndRunExpression(AST_Expression* expr, BoxedModule* bm) {
     CompiledFunction* cf;
-    AST_stmt* stmt = new AST_Expr(expr);
 
     { // scope for limiting the locked region:
         LOCK_REGION(codegen_rwlock.asWrite());
 
-        Timer _t("for compileModule()");
+        Timer _t("for compileEval()");
+
+        ScopingAnalysis* scoping = new ScopingAnalysis(expr);
+
+        AST_Return* stmt = new AST_Return();
+        stmt->value = expr->body;
+        SourceInfo* si = new SourceInfo(bm, scoping, expr, { stmt });
+        CLFunction* cl_f = new CLFunction(0, 0, false, false, si);
+
+        EffortLevel::EffortLevel effort = initialEffort();
+
+        cf = compileFunction(cl_f, new FunctionSpecialization(VOID), effort, NULL);
+        assert(cf->clfunc->versions.size());
+    }
+
+    if (cf->is_interpreted)
+        return astInterpretFunction(cf, 0, NULL, NULL, NULL, NULL, NULL, NULL);
+    else
+        return ((Box * (*)())cf->code)();
+}
+
+Box* runEval(const char* code, BoxedDict* locals, BoxedModule* module) {
+    // TODO error message if parse fails or if it isn't an expr
+    // TODO should have a cleaner interface that can parse the Expression directly
+    // TODO this memory leaks
+    AST_Module* parsedModule = parse_string(code);
+    assert(parsedModule->body[0]->type == AST_TYPE::Expr);
+    AST_Expression* parsedExpr = new AST_Expression(std::unique_ptr<InternedStringPool>(new InternedStringPool()));
+    parsedExpr->body = static_cast<AST_Expr*>(parsedModule->body[0])->value;
 
-        ScopingAnalysis* scoping = runScopingAnalysis(
+    return compileAndRunExpression(parsedExpr, module);
 }
-*/
 
 // If a function version keeps failing its speculations, kill it (remove it
 // from the list of valid function versions).  The next time we go to call

src/codegen/irgen/hooks.h

@@ -22,6 +22,8 @@ namespace pyston {
 struct CompiledFunction;
 class CLFunction;
 class OSRExit;
+class Box;
+class BoxedDict;
 
 CompiledFunction* compilePartialFuncInternal(OSRExit* exit);
 void* compilePartialFunc(OSRExit*);
@@ -34,6 +36,8 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm);
 
 // will we always want to generate unique function names? (ie will this function always be reasonable?)
 CompiledFunction* cfForMachineFunctionName(const std::string&);
+
+Box* runEval(const char* code, BoxedDict* locals, BoxedModule* module);
 }
 
 #endif

src/codegen/parser.cpp

@@ -935,7 +935,27 @@ static std::string getParserCommandLine(const char* fn) {
     return std::string("python -S ") + parse_ast_fn.str().str() + " " + fn;
 }
 
-AST_Module* parse(const char* fn) {
+AST_Module* parse_string(const char* code) {
+    int size = strlen(code);
+    char buf[] = "pystontmp_XXXXXX";
+    char* tmpdir = mkdtemp(buf);
+    assert(tmpdir);
+    std::string tmp = std::string(tmpdir) + "/in.py";
+    if (VERBOSITY() >= 1) {
+        printf("writing %d bytes to %s\n", size, tmp.c_str());
+    }
+
+    FILE* f = fopen(tmp.c_str(), "w");
+    fwrite(code, 1, size, f);
+    fclose(f);
+
+    AST_Module* m = parse_file(tmp.c_str());
+    removeDirectoryIfExists(tmpdir);
+
+    return m;
+}
+
+AST_Module* parse_file(const char* fn) {
     Timer _t("parsing");
 
     if (ENABLE_PYPA_PARSER) {
@@ -1009,7 +1029,7 @@ static ParseResult _reparse(const char* fn, const std::string& cache_fn) {
 // Parsing the file is somewhat expensive since we have to shell out to cpython;
 // it's not a huge deal right now, but this caching version can significantly cut down
 // on the startup time (40ms -> 10ms).
-AST_Module* caching_parse(const char* fn) {
+AST_Module* caching_parse_file(const char* fn) {
     Timer _t("parsing");
 
     if (ENABLE_PYPA_PARSER) {

src/codegen/parser.h

@@ -19,8 +19,10 @@ namespace pyston {
 
 class AST_Module;
 
-AST_Module* parse(const char* fn);
-AST_Module* caching_parse(const char* fn);
+AST_Module* parse_string(const char* code);
+
+AST_Module* parse_file(const char* fn);
+AST_Module* caching_parse_file(const char* fn);
 }
 
 #endif

src/core/ast.cpp

@@ -692,6 +692,14 @@ void AST_Module::accept(ASTVisitor* v) {
     visitVector(body, v);
 }
 
+void AST_Expression::accept(ASTVisitor* v) {
+    bool skip = v->visit_expression(this);
+    if (skip)
+        return;
+
+    body->accept(v);
+}
+
 void AST_Name::accept(ASTVisitor* v) {
     bool skip = v->visit_name(this);
 }
@@ -1507,6 +1515,12 @@ bool PrintVisitor::visit_module(AST_Module* node) {
     return true;
 }
 
+bool PrintVisitor::visit_expression(AST_Expression* node) {
+    node->body->accept(this);
+    printf("\n");
+    return true;
+}
+
 bool PrintVisitor::visit_name(AST_Name* node) {
     printf("%s", node->id.c_str());
     // printf("%s(%d)", node->id.c_str(), node->ctx_type);

src/core/ast.h

@@ -118,6 +118,7 @@ enum AST_TYPE {
     DictComp = 15,
     Set = 43,
     Ellipsis = 87,
+    Expression = 88,
 
     // Pseudo-nodes that are specific to this compiler:
     Branch = 200,
@@ -657,6 +658,21 @@ public:
     static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Module;
 };
 
+// (Alternative to AST_Module, used for, e.g., eval)
+class AST_Expression : public AST {
+public:
+    std::unique_ptr<InternedStringPool> interned_strings;
+
+    AST_expr* body;
+
+    virtual void accept(ASTVisitor* v);
+
+    AST_Expression(std::unique_ptr<InternedStringPool> interned_strings)
+        : AST(AST_TYPE::Expression), interned_strings(std::move(interned_strings)) {}
+
+    static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Expression;
+};
+
 class AST_Name : public AST_expr {
 public:
     AST_TYPE::AST_TYPE ctx_type;
@@ -1044,6 +1060,7 @@ public:
     virtual bool visit_excepthandler(AST_ExceptHandler* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_exec(AST_Exec* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_expr(AST_Expr* node) { RELEASE_ASSERT(0, ""); }
+    virtual bool visit_expression(AST_Expression* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_extslice(AST_ExtSlice* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_for(AST_For* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_functiondef(AST_FunctionDef* node) { RELEASE_ASSERT(0, ""); }
@@ -1112,6 +1129,7 @@ public:
     virtual bool visit_excepthandler(AST_ExceptHandler* node) { return false; }
     virtual bool visit_exec(AST_Exec* node) { return false; }
     virtual bool visit_expr(AST_Expr* node) { return false; }
+    virtual bool visit_expr(AST_Expression* node) { return false; }
     virtual bool visit_extslice(AST_ExtSlice* node) { return false; }
     virtual bool visit_for(AST_For* node) { return false; }
     virtual bool visit_functiondef(AST_FunctionDef* node) { return false; }
@@ -1254,6 +1272,7 @@ public:
     virtual bool visit_excepthandler(AST_ExceptHandler* node);
     virtual bool visit_exec(AST_Exec* node);
     virtual bool visit_expr(AST_Expr* node);
+    virtual bool visit_expression(AST_Expression* node);
     virtual bool visit_extslice(AST_ExtSlice* node);
     virtual bool visit_for(AST_For* node);
     virtual bool visit_functiondef(AST_FunctionDef* node);

src/core/cfg.cpp

@@ -1609,7 +1609,7 @@ public:
     }
 
     bool visit_return(AST_Return* node) override {
-        if (root_type != AST_TYPE::FunctionDef && root_type != AST_TYPE::Lambda) {
+        if (root_type != AST_TYPE::FunctionDef && root_type != AST_TYPE::Lambda && root_type != AST_TYPE::Expression) {
             raiseExcHelper(SyntaxError, "'return' outside function");
         }
 

src/jit.cpp

@@ -179,29 +179,14 @@ int main(int argc, char** argv) {
         while (repl) {
             char* line = readline(">> ");
 
+            AST_Module* m = parse_string(line);
+
             if (!line) {
                 repl = false;
             } else {
                 add_history(line);
-                int size = strlen(line);
 
                 Timer _t("repl");
-
-                char buf[] = "pystontmp_XXXXXX";
-                char* tmpdir = mkdtemp(buf);
-                assert(tmpdir);
-                std::string tmp = std::string(tmpdir) + "/in.py";
-                if (VERBOSITY() >= 1) {
-                    printf("writing %d bytes to %s\n", size, tmp.c_str());
-                }
-
-                FILE* f = fopen(tmp.c_str(), "w");
-                fwrite(line, 1, size, f);
-                fclose(f);
-
-                AST_Module* m = parse(tmp.c_str());
-                removeDirectoryIfExists(tmpdir);
-
                 if (m->body.size() > 0 && m->body[0]->type == AST_TYPE::Expr) {
                     AST_Expr* e = ast_cast<AST_Expr>(m->body[0]);
                     AST_Call* c = new AST_Call();

src/runtime/builtin_modules/builtins.cpp

@@ -576,6 +576,16 @@ Box* zip2(Box* container1, Box* container2) {
     return rtn;
 }
 
+Box* eval(Box* code) {
+    // TODO implement full functionality (args and stuff)
+    RELEASE_ASSERT(code->cls == str_cls, "eval not implemented for non-strings");
+
+    BoxedDict* locals = getLocals(true /* only_user_visible */);
+    BoxedModule* module = getCurrentModule();
+
+    return runEval(static_cast<BoxedString*>(code)->s.c_str(), locals, module);
+}
+
 BoxedClass* notimplemented_cls;
 BoxedModule* builtins_module;
 
@@ -776,7 +786,7 @@ Box* execfile(Box* _fn) {
         raiseExcHelper(IOError, "No such file or directory: '%s'", fn->s.c_str());
 
     // Run directly inside the current module:
-    AST_Module* ast = caching_parse(fn->s.c_str());
+    AST_Module* ast = caching_parse_file(fn->s.c_str());
     compileAndRunModule(ast, getCurrentModule());
 
     return None;
@@ -1188,7 +1198,8 @@ void setupBuiltins() {
     builtins_module->giveAttr("property", property_cls);
     builtins_module->giveAttr("staticmethod", staticmethod_cls);
     builtins_module->giveAttr("classmethod", classmethod_cls);
-
+    builtins_module->giveAttr(
+        "eval", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)eval, UNKNOWN, 1, 0, false, false)));
 
     PyExc_RecursionErrorInst = new (RuntimeError) BoxedException();
     gc::registerPermanentRoot(PyExc_RecursionErrorInst);

src/runtime/import.cpp

@@ -27,7 +27,7 @@ namespace pyston {
 BoxedModule* createAndRunModule(const std::string& name, const std::string& fn) {
     BoxedModule* module = createModule(name, fn);
 
-    AST_Module* ast = caching_parse(fn.c_str());
+    AST_Module* ast = caching_parse_file(fn.c_str());
     compileAndRunModule(ast, module);
     return module;
 }
@@ -42,7 +42,7 @@ static BoxedModule* createAndRunModule(const std::string& name, const std::strin
 
     module->setattr("__path__", path_list, NULL);
 
-    AST_Module* ast = caching_parse(fn.c_str());
+    AST_Module* ast = caching_parse_file(fn.c_str());
     compileAndRunModule(ast, module);
     return module;
 }

test/tests/eval_test.py

@@ -36,10 +36,25 @@ def func():
     except NameError:
         print 'g not found'
 
+    eval("[g2 for g2 in range(5)]")
+    try:
+        print g2
+    except NameError:
+        print 'g2 not found'
+    g2 = 5
+
     h = 2
     eval("[h for h in range(5)]")
     print h
 
+    h2 = 2
+    print eval("h2 + sum([h2 for h2 in range(5)])")
+    print 'h2', h2
+
+    h3 = 2
+    print eval("sum([h3 for h3 in range(5)]) + h3")
+    print 'h3', h3
+
     eval("[i for i in range(5)]")
 
     j = 24
@@ -65,6 +80,27 @@ print eval("(lambda k : [n for n in range(5)])(3)")
 print n
 
 print eval("eval('3 + 2342')")
+o = 300
+print 'eval eval o', eval("eval('o')")
+
+print eval("[(lambda p : p + o)(5) for o in range(5)]")
+
+shadow1 = 1000
+shadow2 = 1000
+shadow3 = 1000
+def func2():
+    shadow1 = 2000
+    print 'shadow1', eval("shadow1")
+
+    shadow2 = 2000
+    eval("[shadow2 for shadow2 in range(5)]")
+    print 'shadow2', shadow2
+
+    print 'shadow3', eval("shadow3 + sum([2 for shadow3 in range(5)]) + shadow3")
+func2()
+print 'shadow1', shadow2
+print 'shadow2', shadow2
+print 'shadow3', shadow3
 
 x = 2
 def wrap():