Some `__future__` import stuff

- Move `FutureFlags` from `BoxedModule` to `SourceInfo`.
- Handle `__future__` imports in exec statements.
- Handle the `division` import in cfg gen rather than later.

src/capi/abstract.cpp

@@ -18,6 +18,7 @@
 #include "Python.h"
 
 #include "capi/types.h"
+#include "core/ast.h"
 #include "core/threading.h"
 #include "core/types.h"
 #include "runtime/classobj.h"

src/codegen/ast_interpreter.cpp

@@ -373,9 +373,6 @@ Value ASTInterpreter::execute(ASTInterpreter& interpreter, CFGBlock* start_block
 }
 
 Value ASTInterpreter::doBinOp(Box* left, Box* right, int op, BinExpType exp_type) {
-    if (op == AST_TYPE::Div && (source_info->parent_module->future_flags & FF_DIVISION)) {
-        op = AST_TYPE::TrueDiv;
-    }
     switch (exp_type) {
         case BinExpType::AugBinOp:
             return augbinop(left, right, op);
@@ -1017,7 +1014,7 @@ Value ASTInterpreter::visit_exec(AST_Exec* node) {
     Box* globals = node->globals == NULL ? NULL : visit_expr(node->globals).o;
     Box* locals = node->locals == NULL ? NULL : visit_expr(node->locals).o;
 
-    exec(code, globals, locals);
+    exec(code, globals, locals, this->source_info->future_flags);
 
     return Value();
 }

src/codegen/codegen.cpp

@@ -34,8 +34,15 @@ namespace pyston {
 
 DS_DEFINE_RWLOCK(codegen_rwlock);
 
-SourceInfo::SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, AST* ast, std::vector<AST_stmt*> body, std::string fn)
-    : parent_module(m), scoping(scoping), ast(ast), cfg(NULL), fn(std::move(fn)), body(std::move(body)) {
+SourceInfo::SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, FutureFlags future_flags, AST* ast,
+                       std::vector<AST_stmt*> body, std::string fn)
+    : parent_module(m),
+      scoping(scoping),
+      future_flags(future_flags),
+      ast(ast),
+      cfg(NULL),
+      fn(std::move(fn)),
+      body(std::move(body)) {
     assert(this->fn.size());
 
     switch (ast->type) {

src/codegen/irgen/future.cpp

@@ -12,7 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "future.h"
+#include "codegen/irgen/future.h"
+
+#include <map>
+
+#include "core/ast.h"
 
 namespace pyston {
 
@@ -59,7 +63,7 @@ inline bool is_stmt_string(AST_stmt* stmt) {
     return stmt->type == AST_TYPE::Expr && static_cast<AST_Expr*>(stmt)->value->type == AST_TYPE::Str;
 }
 
-FutureFlags getFutureFlags(AST_Module* m, const char* file) {
+FutureFlags getFutureFlags(std::vector<AST_stmt*> const& body, const char* file) {
     FutureFlags ff = 0;
 
     // Set the defaults for the future flags depending on what version we are
@@ -73,8 +77,8 @@ FutureFlags getFutureFlags(AST_Module* m, const char* file) {
     // occur at the beginning of the file.
     bool future_import_allowed = true;
     BadFutureImportVisitor import_visitor(file);
-    for (int i = 0; i < m->body.size(); i++) {
-        AST_stmt* stmt = m->body[i];
+    for (int i = 0; i < body.size(); i++) {
+        AST_stmt* stmt = body[i];
 
         if (stmt->type == AST_TYPE::ImportFrom && static_cast<AST_ImportFrom*>(stmt)->module.str() == "__future__") {
             if (future_import_allowed) {

src/codegen/irgen/future.h

@@ -15,10 +15,8 @@
 #ifndef PYSTON_CODEGEN_IRGEN_FUTURE_H
 #define PYSTON_CODEGEN_IRGEN_FUTURE_H
 
-#include <map>
+#include <vector>
 
-#include "core/ast.h"
-#include "core/options.h"
 #include "core/types.h"
 
 namespace pyston {
@@ -31,13 +29,11 @@ namespace pyston {
 #define FF_NESTED_SCOPES 0x20
 #define FF_WITH_STATEMENT 0x40
 
-typedef int FutureFlags;
-
 // Loop through import statements to find __future__ imports throwing errors for
 // bad __future__ imports. Returns the futures that are turned on. This is used
 // for irgeneration; the parser still has to handle some futures on its own,
 // when they are relevant for the parser.
-FutureFlags getFutureFlags(AST_Module* m, const char* file);
+FutureFlags getFutureFlags(std::vector<AST_stmt*> const& body, const char* file);
 }
 
 #endif

src/codegen/irgen/hooks.cpp

@@ -309,11 +309,11 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm) {
 
         const char* fn = PyModule_GetFilename(bm);
         RELEASE_ASSERT(fn, "");
-        bm->future_flags = getFutureFlags(m, fn);
 
+        FutureFlags future_flags = getFutureFlags(m->body, fn);
         ScopingAnalysis* scoping = new ScopingAnalysis(m, true);
 
-        std::unique_ptr<SourceInfo> si(new SourceInfo(bm, scoping, m, m->body, fn));
+        std::unique_ptr<SourceInfo> si(new SourceInfo(bm, scoping, future_flags, m, m->body, fn));
         bm->setattr("__doc__", si->getDocString(), NULL);
         if (!bm->hasattr("__builtins__"))
             bm->giveAttr("__builtins__", PyModule_GetDict(builtins_module));
@@ -360,14 +360,22 @@ Box* evalOrExec(CLFunction* cl, Box* globals, Box* boxedLocals) {
     return astInterpretFunctionEval(cf, globals, boxedLocals);
 }
 
-CLFunction* compileForEvalOrExec(AST* source, std::vector<AST_stmt*> body, std::string fn) {
+CLFunction* compileForEvalOrExec(AST* source, std::vector<AST_stmt*> body, std::string fn,
+                                 FutureFlags caller_future_flags) {
     LOCK_REGION(codegen_rwlock.asWrite());
 
     Timer _t("for evalOrExec()");
 
     ScopingAnalysis* scoping = new ScopingAnalysis(source, false);
 
-    std::unique_ptr<SourceInfo> si(new SourceInfo(getCurrentModule(), scoping, source, std::move(body), std::move(fn)));
+    // `my_future_flags` are the future flags enabled in the exec's code.
+    // `caller_future_flags` are the future flags of the source that the exec statement is in.
+    // We need to enable features that are enabled in either.
+    FutureFlags my_future_flags = getFutureFlags(body, fn.c_str());
+    FutureFlags future_flags = caller_future_flags | my_future_flags;
+
+    std::unique_ptr<SourceInfo> si(
+        new SourceInfo(getCurrentModule(), scoping, future_flags, source, std::move(body), std::move(fn)));
     CLFunction* cl_f = new CLFunction(0, 0, false, false, std::move(si));
 
     return cl_f;
@@ -405,8 +413,8 @@ static AST_Suite* parseExec(llvm::StringRef source, bool interactive = false) {
     return parsedSuite;
 }
 
-static CLFunction* compileExec(AST_Suite* parsedSuite, llvm::StringRef fn) {
-    return compileForEvalOrExec(parsedSuite, parsedSuite->body, fn);
+static CLFunction* compileExec(AST_Suite* parsedSuite, llvm::StringRef fn, FutureFlags caller_future_flags) {
+    return compileForEvalOrExec(parsedSuite, parsedSuite->body, fn, caller_future_flags);
 }
 
 static AST_Expression* parseEval(llvm::StringRef source) {
@@ -439,7 +447,12 @@ static CLFunction* compileEval(AST_Expression* parsedExpr, llvm::StringRef fn) {
     stmt->value = parsedExpr->body;
     std::vector<AST_stmt*> body = { stmt };
 
-    return compileForEvalOrExec(parsedExpr, std::move(body), fn);
+    CompiledFunction* caller_cf = getTopCompiledFunction();
+    assert(caller_cf != NULL);
+    assert(caller_cf->clfunc->source != NULL);
+    FutureFlags caller_future_flags = caller_cf->clfunc->source->future_flags;
+
+    return compileForEvalOrExec(parsedExpr, std::move(body), fn, caller_future_flags);
 }
 
 Box* compile(Box* source, Box* fn, Box* type, Box** _args) {
@@ -485,6 +498,11 @@ Box* compile(Box* source, Box* fn, Box* type, Box** _args) {
 
     AST* parsed;
 
+    CompiledFunction* caller_cf = getTopCompiledFunction();
+    assert(caller_cf != NULL);
+    assert(caller_cf->clfunc->source != NULL);
+    FutureFlags caller_future_flags = caller_cf->clfunc->source->future_flags;
+
     if (PyAST_Check(source)) {
         parsed = unboxAst(source);
     } else {
@@ -510,7 +528,7 @@ Box* compile(Box* source, Box* fn, Box* type, Box** _args) {
         // TODO: CPython parses execs as Modules
         if (parsed->type != AST_TYPE::Suite)
             raiseExcHelper(TypeError, "expected Suite node, got %s", boxAst(parsed)->cls->tp_name);
-        cl = compileExec(static_cast<AST_Suite*>(parsed), filename_str);
+        cl = compileExec(static_cast<AST_Suite*>(parsed), filename_str, caller_future_flags);
     } else if (type_str == "eval") {
         if (parsed->type != AST_TYPE::Expression)
             raiseExcHelper(TypeError, "expected Expression node, got %s", boxAst(parsed)->cls->tp_name);
@@ -565,10 +583,11 @@ Box* eval(Box* boxedCode, Box* globals, Box* locals) {
     } else {
         abort();
     }
+
     return evalOrExec(cl, globals, locals);
 }
 
-Box* exec(Box* boxedCode, Box* globals, Box* locals) {
+Box* exec(Box* boxedCode, Box* globals, Box* locals, FutureFlags caller_future_flags) {
     if (isSubclass(boxedCode->cls, tuple_cls)) {
         RELEASE_ASSERT(!globals, "");
         RELEASE_ASSERT(!locals, "");
@@ -626,7 +645,7 @@ Box* exec(Box* boxedCode, Box* globals, Box* locals) {
     CLFunction* cl;
     if (boxedCode->cls == str_cls) {
         AST_Suite* parsed = parseExec(static_cast<BoxedString*>(boxedCode)->s());
-        cl = compileExec(parsed, "<string>");
+        cl = compileExec(parsed, "<string>", caller_future_flags);
     } else if (boxedCode->cls == code_cls) {
         cl = clfunctionFromCode(boxedCode);
     } else {
@@ -641,8 +660,10 @@ extern "C" PyObject* PyRun_StringFlags(const char* str, int start, PyObject* glo
                                        PyCompilerFlags* flags) noexcept {
 
     try {
+        // TODO pass future_flags (the information is in PyCompilerFlags but we need to
+        // unify the format...)
         if (start == Py_file_input)
-            return exec(boxString(str), globals, locals);
+            return exec(boxString(str), globals, locals, 0);
         else if (start == Py_eval_input)
             return eval(boxString(str), globals, locals);
     } catch (ExcInfo e) {

src/codegen/irgen/hooks.h

@@ -17,6 +17,8 @@
 
 #include <string>
 
+#include "core/types.h"
+
 namespace pyston {
 
 struct CompiledFunction;
@@ -37,7 +39,7 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm);
 // will we always want to generate unique function names? (ie will this function always be reasonable?)
 CompiledFunction* cfForMachineFunctionName(const std::string&);
 
-extern "C" Box* exec(Box* boxedCode, Box* globals, Box* locals);
+extern "C" Box* exec(Box* boxedCode, Box* globals, Box* locals, FutureFlags caller_future_flags);
 extern "C" Box* eval(Box* boxedCode, Box* globals, Box* locals);
 extern "C" Box* compile(Box* source, Box* filename, Box* mode, Box** _args /* flags, dont_inherit */);
 }

src/codegen/irgen/irgenerator.cpp

@@ -747,10 +747,6 @@ private:
         assert(left);
         assert(right);
 
-        if (type == AST_TYPE::Div && (irstate->getSourceInfo()->parent_module->future_flags & FF_DIVISION)) {
-            type = AST_TYPE::TrueDiv;
-        }
-
         return left->binexp(emitter, getOpInfoForNode(node, unw_info), right, type, exp_type);
     }
 
@@ -1842,7 +1838,9 @@ private:
             vlocals = getNullPtr(g.llvm_value_type_ptr);
         }
 
-        emitter.createCall3(unw_info, g.funcs.exec, vbody, vglobals, vlocals);
+        static_assert(sizeof(FutureFlags) == 4, "");
+        emitter.createCall(unw_info, g.funcs.exec,
+                           { vbody, vglobals, vlocals, getConstantInt(irstate->getSourceInfo()->future_flags, g.i32) });
     }
 
     void doPrint(AST_Print* node, UnwindInfo unw_info) {
@@ -2655,7 +2653,8 @@ CLFunction* wrapFunction(AST* node, AST_arguments* args, const std::vector<AST_s
 
     CLFunction*& cl = made[node];
     if (cl == NULL) {
-        std::unique_ptr<SourceInfo> si(new SourceInfo(source->parent_module, source->scoping, node, body, source->fn));
+        std::unique_ptr<SourceInfo> si(
+            new SourceInfo(source->parent_module, source->scoping, source->future_flags, node, body, source->fn));
         if (args)
             cl = new CLFunction(args->args.size(), args->defaults.size(), args->vararg.str().size(),
                                 args->kwarg.str().size(), std::move(si));

src/codegen/unwinding.h

@@ -98,6 +98,8 @@ struct FrameStackState {
 
 // Returns all the stack locals, including hidden ones.
 FrameStackState getFrameStackState();
+
+CompiledFunction* getTopCompiledFunction();
 }
 
 #endif

src/core/cfg.cpp

@@ -637,7 +637,7 @@ private:
         AST_BinOp* rtn = new AST_BinOp();
         rtn->lineno = node->lineno;
         rtn->col_offset = node->col_offset;
-        rtn->op_type = node->op_type;
+        rtn->op_type = remapBinOpType(node->op_type);
         rtn->left = remapExpr(node->left);
         rtn->right = remapExpr(node->right);
         return rtn;
@@ -1716,7 +1716,7 @@ public:
         }
 
         AST_AugBinOp* binop = new AST_AugBinOp();
-        binop->op_type = node->op_type;
+        binop->op_type = remapBinOpType(node->op_type);
         binop->left = remapped_lhs;
         binop->right = remapExpr(node->value);
         binop->col_offset = node->col_offset;
@@ -1728,6 +1728,14 @@ public:
         return true;
     }
 
+    AST_TYPE::AST_TYPE remapBinOpType(AST_TYPE::AST_TYPE op_type) {
+        if (op_type == AST_TYPE::Div && (future_flags & (FF_DIVISION))) {
+            return AST_TYPE::TrueDiv;
+        } else {
+            return op_type;
+        }
+    }
+
     bool visit_delete(AST_Delete* node) override {
         for (auto t : node->targets) {
             AST_Delete* astdel = new AST_Delete();
@@ -2466,7 +2474,7 @@ CFG* computeCFG(SourceInfo* source, std::vector<AST_stmt*> body) {
 
     ScopingAnalysis* scoping_analysis = source->scoping;
 
-    CFGVisitor visitor(source, source->ast->type, source->parent_module->future_flags, scoping_analysis, rtn);
+    CFGVisitor visitor(source, source->ast->type, source->future_flags, scoping_analysis, rtn);
 
     bool skip_first = false;
 

src/core/types.h

@@ -242,6 +242,8 @@ private:
     ParamNames() : takes_param_names(false) {}
 };
 
+typedef int FutureFlags;
+
 class BoxedModule;
 class ScopeInfo;
 class InternedStringPool;
@@ -249,6 +251,7 @@ class SourceInfo {
 public:
     BoxedModule* parent_module;
     ScopingAnalysis* scoping;
+    FutureFlags future_flags;
     AST* ast;
     CFG* cfg;
     bool is_generator;
@@ -267,7 +270,8 @@ public:
 
     Box* getDocString();
 
-    SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, AST* ast, std::vector<AST_stmt*> body, std::string fn);
+    SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, FutureFlags future_flags, AST* ast,
+               std::vector<AST_stmt*> body, std::string fn);
 };
 
 typedef std::vector<CompiledFunction*> FunctionList;

src/runtime/builtin_modules/ast.cpp

@@ -21,6 +21,7 @@
 #include "llvm/Support/Path.h"
 
 #include "codegen/unwinding.h"
+#include "core/ast.h"
 #include "core/types.h"
 #include "gc/collector.h"
 #include "runtime/file.h"

src/runtime/code.cpp

@@ -18,6 +18,7 @@
 
 #include "code.h"
 
+#include "core/ast.h"
 #include "gc/collector.h"
 #include "runtime/objmodel.h"
 #include "runtime/set.h"

src/runtime/cxx_unwind.cpp

@@ -22,10 +22,11 @@
 
 #include "codegen/ast_interpreter.h" // interpreter_instr_addr
 #include "codegen/unwinding.h"       // getCFForAddress
-#include "core/stats.h"              // StatCounter
-#include "core/types.h"              // for ExcInfo
-#include "core/util.h"               // Timer
-#include "runtime/generator.h"       // generatorEntry
+#include "core/ast.h"
+#include "core/stats.h"        // StatCounter
+#include "core/types.h"        // for ExcInfo
+#include "core/util.h"         // Timer
+#include "runtime/generator.h" // generatorEntry
 
 #define UNW_LOCAL_ONLY
 #include <libunwind.h>

src/runtime/dict.cpp

@@ -15,6 +15,7 @@
 #include "runtime/dict.h"
 
 #include "capi/types.h"
+#include "core/ast.h"
 #include "core/common.h"
 #include "core/stats.h"
 #include "core/types.h"

src/runtime/frame.cpp

@@ -16,6 +16,7 @@
 #include "pythread.h"
 
 #include "codegen/unwinding.h"
+#include "core/ast.h"
 #include "runtime/types.h"
 
 namespace pyston {

src/runtime/import.cpp

@@ -22,6 +22,7 @@
 #include "codegen/irgen/hooks.h"
 #include "codegen/parser.h"
 #include "codegen/unwinding.h"
+#include "core/ast.h"
 #include "runtime/capi.h"
 #include "runtime/objmodel.h"
 

src/runtime/int.cpp

@@ -18,6 +18,7 @@
 #include <sstream>
 
 #include "capi/typeobject.h"
+#include "core/ast.h"
 #include "core/common.h"
 #include "core/options.h"
 #include "core/stats.h"

src/runtime/stacktrace.cpp

@@ -17,6 +17,7 @@
 #include <dlfcn.h>
 
 #include "codegen/unwinding.h"
+#include "core/ast.h"
 #include "core/options.h"
 #include "gc/collector.h"
 #include "runtime/objmodel.h"

src/runtime/types.cpp

@@ -22,10 +22,12 @@
 
 #include "llvm/Support/raw_ostream.h"
 
+#include "analysis/scoping_analysis.h"
 #include "capi/typeobject.h"
 #include "capi/types.h"
 #include "codegen/ast_interpreter.h"
 #include "codegen/unwinding.h"
+#include "core/ast.h"
 #include "core/options.h"
 #include "core/stats.h"
 #include "core/types.h"

test/tests/exec_future_import.py

+print 1 / 2
+
+exec """
+print 1 / 2
+"""
+
+exec """
+from __future__ import division
+print 1 / 2
+"""
+
+exec """
+from __future__ import division
+exec "print 1 / 2"
+"""
+
+print 1 / 2
+
+print eval("1 / 2")
+
+exec """
+print eval("1 / 2")
+"""
+
+exec """
+from __future__ import division
+print eval("1 / 2")
+"""

test/tests/exec_future_import2.py

+from __future__ import division
+
+print 1 / 2
+
+exec """print 1 / 2"""
+
+exec """
+from __future__ import division
+print 1 / 2
+"""
+
+print eval("1 / 2")
+
+exec """
+print eval("1 / 2")
+"""
+
+exec """
+from __future__ import division
+print eval("1 / 2")
+"""

test/unittests/analysis.cpp

@@ -7,6 +7,7 @@
 
 #include "analysis/function_analysis.h"
 #include "analysis/scoping_analysis.h"
+#include "codegen/irgen/future.h"
 #include "codegen/osrentry.h"
 #include "codegen/parser.h"
 #include "core/ast.h"
@@ -36,7 +37,9 @@ TEST_F(AnalysisTest, augassign) {
     ASSERT_FALSE(scope_info->getScopeTypeOfName(module->interned_strings->get("a")) == ScopeInfo::VarScopeType::GLOBAL);
     ASSERT_FALSE(scope_info->getScopeTypeOfName(module->interned_strings->get("b")) == ScopeInfo::VarScopeType::GLOBAL);
 
-    SourceInfo* si = new SourceInfo(createModule("augassign", fn.c_str()), scoping, func, func->body, fn);
+    FutureFlags future_flags = getFutureFlags(module->body, fn.c_str());
+
+    SourceInfo* si = new SourceInfo(createModule("augassign", fn.c_str()), scoping, future_flags, func, func->body, fn);
 
     CFG* cfg = computeCFG(si, func->body);
     std::unique_ptr<LivenessAnalysis> liveness = computeLivenessInfo(cfg);
@@ -62,9 +65,11 @@ void doOsrTest(bool is_osr, bool i_maybe_undefined) {
     assert(module->body[0]->type == AST_TYPE::FunctionDef);
     AST_FunctionDef* func = static_cast<AST_FunctionDef*>(module->body[0]);
 
+    FutureFlags future_flags = getFutureFlags(module->body, fn.c_str());
+
     ScopeInfo* scope_info = scoping->getScopeInfoForNode(func);
     SourceInfo* si = new SourceInfo(createModule("osr" + std::to_string((is_osr << 1) + i_maybe_undefined), fn.c_str()),
-            scoping, func, func->body, fn);
+            scoping, future_flags, func, func->body, fn);
 
     CFG* cfg = computeCFG(si, func->body);
     std::unique_ptr<LivenessAnalysis> liveness = computeLivenessInfo(cfg);