Merge pull request #910 from kmod/decorator

Support the "decorator" library

.gitmodules

@@ -34,3 +34,6 @@
 [submodule "test/lib/numpy"]
 	path = test/lib/numpy
 	url = https://github.com/numpy/numpy
+[submodule "test/lib/decorator"]
+	path = test/lib/decorator
+	url = https://github.com/micheles/decorator

libpypa @ 728c4bc8

-Subproject commit 1f62a285c57da50c00249a704deac00dae3c247b
+Subproject commit 728c4bc8c99d09da224ff3a2be82c98a1b026e82

src/codegen/irgen/hooks.cpp

@@ -370,12 +370,12 @@ static CLFunction* compileForEvalOrExec(AST* source, std::vector<AST_stmt*> body
     return cl_f;
 }
 
-static AST_Module* parseExec(llvm::StringRef source, bool interactive = false) {
+static AST_Module* parseExec(llvm::StringRef source, FutureFlags future_flags, bool interactive = false) {
     // TODO error message if parse fails or if it isn't an expr
     // TODO should have a cleaner interface that can parse the Expression directly
     // TODO this memory leaks
     const char* code = source.data();
-    AST_Module* parsedModule = parse_string(code);
+    AST_Module* parsedModule = parse_string(code, future_flags);
 
     if (interactive) {
         for (int i = 0; i < parsedModule->body.size(); ++i) {
@@ -397,7 +397,7 @@ static CLFunction* compileExec(AST_Module* parsedModule, BoxedString* fn, PyComp
     return compileForEvalOrExec(parsedModule, parsedModule->body, fn, flags);
 }
 
-static AST_Expression* parseEval(llvm::StringRef source) {
+static AST_Expression* parseEval(llvm::StringRef source, FutureFlags future_flags) {
     const char* code = source.data();
 
     // TODO error message if parse fails or if it isn't an expr
@@ -409,7 +409,7 @@ static AST_Expression* parseEval(llvm::StringRef source) {
     while (*code == ' ' || *code == '\t' || *code == '\n' || *code == '\r')
         code++;
 
-    AST_Module* parsedModule = parse_string(code);
+    AST_Module* parsedModule = parse_string(code, future_flags);
     if (parsedModule->body.size() == 0)
         raiseSyntaxError("unexpected EOF while parsing", 0, 0, "<string>", "");
 
@@ -496,11 +496,11 @@ Box* compile(Box* source, Box* fn, Box* type, Box** _args) {
         llvm::StringRef source_str = static_cast<BoxedString*>(source)->s();
 
         if (type_str->s() == "exec") {
-            parsed = parseExec(source_str);
+            parsed = parseExec(source_str, future_flags);
         } else if (type_str->s() == "eval") {
-            parsed = parseEval(source_str);
+            parsed = parseEval(source_str, future_flags);
         } else if (type_str->s() == "single") {
-            parsed = parseExec(source_str, true);
+            parsed = parseExec(source_str, future_flags, true);
         } else {
             raiseExcHelper(ValueError, "compile() arg 3 must be 'exec', 'eval' or 'single'");
         }
@@ -578,7 +578,11 @@ static Box* evalMain(Box* boxedCode, Box* globals, Box* locals, PyCompilerFlags*
 
     CLFunction* cl;
     if (boxedCode->cls == str_cls) {
-        AST_Expression* parsed = parseEval(static_cast<BoxedString*>(boxedCode)->s());
+        CLFunction* caller_cl = getTopPythonFunction();
+        assert(caller_cl != NULL);
+        assert(caller_cl->source != NULL);
+
+        AST_Expression* parsed = parseEval(static_cast<BoxedString*>(boxedCode)->s(), caller_cl->source->future_flags);
         static BoxedString* string_string = internStringImmortal("<string>");
         cl = compileEval(parsed, string_string, flags);
     } else if (boxedCode->cls == code_cls) {
@@ -627,7 +631,7 @@ Box* execfile(Box* _fn, Box* globals, Box* locals) {
     if (!exists)
         raiseExcHelper(IOError, "No such file or directory: '%s'", fn->s().data());
 
-    AST_Module* parsed = caching_parse_file(fn->s().data());
+    AST_Module* parsed = caching_parse_file(fn->s().data(), /* future_flags = */ 0);
     assert(parsed);
 
     CLFunction* caller_cl = getTopPythonFunction();
@@ -667,7 +671,11 @@ Box* execMain(Box* boxedCode, Box* globals, Box* locals, PyCompilerFlags* flags)
 
     CLFunction* cl;
     if (boxedCode->cls == str_cls) {
-        auto parsed = parseExec(static_cast<BoxedString*>(boxedCode)->s());
+        CLFunction* caller_cl = getTopPythonFunction();
+        assert(caller_cl != NULL);
+        assert(caller_cl->source != NULL);
+
+        auto parsed = parseExec(static_cast<BoxedString*>(boxedCode)->s(), caller_cl->source->future_flags);
         static BoxedString* string_string = internStringImmortal("<string>");
         cl = compileExec(parsed, string_string, flags);
     } else if (boxedCode->cls == code_cls) {

src/codegen/parser.cpp

@@ -1012,13 +1012,17 @@ static std::string getParserCommandLine(const char* fn) {
     return std::string("/usr/bin/python -S ") + parse_ast_fn.str().str() + " " + fn;
 }
 
-AST_Module* parse_string(const char* code) {
-    if (ENABLE_PYPA_PARSER) {
-        AST_Module* rtn = pypa_parse_string(code);
+AST_Module* parse_string(const char* code, FutureFlags inherited_flags) {
+    inherited_flags &= ~(CO_NESTED | CO_FUTURE_DIVISION);
+
+    if (ENABLE_PYPA_PARSER || inherited_flags) {
+        AST_Module* rtn = pypa_parse_string(code, inherited_flags);
         RELEASE_ASSERT(rtn, "unknown parse error (possibly: '%s'?)", strerror(errno));
         return rtn;
     }
 
+    ASSERT(!inherited_flags, "the old cpython parser doesn't support specifying initial future flags");
+
     int size = strlen(code);
     char buf[] = "pystontmp_XXXXXX";
     char* tmpdir = mkdtemp(buf);
@@ -1033,17 +1037,17 @@ AST_Module* parse_string(const char* code) {
     fputc('\n', f);
     fclose(f);
 
-    AST_Module* m = parse_file(tmp.c_str());
+    AST_Module* m = parse_file(tmp.c_str(), inherited_flags);
     removeDirectoryIfExists(tmpdir);
 
     return m;
 }
 
-AST_Module* parse_file(const char* fn) {
+AST_Module* parse_file(const char* fn, FutureFlags inherited_flags) {
     Timer _t("parsing");
 
     if (ENABLE_PYPA_PARSER) {
-        AST_Module* rtn = pypa_parse(fn);
+        AST_Module* rtn = pypa_parse(fn, inherited_flags);
         RELEASE_ASSERT(rtn, "unknown parse error (possibly: '%s'?)", strerror(errno));
         return rtn;
     }
@@ -1070,9 +1074,9 @@ AST_Module* parse_file(const char* fn) {
 
 const char* getMagic() {
     if (ENABLE_PYPA_PARSER)
-        return "a\ncM";
+        return "a\ncN";
     else
-        return "a\ncm";
+        return "a\ncn";
 }
 
 #define MAGIC_STRING_LENGTH 4
@@ -1080,7 +1084,9 @@ const char* getMagic() {
 #define CHECKSUM_LENGTH 1
 
 // Does at least one of: returns a valid file_data vector, or fills in 'module'
-static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, AST_Module*& module) {
+static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, AST_Module*& module,
+                                  FutureFlags inherited_flags) {
+    inherited_flags &= ~(CO_NESTED | CO_FUTURE_DIVISION);
     FILE* cache_fp = fopen(cache_fn.c_str(), "w");
 
     if (DEBUG_PARSING) {
@@ -1110,8 +1116,8 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A
     file_data.insert(file_data.end(), (char*)&checksum, (char*)&checksum + CHECKSUM_LENGTH);
     checksum = 0;
 
-    if (ENABLE_PYPA_PARSER) {
-        module = pypa_parse(fn);
+    if (ENABLE_PYPA_PARSER || inherited_flags) {
+        module = pypa_parse(fn, inherited_flags);
         RELEASE_ASSERT(module, "unknown parse error");
 
         if (!cache_fp)
@@ -1121,6 +1127,7 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A
         checksum = p.second;
         bytes_written += p.first;
     } else {
+        ASSERT(!inherited_flags, "the old cpython parser doesn't support specifying initial future flags");
         FILE* parser = popen(getParserCommandLine(fn).c_str(), "r");
         char buf[80];
         while (true) {
@@ -1157,7 +1164,7 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A
 // Parsing the file is somewhat expensive since we have to shell out to cpython;
 // it's not a huge deal right now, but this caching version can significantly cut down
 // on the startup time (40ms -> 10ms).
-AST_Module* caching_parse_file(const char* fn) {
+AST_Module* caching_parse_file(const char* fn, FutureFlags inherited_flags) {
     std::ostringstream oss;
     if (DEBUG_PARSING) {
         oss << "caching_parse_file() on " << fn << '\n';
@@ -1192,7 +1199,7 @@ AST_Module* caching_parse_file(const char* fn) {
                     if (ferror(cache_fp)) {
                         oss << "encountered io error reading from the file\n";
                         AST_Module* mod = 0;
-                        file_data = _reparse(fn, cache_fn, mod);
+                        file_data = _reparse(fn, cache_fn, mod, inherited_flags);
                         if (mod)
                             return mod;
                         assert(file_data.size());
@@ -1295,7 +1302,7 @@ AST_Module* caching_parse_file(const char* fn) {
             file_data.clear();
 
             AST_Module* mod = 0;
-            file_data = _reparse(fn, cache_fn, mod);
+            file_data = _reparse(fn, cache_fn, mod, inherited_flags);
             if (mod)
                 return mod;
             assert(file_data.size());

src/codegen/parser.h

@@ -15,14 +15,15 @@
 #ifndef PYSTON_CODEGEN_PARSER_H
 #define PYSTON_CODEGEN_PARSER_H
 
+#include "core/types.h"
+
 namespace pyston {
 
 class AST_Module;
 
-AST_Module* parse_string(const char* code);
-
-AST_Module* parse_file(const char* fn);
-AST_Module* caching_parse_file(const char* fn);
+AST_Module* parse_string(const char* code, FutureFlags inherited_flags);
+AST_Module* parse_file(const char* fn, FutureFlags inherited_flags);
+AST_Module* caching_parse_file(const char* fn, FutureFlags inherited_flags);
 }
 
 #endif

src/codegen/pypa-parser.cpp

@@ -1163,7 +1163,7 @@ private:
     int position;
 };
 
-static AST_Module* parse_with_reader(std::unique_ptr<pypa::Reader> reader) {
+static AST_Module* parse_with_reader(std::unique_ptr<pypa::Reader> reader, FutureFlags future_flags) {
     pypa::Lexer lexer(std::move(reader));
     pypa::SymbolTablePtr symbols;
     pypa::AstModulePtr module;
@@ -1176,6 +1176,35 @@ static AST_Module* parse_with_reader(std::unique_ptr<pypa::Reader> reader) {
     options.error_handler = pypaErrorHandler;
     options.escape_handler = pypaEscapeDecoder;
 
+    if (future_flags & CO_FUTURE_PRINT_FUNCTION) {
+        future_flags &= ~CO_FUTURE_PRINT_FUNCTION;
+        options.initial_future_features.print_function = true;
+    }
+
+    if (future_flags & CO_FUTURE_DIVISION) {
+        future_flags &= ~CO_FUTURE_DIVISION;
+        options.initial_future_features.division = true;
+    }
+
+    if (future_flags & CO_FUTURE_ABSOLUTE_IMPORT) {
+        future_flags &= ~CO_FUTURE_ABSOLUTE_IMPORT;
+        options.initial_future_features.absolute_imports = true;
+    }
+
+    if (future_flags & CO_FUTURE_WITH_STATEMENT) {
+        future_flags &= ~CO_FUTURE_WITH_STATEMENT;
+        options.initial_future_features.with_statement = true;
+    }
+
+    if (future_flags & CO_FUTURE_UNICODE_LITERALS) {
+        future_flags &= ~CO_FUTURE_UNICODE_LITERALS;
+        options.initial_future_features.unicode_literals = true;
+    }
+
+    // Strip out some flags:
+    future_flags &= ~(CO_NESTED);
+    RELEASE_ASSERT(!future_flags, "0x%x", future_flags);
+
     if (pypa::parse(lexer, module, symbols, options) && module) {
         return readModule(*module);
     }
@@ -1199,17 +1228,17 @@ PyObject* PystonStringReader::open_python_file() noexcept {
     return PycStringIO->NewInput(s);
 }
 
-AST_Module* pypa_parse(char const* file_path) {
+AST_Module* pypa_parse(char const* file_path, FutureFlags future_flags) {
     auto reader = PystonFileReader::create(file_path);
     if (!reader)
         return nullptr;
 
-    return parse_with_reader(std::move(reader));
+    return parse_with_reader(std::move(reader), future_flags);
 }
 
-AST_Module* pypa_parse_string(char const* str) {
+AST_Module* pypa_parse_string(char const* str, FutureFlags future_flags) {
     auto reader = llvm::make_unique<PystonStringReader>(str);
 
-    return parse_with_reader(std::move(reader));
+    return parse_with_reader(std::move(reader), future_flags);
 }
 }

src/codegen/pypa-parser.h

@@ -17,10 +17,12 @@
 
 #include <cstdio>
 
+#include "core/types.h"
+
 namespace pyston {
 class AST_Module;
-AST_Module* pypa_parse(char const* file_path);
-AST_Module* pypa_parse_string(char const* str);
+AST_Module* pypa_parse(char const* file_path, FutureFlags future_flags);
+AST_Module* pypa_parse_string(char const* str, FutureFlags future_flags);
 }
 
 #endif // PYSTON_CODEGEN_PYPAPARSER_H

src/jit.cpp

@@ -418,7 +418,7 @@ static int main(int argc, char** argv) {
         if (command != NULL) {
             try {
                 main_module = createModule(boxString("__main__"), "<string>");
-                AST_Module* m = parse_string(command);
+                AST_Module* m = parse_string(command, /* future_flags = */ 0);
                 compileAndRunModule(m, main_module);
                 rtncode = 0;
             } catch (ExcInfo e) {
@@ -457,7 +457,7 @@ static int main(int argc, char** argv) {
 
                 main_module = createModule(boxString("__main__"), fn);
                 try {
-                    AST_Module* ast = caching_parse_file(fn);
+                    AST_Module* ast = caching_parse_file(fn, /* future_flags = */ 0);
                     compileAndRunModule(ast, main_module);
                 } catch (ExcInfo e) {
                     setCAPIException(e);
@@ -487,7 +487,7 @@ static int main(int argc, char** argv) {
                 add_history(line);
 
                 try {
-                    AST_Module* m = parse_string(line);
+                    AST_Module* m = parse_string(line, /* future_flags = */ 0);
 
                     Timer _t("repl");
 

src/runtime/import.cpp

@@ -43,7 +43,7 @@ static void removeModule(BoxedString* name) {
 Box* createAndRunModule(BoxedString* name, const std::string& fn) {
     BoxedModule* module = createModule(name, fn.c_str());
 
-    AST_Module* ast = caching_parse_file(fn.c_str());
+    AST_Module* ast = caching_parse_file(fn.c_str(), /* future_flags = */ 0);
     assert(ast);
     try {
         compileAndRunModule(ast, module);
@@ -69,7 +69,7 @@ static Box* createAndRunModule(BoxedString* name, const std::string& fn, const s
     static BoxedString* path_str = internStringImmortal("__path__");
     module->setattr(path_str, path_list, NULL);
 
-    AST_Module* ast = caching_parse_file(fn.c_str());
+    AST_Module* ast = caching_parse_file(fn.c_str(), /* future_flags = */ 0);
     assert(ast);
     try {
         compileAndRunModule(ast, module);
@@ -646,7 +646,7 @@ extern "C" PyObject* PyImport_ExecCodeModuleEx(char* name, PyObject* co, char* p
 
         static BoxedString* file_str = internStringImmortal("__file__");
         module->setattr(file_str, boxString(pathname), NULL);
-        AST_Module* ast = parse_string(code->data());
+        AST_Module* ast = parse_string(code->data(), /* future_flags = */ 0);
         compileAndRunModule(ast, module);
         return module;
     } catch (ExcInfo e) {

test/integration/decorator_test.py

+import os, sys
+sys.path.append(os.path.dirname(__file__) + "/../lib")
+
+EXTRA_PATH = os.path.dirname(os.path.abspath(__file__)) + "/../lib/decorator/src"
+sys.path.insert(0, EXTRA_PATH)
+EXTRA_PATH = os.path.dirname(os.path.abspath(__file__)) + "/../lib/decorator"
+sys.path.insert(0, EXTRA_PATH)
+
+import decorator
+import tests.test
+
+import unittest
+unittest.main(tests.test)

decorator @ e13cf070

+Subproject commit e13cf0704b7c28f1de1e71cb7c59b47a84daa9ca

test/tests/print_function.py

@@ -52,3 +52,9 @@ except TypeError:
 
 for i in xrange(10):
     print()
+
+# These two should output "1 2"
+exec """print(1, 2)"""
+eval("print(1, 2)")
+# This one should output "(1, 2)"
+exec compile("print(1, 2)", "<string>", "exec", dont_inherit=True)

test/unittests/analysis.cpp

@@ -26,7 +26,7 @@ protected:
 
 TEST_F(AnalysisTest, augassign) {
     const std::string fn("test/unittests/analysis_listcomp.py");
-    AST_Module* module = caching_parse_file(fn.c_str());
+    AST_Module* module = caching_parse_file(fn.c_str(), 0);
     assert(module);
 
     ScopingAnalysis *scoping = new ScopingAnalysis(module, true);
@@ -59,7 +59,7 @@ TEST_F(AnalysisTest, augassign) {
 
 void doOsrTest(bool is_osr, bool i_maybe_undefined) {
     const std::string fn("test/unittests/analysis_osr.py");
-    AST_Module* module = caching_parse_file(fn.c_str());
+    AST_Module* module = caching_parse_file(fn.c_str(), 0);
     assert(module);
 
     ScopingAnalysis *scoping = new ScopingAnalysis(module, true);