libpypa integration

(Committed by kmod)
Closes #93

docs/INSTALLING.md

@@ -108,6 +108,19 @@ sudo apt-get install libreadline-dev
 sudo apt-get install libgmp3-dev
 ```
 
+### libpypa
+
+```
+cd ~/pyston_deps
+git clone git://github.com/vinzenz/pypa
+mkdir pypa-install
+cd pypa
+./autogen.sh
+./configure --prefix=$HOME/pyston_deps/pypa-install CXX=$HOME/pyston_deps/gcc-4.8.2-install/bin/g++
+make -j4
+make install
+```
+
 # Optional dependencies
 
 There are a number of optional dependencies that the build system knows about, but aren't strictly necessary for building and running Pyston.  Most of them are related to developing and debugging:

src/Makefile

@@ -137,6 +137,7 @@ COMMON_CXXFLAGS += -Woverloaded-virtual
 COMMON_CXXFLAGS += -fexceptions -fno-rtti
 COMMON_CXXFLAGS += -Wno-invalid-offsetof # allow the use of "offsetof", and we'll just have to make sure to only use it legally.
 COMMON_CXXFLAGS += -DENABLE_INTEL_JIT_EVENTS=$(ENABLE_INTEL_JIT_EVENTS)
+COMMON_CXXFLAGS += -I$(DEPS_DIR)/pypa-install/include
 
 ifeq ($(ENABLE_VALGRIND),0)
 	COMMON_CXXFLAGS += -DNVALGRIND
@@ -151,6 +152,7 @@ COMMON_CXXFLAGS += -DDEFAULT_PYTHON_MAJOR_VERSION=$(PYTHON_MAJOR_VERSION) -DDEFA
 
 # Use our "custom linker" that calls gold if available
 COMMON_LDFLAGS := -B../tools/build_system -L/usr/local/lib -lpthread -ldl -lcurses -lm -lunwind -lz -llzma -L$(DEPS_DIR)/gcc-4.8.2-install/lib64 -lreadline -lgmp
+COMMON_LDFLAGS += $(DEPS_DIR)/pypa-install/lib/libpypa.a
 # Make sure that we put all symbols in the dynamic symbol table so that MCJIT can load them;
 # TODO should probably do the linking before MCJIT
 COMMON_LDFLAGS += -Wl,-E
@@ -746,6 +748,7 @@ $(eval \
 .PHONY: test$1 check$1
 check$1 test$1: pyston$1 ext_pyston
 	python ../tools/tester.py -R pyston$1 -j$(TEST_THREADS) -k $(TESTS_DIR) $(ARGS)
+	python ../tools/tester.py -a -x -R pyston$1 -j$(TEST_THREADS) -k $(TESTS_DIR) $(ARGS)
 	python ../tools/tester.py -R pyston$1 -j$(TEST_THREADS) -a -n -k $(TESTS_DIR) $(ARGS)
 	python ../tools/tester.py -R pyston$1 -j$(TEST_THREADS) -a -O -k $(TESTS_DIR) $(ARGS)
 

src/codegen/parse_ast.py

@@ -93,6 +93,7 @@ TYPE_MAP = {
         _ast.Invert: 84,
         _ast.UAdd: 85,
         _ast.FloorDiv: 86,
+        _ast.Ellipsis: 87,
     }
 
 if sys.version_info >= (2,7):

src/codegen/parser.cpp

@@ -24,6 +24,7 @@
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 
+#include "codegen/pypa-parser.h"
 #include "core/ast.h"
 #include "core/options.h"
 #include "core/stats.h"
@@ -899,6 +900,10 @@ static std::string getParserCommandLine(const char* fn) {
 AST_Module* parse(const char* fn) {
     Timer _t("parsing");
 
+    if (ENABLE_PYPA_PARSER) {
+        return pypa_parse(fn);
+    }
+
     FILE* fp = popen(getParserCommandLine(fn).c_str(), "r");
 
     BufferedReader* reader = new BufferedReader(fp);
@@ -962,6 +967,10 @@ static void _reparse(const char* fn, const std::string& cache_fn) {
 AST_Module* caching_parse(const char* fn) {
     Timer _t("parsing");
 
+    if (ENABLE_PYPA_PARSER) {
+        return pypa_parse(fn);
+    }
+
     int code;
     std::string cache_fn = std::string(fn) + "c";
 

src/codegen/pypa-parser.h

+// Copyright (c) 2014 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef PYSTON_CODEGEN_PYPAPARSER_H
+#define PYSTON_CODEGEN_PYPAPARSER_H
+
+namespace pyston {
+class AST_Module;
+AST_Module* pypa_parse(char const* file_path);
+}
+
+#endif // PYSTON_CODEGEN_PYPAPARSER_H

src/core/ast.cpp

@@ -423,6 +423,16 @@ void* AST_DictComp::accept_expr(ExprVisitor* v) {
     return v->visit_dictcomp(this);
 }
 
+void AST_Ellipsis::accept(ASTVisitor* v) {
+    bool skip = v->visit_ellipsis(this);
+    if (skip)
+        return;
+}
+
+void* AST_Ellipsis::accept_expr(ExprVisitor* v) {
+    return v->visit_ellipsis(this);
+}
+
 void AST_ExceptHandler::accept(ASTVisitor* v) {
     bool skip = v->visit_excepthandler(this);
     if (skip)
@@ -447,6 +457,18 @@ void AST_Expr::accept_stmt(StmtVisitor* v) {
     v->visit_expr(this);
 }
 
+
+void AST_ExtSlice::accept(ASTVisitor* v) {
+    bool skip = v->visit_extslice(this);
+    if (skip)
+        return;
+    visitVector(dims, v);
+}
+
+void* AST_ExtSlice::accept_expr(ExprVisitor* v) {
+    return v->visit_extslice(this);
+}
+
 void AST_For::accept(ASTVisitor* v) {
     bool skip = v->visit_for(this);
     if (skip)
@@ -1196,6 +1218,11 @@ bool PrintVisitor::visit_dictcomp(AST_DictComp* node) {
     return true;
 }
 
+bool PrintVisitor::visit_ellipsis(AST_Ellipsis*) {
+    printf("...");
+    return true;
+}
+
 bool PrintVisitor::visit_excepthandler(AST_ExceptHandler* node) {
     printf("except");
     if (node->type) {
@@ -1222,6 +1249,15 @@ bool PrintVisitor::visit_expr(AST_Expr* node) {
     return false;
 }
 
+bool PrintVisitor::visit_extslice(AST_ExtSlice* node) {
+    for (int i = 0; i < node->dims.size(); ++i) {
+        if (i > 0)
+            printf(", ");
+        node->dims[i]->accept(this);
+    }
+    return true;
+}
+
 bool PrintVisitor::visit_for(AST_For* node) {
     printf("<for loop>\n");
     return true;

src/core/ast.h

@@ -116,6 +116,7 @@ enum AST_TYPE {
     FloorDiv = 86,
     DictComp = 15,
     Set = 43,
+    Ellipsis = 87,
 
     // Pseudo-nodes that are specific to this compiler:
     Branch = 200,
@@ -394,11 +395,21 @@ public:
     virtual void accept(ASTVisitor* v);
     virtual void accept_stmt(StmtVisitor* v);
 
-    AST_Delete() : AST_stmt(AST_TYPE::Delete){};
+    AST_Delete() : AST_stmt(AST_TYPE::Delete) {}
 
     static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Delete;
 };
 
+class AST_Ellipsis : public AST_expr {
+public:
+    virtual void accept(ASTVisitor* v);
+    virtual void* accept_expr(ExprVisitor* v);
+
+    AST_Ellipsis() : AST_expr(AST_TYPE::Ellipsis) {}
+
+    static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Ellipsis;
+};
+
 class AST_Expr : public AST_stmt {
 public:
     AST_expr* value;
@@ -424,6 +435,19 @@ public:
     static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::ExceptHandler;
 };
 
+
+class AST_ExtSlice : public AST_expr {
+public:
+    std::vector<AST_expr*> dims;
+
+    virtual void accept(ASTVisitor* v);
+    virtual void* accept_expr(ExprVisitor* v);
+
+    AST_ExtSlice() : AST_expr(AST_TYPE::ExtSlice) {}
+
+    static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::ExtSlice;
+};
+
 class AST_For : public AST_stmt {
 public:
     std::vector<AST_stmt*> body, orelse;
@@ -981,8 +1005,10 @@ public:
     virtual bool visit_delete(AST_Delete* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_dict(AST_Dict* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_dictcomp(AST_DictComp* node) { RELEASE_ASSERT(0, ""); }
+    virtual bool visit_ellipsis(AST_Ellipsis* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_excepthandler(AST_ExceptHandler* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_expr(AST_Expr* node) { RELEASE_ASSERT(0, ""); }
+    virtual bool visit_extslice(AST_ExtSlice* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_for(AST_For* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_functiondef(AST_FunctionDef* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_generatorexp(AST_GeneratorExp* node) { RELEASE_ASSERT(0, ""); }
@@ -1047,8 +1073,10 @@ public:
     virtual bool visit_delete(AST_Delete* node) { return false; }
     virtual bool visit_dict(AST_Dict* node) { return false; }
     virtual bool visit_dictcomp(AST_DictComp* node) { return false; }
+    virtual bool visit_ellipsis(AST_Ellipsis* node) { return false; }
     virtual bool visit_excepthandler(AST_ExceptHandler* node) { return false; }
     virtual bool visit_expr(AST_Expr* node) { return false; }
+    virtual bool visit_extslice(AST_ExtSlice* node) { return false; }
     virtual bool visit_for(AST_For* node) { return false; }
     virtual bool visit_functiondef(AST_FunctionDef* node) { return false; }
     virtual bool visit_generatorexp(AST_GeneratorExp* node) { return false; }
@@ -1103,6 +1131,8 @@ public:
     virtual void* visit_compare(AST_Compare* node) { RELEASE_ASSERT(0, ""); }
     virtual void* visit_dict(AST_Dict* node) { RELEASE_ASSERT(0, ""); }
     virtual void* visit_dictcomp(AST_DictComp* node) { RELEASE_ASSERT(0, ""); }
+    virtual void* visit_ellipsis(AST_Ellipsis* node) { RELEASE_ASSERT(0, ""); }
+    virtual void* visit_extslice(AST_ExtSlice* node) { RELEASE_ASSERT(0, ""); }
     virtual void* visit_generatorexp(AST_GeneratorExp* node) { RELEASE_ASSERT(0, ""); }
     virtual void* visit_ifexp(AST_IfExp* node) { RELEASE_ASSERT(0, ""); }
     virtual void* visit_index(AST_Index* node) { RELEASE_ASSERT(0, ""); }
@@ -1185,8 +1215,10 @@ public:
     virtual bool visit_delete(AST_Delete* node);
     virtual bool visit_dict(AST_Dict* node);
     virtual bool visit_dictcomp(AST_DictComp* node);
+    virtual bool visit_ellipsis(AST_Ellipsis* node);
     virtual bool visit_excepthandler(AST_ExceptHandler* node);
     virtual bool visit_expr(AST_Expr* node);
+    virtual bool visit_extslice(AST_ExtSlice* node);
     virtual bool visit_for(AST_For* node);
     virtual bool visit_functiondef(AST_FunctionDef* node);
     virtual bool visit_generatorexp(AST_GeneratorExp* node);

src/core/options.cpp

@@ -34,6 +34,7 @@ bool DUMPJIT = false;
 bool TRAP = false;
 bool USE_STRIPPED_STDLIB = true; // always true
 bool ENABLE_INTERPRETER = true;
+bool ENABLE_PYPA_PARSER = false;
 
 static bool _GLOBAL_ENABLE = 1;
 bool ENABLE_ICS = 1 && _GLOBAL_ENABLE;

src/core/options.h

@@ -30,7 +30,8 @@ inline int version_hex(int major, int minor, int micro, int level = 0, int seria
 
 extern int MAX_OPT_ITERATIONS;
 
-extern bool SHOW_DISASM, FORCE_OPTIMIZE, BENCH, PROFILE, DUMPJIT, TRAP, USE_STRIPPED_STDLIB, ENABLE_INTERPRETER;
+extern bool SHOW_DISASM, FORCE_OPTIMIZE, BENCH, PROFILE, DUMPJIT, TRAP, USE_STRIPPED_STDLIB, ENABLE_INTERPRETER,
+    ENABLE_PYPA_PARSER;
 
 extern bool ENABLE_ICS, ENABLE_ICGENERICS, ENABLE_ICGETITEMS, ENABLE_ICSETITEMS, ENABLE_ICDELITEMS, ENABLE_ICBINEXPS,
     ENABLE_ICNONZEROS, ENABLE_ICCALLSITES, ENABLE_ICSETATTRS, ENABLE_ICGETATTRS, ENALBE_ICDELATTRS, ENABLE_ICGETGLOBALS,

src/jit.cpp

@@ -57,7 +57,7 @@ int main(int argc, char** argv) {
     bool force_repl = false;
     bool repl = true;
     bool stats = false;
-    while ((code = getopt(argc, argv, "+Oqcdibpjtrsvn")) != -1) {
+    while ((code = getopt(argc, argv, "+Oqcdibpjtrsvnx")) != -1) {
         if (code == 'O')
             FORCE_OPTIMIZE = true;
         else if (code == 't')
@@ -84,6 +84,8 @@ int main(int argc, char** argv) {
             stats = true;
         } else if (code == 'r') {
             USE_STRIPPED_STDLIB = true;
+        } else if (code == 'x') {
+            ENABLE_PYPA_PARSER = true;
         } else if (code == '?')
             abort();
     }