Merge branch 'master' of https://github.com/dropbox/pyston into generators

Conflicts:
	src/codegen/runtime_hooks.h
	src/runtime/builtin_modules/builtins.cpp

src/analysis/scoping_analysis.cpp

@@ -327,6 +327,17 @@ public:
         return true;
     }
 
+    virtual bool visit_importfrom(AST_ImportFrom* node) {
+        for (int i = 0; i < node->names.size(); i++) {
+            AST_alias* alias = node->names[i];
+            if (alias->asname.size())
+                doWrite(alias->asname);
+            else
+                doWrite(alias->name);
+        }
+        return true;
+    }
+
     static void collect(AST* node, ScopingAnalysis::NameUsageMap* map) {
         assert(map);
         assert(map->count(node));

src/codegen/irgen/irgenerator.cpp

@@ -1479,6 +1479,14 @@ private:
             const std::string& name = alias->name;
             const std::string& asname = alias->asname.size() ? alias->asname : alias->name;
 
+            if (name == "*") {
+                RELEASE_ASSERT(irstate->getSourceInfo()->ast->type == AST_TYPE::Module,
+                               "import * not supported in functions");
+                emitter.createCall2(exc_info, g.funcs.importStar, imported_v,
+                                    embedConstantPtr(irstate->getSourceInfo()->parent_module, g.llvm_module_type_ptr));
+                continue;
+            }
+
             // TODO add patchpoints to this?
             llvm::Value* r = emitter.createCall2(exc_info, g.funcs.importFrom, module->getValue(),
                                                  embedConstantPtr(&name, g.llvm_str_type_ptr)).getInstruction();

src/codegen/runtime_hooks.cpp

@@ -188,6 +188,7 @@ void initGlobalFuncs(GlobalState& g) {
     GET(unaryop);
     GET(import);
     GET(importFrom);
+    GET(importStar);
     GET(repr);
     GET(isinstance);
     GET(yield);

src/codegen/runtime_hooks.h

@@ -34,7 +34,7 @@ struct GlobalFuncs {
         *boxInstanceMethod, *boxBool, *unboxBool, *createTuple, *createDict, *createList, *createSlice,
         *createUserClass, *createClosure, *createGenerator, *insideGenerator;
     llvm::Value* getattr, *setattr, *print, *nonzero, *binop, *compare, *augbinop, *unboxedLen, *getitem, *getclsattr,
-        *getGlobal, *setitem, *delitem, *unaryop, *import, *importFrom, *repr, *isinstance, *yield;
+        *getGlobal, *setitem, *delitem, *unaryop, *import, *importFrom, *importStar, *repr, *isinstance, *yield;
 
     llvm::Value* checkUnpackingLength, *raiseAttributeError, *raiseAttributeErrorStr, *raiseNotIterableError,
         *assertNameDefined, *assertFail;

src/runtime/builtin_modules/_sre.cpp

+// Copyright (c) 2014 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cmath>
+
+#include "core/types.h"
+#include "gc/collector.h"
+#include "runtime/gc_runtime.h"
+#include "runtime/inline/boxing.h"
+#include "runtime/types.h"
+#include "runtime/util.h"
+
+namespace pyston {
+
+BoxedModule* sre_module;
+
+void setupSre() {
+    sre_module = createModule("_sre", "__builtin__");
+
+    sre_module->giveAttr("MAGIC", boxInt(20031017));
+    sre_module->giveAttr("CODESIZE", boxInt(4));
+}
+}

src/runtime/builtin_modules/builtins.cpp

@@ -370,9 +370,9 @@ BoxedClass* notimplemented_cls;
 BoxedModule* builtins_module;
 
 // TODO looks like CPython and pypy put this into an "exceptions" module:
-BoxedClass* Exception, *AssertionError, *AttributeError, *TypeError, *NameError, *KeyError, *IndexError, *IOError,
-    *OSError, *ZeroDivisionError, *ValueError, *UnboundLocalError, *RuntimeError, *ImportError, *StopIteration,
-    *GeneratorExit;
+BoxedClass* Exception, *AssertionError, *AttributeError, *GeneratorExit, *TypeError, *NameError, *KeyError, *IndexError,
+    *IOError, *OSError, *ZeroDivisionError, *ValueError, *UnboundLocalError, *RuntimeError, *ImportError,
+    *StopIteration, *Warning;
 
 const ObjectFlavor exception_flavor(&boxGCHandler, NULL);
 Box* exceptionNew1(BoxedClass* cls) {
@@ -448,6 +448,7 @@ void setupBuiltins() {
     Exception = makeBuiltinException(object_cls, "Exception");
     AssertionError = makeBuiltinException(Exception, "AssertionError");
     AttributeError = makeBuiltinException(Exception, "AttributeError");
+    GeneratorExit = makeBuiltinException(Exception, "GeneratorExit");
     TypeError = makeBuiltinException(Exception, "TypeError");
     NameError = makeBuiltinException(Exception, "NameError");
     KeyError = makeBuiltinException(Exception, "KeyError");
@@ -460,7 +461,9 @@ void setupBuiltins() {
     RuntimeError = makeBuiltinException(Exception, "RuntimeError");
     ImportError = makeBuiltinException(Exception, "ImportError");
     StopIteration = makeBuiltinException(Exception, "StopIteration");
-    GeneratorExit = makeBuiltinException(Exception, "GeneratorExit");
+    Warning = makeBuiltinException(Exception, "Warning");
+    /*ImportWarning =*/makeBuiltinException(Warning, "ImportWarning");
+    /*PendingDeprecationWarning =*/makeBuiltinException(Warning, "PendingDeprecationWarning");
 
     repr_obj = new BoxedFunction(boxRTFunction((void*)repr, UNKNOWN, 1));
     builtins_module->giveAttr("repr", repr_obj);

src/runtime/builtin_modules/posix.cpp

@@ -27,5 +27,7 @@ BoxedModule* posix_module;
 
 void setupPosix() {
     posix_module = createModule("posix", "__builtin__");
+
+    posix_module->giveAttr("error", OSError);
 }
 }

src/runtime/builtin_modules/sys.cpp

@@ -87,6 +87,12 @@ void setupSys() {
     sys_module->giveAttr("stdout", new BoxedFile(stdout));
     sys_module->giveAttr("stdin", new BoxedFile(stdin));
     sys_module->giveAttr("stderr", new BoxedFile(stderr));
+
+    sys_module->giveAttr("warnoptions", new BoxedList());
+    sys_module->giveAttr("py3kwarning", False);
+
+    sys_module->giveAttr("hexversion", boxInt(0x01000000 * PYTHON_VERSION_MAJOR + 0x010000 * PYTHON_VERSION_MINOR
+                                              + 0x0100 * PYTHON_VERSION_MICRO));
 }
 
 void setupSysEnd() {

src/runtime/inline/link_forcer.cpp

@@ -77,6 +77,7 @@ void force() {
     FORCE(unaryop);
     FORCE(import);
     FORCE(importFrom);
+    FORCE(importStar);
     FORCE(repr);
     FORCE(isinstance);
     FORCE(yield);

src/runtime/objmodel.cpp

@@ -1963,7 +1963,8 @@ Box* callFunc(BoxedFunction* func, CallRewriteArgs* rewrite_args, ArgPassSpec ar
         return createGenerator(func, oarg1, oarg2, oarg3, oargs);
     }
 
-    return callCLFunc(f, rewrite_args, num_output_args, closure, (BoxedGenerator*)func->isGenerator, oarg1, oarg2, oarg3, oargs);
+    return callCLFunc(f, rewrite_args, num_output_args, closure, (BoxedGenerator*)func->isGenerator, oarg1, oarg2,
+                      oarg3, oargs);
 }
 
 Box* callCLFunc(CLFunction* f, CallRewriteArgs* rewrite_args, int num_output_args, BoxedClosure* closure,
@@ -2993,4 +2994,52 @@ extern "C" Box* importFrom(Box* _m, const std::string* name) {
 
     raiseExcHelper(ImportError, "cannot import name %s", name->c_str());
 }
+
+extern "C" void importStar(Box* _from_module, BoxedModule* to_module) {
+    assert(_from_module->cls == module_cls);
+    BoxedModule* from_module = static_cast<BoxedModule*>(_from_module);
+
+    static std::string all_str("__all__");
+    static std::string getitem_str("__getitem__");
+    Box* all = from_module->getattr(all_str);
+
+    if (all) {
+        Box* all_getitem = typeLookup(all->cls, getitem_str, NULL, NULL);
+        if (!all_getitem)
+            raiseExcHelper(TypeError, "'%s' object does not support indexing", getTypeName(all)->c_str());
+
+        int idx = 0;
+        while (true) {
+            Box* attr_name;
+            try {
+                attr_name = runtimeCallInternal2(all_getitem, NULL, ArgPassSpec(2), all, boxInt(idx));
+            } catch (Box* b) {
+                if (b->cls == IndexError)
+                    break;
+                throw;
+            }
+            idx++;
+
+            if (attr_name->cls != str_cls)
+                raiseExcHelper(TypeError, "attribute name must be string, not '%s'", getTypeName(attr_name)->c_str());
+
+            BoxedString* casted_attr_name = static_cast<BoxedString*>(attr_name);
+            Box* attr_value = from_module->getattr(casted_attr_name->s);
+
+            if (!attr_value)
+                raiseExcHelper(AttributeError, "'module' object has no attribute '%s'", casted_attr_name->s.c_str());
+
+            to_module->setattr(casted_attr_name->s, attr_value, NULL);
+        }
+        return;
+    }
+
+    HCAttrs* module_attrs = from_module->getAttrsPtr();
+    for (auto& p : module_attrs->hcls->attr_offsets) {
+        if (p.first[0] == '_')
+            continue;
+
+        to_module->setattr(p.first, module_attrs->attr_list->attrs[p.second], NULL);
+    }
+}
 }

src/runtime/objmodel.h

@@ -44,7 +44,6 @@ extern "C" const std::string* getNameOfClass(BoxedClass* cls);
 
 // TODO sort this
 extern "C" void my_assert(bool b);
-extern "C" Box* importFrom(Box* obj, const std::string* attr);
 extern "C" Box* getattr(Box* obj, const char* attr);
 extern "C" void setattr(Box* obj, const char* attr, Box* attr_val);
 extern "C" bool nonzero(Box* obj);
@@ -74,6 +73,8 @@ extern "C" void delitem(Box* target, Box* slice);
 extern "C" Box* getclsattr(Box* obj, const char* attr);
 extern "C" Box* unaryop(Box* operand, int op_type);
 extern "C" Box* import(const std::string* name);
+extern "C" Box* importFrom(Box* obj, const std::string* attr);
+extern "C" void importStar(Box* from_module, BoxedModule* to_module);
 extern "C" void checkUnpackingLength(i64 expected, i64 given);
 extern "C" void assertNameDefined(bool b, const char* name);
 extern "C" void assertFail(BoxedModule* inModule, Box* msg);

src/runtime/types.cpp

@@ -87,6 +87,8 @@ extern "C" BoxedFunction::BoxedFunction(CLFunction* f)
         this->giveAttr("__module__", modname);
     }
 
+    this->giveAttr("__doc__", None);
+
     assert(f->num_defaults == ndefaults);
 }
 
@@ -111,6 +113,8 @@ extern "C" BoxedFunction::BoxedFunction(CLFunction* f, std::initializer_list<Box
         this->giveAttr("__module__", modname);
     }
 
+    this->giveAttr("__doc__", None);
+
     assert(f->num_defaults == ndefaults);
 }
 
@@ -654,6 +658,7 @@ void setupRuntime() {
     setupThread();
     setupErrno();
     setupPosix();
+    setupSre();
 
     setupCAPI();
 

src/runtime/types.h

@@ -63,6 +63,7 @@ void setupTime();
 void setupThread();
 void setupErrno();
 void setupPosix();
+void setupSre();
 void setupSysEnd();
 
 BoxedDict* getSysModulesDict();

test/tests/import_nested_target.py

 print "importing nested", __name__
 
+_y = 2
 y = 2
 
 def bar():

test/tests/import_star.py

+# import_target defines __all__ to be ['x']
+from import_target import *
+
+print x
+try:
+    print foo
+    assert 0
+except NameError:
+    pass
+try:
+    print _x
+    assert 0
+except NameError:
+    pass
+
+# import_nested_target doesn't define __all__
+from import_nested_target import *
+
+print y
+try:
+    print _y
+    assert 0
+except NameError:
+    pass
+
+# import_target_bad_all defines an __all__ with a nonexistent name
+try:
+    from import_target_bad_all import *
+    assert 0
+except AttributeError:
+    pass
+
+from import_target_custom_all import *
+print z

test/tests/import_target.py

@@ -14,3 +14,5 @@ def foo():
 class C(object):
     pass
 
+_x = 1
+__all__ = ['x']

test/tests/import_target_bad_all.py

+# No 'x' defined in this file, importing * will fail with an AttributeError
+__all__ = ['x']

test/tests/import_target_custom_all.py

+# The __all__ variable must support iteration, through the old-style (__getitem__) protocol
+
+z = 123
+
+class C(object):
+    def __getitem__(self, idx):
+        print "__getitem__", idx
+        if idx == 0:
+            return 'z'
+        raise IndexError
+
+__all__ = C()
+

tools/tester.py

@@ -185,13 +185,15 @@ def run_test(fn, check_stats, run_memcheck):
                 r += "    \033[31mFAILED\033[0m (bad output)"
                 failed.append(fn)
                 return r
-            exp_fd, exp_fn = tempfile.mkstemp()
-            out_fd, out_fn = tempfile.mkstemp()
+            exp_fd, exp_fn = tempfile.mkstemp(prefix="expected_")
+            out_fd, out_fn = tempfile.mkstemp(prefix="received_")
             os.fdopen(exp_fd, 'w').write(expected_out)
             os.fdopen(out_fd, 'w').write(out)
             p = subprocess.Popen(["diff", "-C2", "-a", exp_fn, out_fn], stdout=subprocess.PIPE, preexec_fn=set_ulimits)
             diff = p.stdout.read()
             assert p.wait() in (0, 1)
+            os.unlink(exp_fn)
+            os.unlink(out_fn)
             raise Exception("Failed on %s:\n%s" % (fn, diff))
     elif not TEST_PYPY and canonicalize_stderr(stderr) != canonicalize_stderr(expected_err):
         if KEEP_GOING: