Merge pull request #807 from kmod/doctest

Add "doctest" support

from_cpython/Lib/test/test_compare.py

-# expected: fail
 import unittest
 from test import test_support
 

from_cpython/Lib/test/test_doctest2.py

-# expected: fail
 # -*- coding: utf-8 -*-
 u"""A module to test whether doctest recognizes some 2.2 features,
 like static and class methods.

from_cpython/Lib/test/test_getopt.py

-# expected: fail
 # test_getopt.py
 # David Goodger <dgoodger@bigfoot.com> 2000-08-19
 

from_cpython/Lib/test/test_pow.py

-# expected: fail
 import test.test_support, unittest
 
 class PowTest(unittest.TestCase):

from_cpython/Lib/test/test_sets.py

-# expected: fail
 import unittest, operator, copy, pickle, random
 from test import test_support
 

from_cpython/Lib/test/test_unpack.py

-# expected: fail
 doctests = """
 
 Unpack tuple

src/analysis/type_analysis.cpp

@@ -381,6 +381,7 @@ private:
             case AST_LangPrimitive::NONE:
             case AST_LangPrimitive::SET_EXC_INFO:
             case AST_LangPrimitive::UNCACHE_EXC_INFO:
+            case AST_LangPrimitive::PRINT_EXPR:
                 return NONE;
             case AST_LangPrimitive::HASNEXT:
             case AST_LangPrimitive::NONZERO:

src/codegen/ast_interpreter.cpp

@@ -875,6 +875,11 @@ Value ASTInterpreter::visit_langPrimitive(AST_LangPrimitive* node) {
         assert(node->args.size() == 1);
         Value obj = visit_expr(node->args[0]);
         v = Value(boxBool(hasnext(obj.o)), jit ? jit->emitHasnext(obj) : NULL);
+    } else if (node->opcode == AST_LangPrimitive::PRINT_EXPR) {
+        abortJITing();
+        Value obj = visit_expr(node->args[0]);
+        printExprHelper(obj.o);
+        v = getNone();
     } else
         RELEASE_ASSERT(0, "unknown opcode %d", node->opcode);
     return v;
@@ -1870,6 +1875,13 @@ Box* astInterpretDeopt(CLFunction* clfunc, AST_expr* after_expr, AST_stmt* enclo
     return v ? v : None;
 }
 
+extern "C" void printExprHelper(Box* obj) {
+    Box* displayhook = PySys_GetObject("displayhook");
+    if (!displayhook)
+        raiseExcHelper(RuntimeError, "lost sys.displayhook");
+    runtimeCall(displayhook, ArgPassSpec(1), obj, 0, 0, 0, 0);
+}
+
 static ASTInterpreter* getInterpreterFromFramePtr(void* frame_ptr) {
     // This offsets have to match the layout inside executeInnerAndSetupFrame
     ASTInterpreter** ptr = (ASTInterpreter**)(((uint8_t*)frame_ptr) - 8);

src/codegen/ast_interpreter.h

@@ -83,6 +83,9 @@ FrameInfo* getFrameInfoForInterpretedFrame(void* frame_ptr);
 BoxedClosure* passedClosureForInterpretedFrame(void* frame_ptr);
 
 BoxedDict* localsForInterpretedFrame(void* frame_ptr, bool only_user_visible);
+
+// Executes the equivalent of CPython's PRINT_EXPR opcode (call sys.displayhook)
+extern "C" void printExprHelper(Box* b);
 }
 
 #endif

src/codegen/irgen/hooks.cpp

@@ -366,13 +366,9 @@ static AST_Suite* parseExec(llvm::StringRef source, bool interactive = false) {
                 continue;
 
             AST_Expr* expr = (AST_Expr*)s;
-            AST_Print* print = new AST_Print;
-            print->lineno = expr->lineno;
-            print->col_offset = expr->col_offset;
-            print->dest = NULL;
-            print->nl = true;
-            print->values.push_back(expr->value);
-            parsedModule->body[i] = print;
+            AST_LangPrimitive* print_expr = new AST_LangPrimitive(AST_LangPrimitive::PRINT_EXPR);
+            print_expr->args.push_back(expr->value);
+            expr->value = print_expr;
         }
     }
 

src/codegen/irgen/irgenerator.cpp

@@ -879,6 +879,16 @@ private:
 
                 return getNone();
             }
+            case AST_LangPrimitive::PRINT_EXPR: {
+                assert(node->args.size() == 1);
+
+                CompilerVariable* obj = evalExpr(node->args[0], unw_info);
+                ConcreteCompilerVariable* converted = obj->makeConverted(emitter, obj->getBoxType());
+
+                emitter.createCall(unw_info, g.funcs.printExprHelper, converted->getValue());
+
+                return getNone();
+            }
             default:
                 RELEASE_ASSERT(0, "%d", node->opcode);
         }

src/codegen/runtime_hooks.cpp

@@ -29,6 +29,7 @@
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Transforms/Scalar.h"
 
+#include "codegen/ast_interpreter.h"
 #include "codegen/codegen.h"
 #include "codegen/irgen.h"
 #include "codegen/irgen/hooks.h"
@@ -233,6 +234,7 @@ void initGlobalFuncs(GlobalState& g) {
     GET(assertNameDefined);
     GET(assertFailDerefNameDefined);
     GET(assertFail);
+    GET(printExprHelper);
 
     GET(printFloat);
     GET(listAppendInternal);

src/codegen/runtime_hooks.h

@@ -41,7 +41,7 @@ struct GlobalFuncs {
 
     llvm::Value* unpackIntoArray, *raiseAttributeError, *raiseAttributeErrorStr, *raiseAttributeErrorCapi,
         *raiseAttributeErrorStrCapi, *raiseNotIterableError, *raiseIndexErrorStr, *raiseIndexErrorStrCapi,
-        *assertNameDefined, *assertFail, *assertFailDerefNameDefined;
+        *assertNameDefined, *assertFail, *assertFailDerefNameDefined, *printExprHelper;
     llvm::Value* printFloat, *listAppendInternal, *getSysStdout;
     llvm::Value* runtimeCall0, *runtimeCall1, *runtimeCall2, *runtimeCall3, *runtimeCall, *runtimeCallN;
     llvm::Value* callattr0, *callattr1, *callattr2, *callattr3, *callattr, *callattrN;

src/codegen/unwinding.cpp

@@ -46,6 +46,8 @@
 
 #define UNW_LOCAL_ONLY
 #include <libunwind.h>
+#undef UNW_LOCAL_ONLY
+
 namespace {
 int _dummy_ = unw_set_caching_policy(unw_local_addr_space, UNW_CACHE_PER_THREAD);
 }

src/codegen/unwinding.h

@@ -19,9 +19,9 @@
 
 #include "codegen/codegen.h"
 
-#define UNW_LOCAL_ONLY
-#include <libunwind.h>
-#undef UNW_LOCAL_ONLY
+// Forward-declare libunwind's typedef'd unw_cursor_t:
+struct unw_cursor;
+typedef struct unw_cursor unw_cursor_t;
 
 namespace pyston {
 

src/core/ast.h

@@ -1075,6 +1075,7 @@ public:
         SET_EXC_INFO,
         UNCACHE_EXC_INFO,
         HASNEXT,
+        PRINT_EXPR,
     } opcode;
     std::vector<AST_expr*> args;
 

src/jit.cpp

@@ -484,20 +484,9 @@ static int main(int argc, char** argv) {
 
                     if (m->body.size() > 0 && m->body[0]->type == AST_TYPE::Expr) {
                         AST_Expr* e = ast_cast<AST_Expr>(m->body[0]);
-                        AST_Call* c = new AST_Call();
-                        AST_Name* r = new AST_Name(m->interned_strings->get("repr"), AST_TYPE::Load, 0);
-                        c->func = r;
-                        c->starargs = NULL;
-                        c->kwargs = NULL;
-                        c->args.push_back(e->value);
-                        c->lineno = 0;
-
-                        AST_Print* p = new AST_Print();
-                        p->dest = NULL;
-                        p->nl = true;
-                        p->values.push_back(c);
-                        p->lineno = 0;
-                        m->body[0] = p;
+                        AST_LangPrimitive* print_expr = new AST_LangPrimitive(AST_LangPrimitive::PRINT_EXPR);
+                        print_expr->args.push_back(e->value);
+                        e->value = print_expr;
                     }
 
                     compileAndRunModule(m, main_module);

src/runtime/builtin_modules/sys.cpp

@@ -397,12 +397,60 @@ static PyObject* sys_excepthook(PyObject* self, PyObject* args) noexcept {
     return Py_None;
 }
 
+static PyObject* sys_displayhook(PyObject* self, PyObject* o) noexcept {
+    PyObject* outf;
+
+    // Pyston change: we currently hardcode the builtins module
+    /*
+    PyInterpreterState* interp = PyThreadState_GET()->interp;
+    PyObject* modules = interp->modules;
+    PyObject* builtins = PyDict_GetItemString(modules, "__builtin__");
+
+    if (builtins == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "lost __builtin__");
+        return NULL;
+    }
+    */
+    PyObject* builtins = builtins_module;
+
+    /* Print value except if None */
+    /* After printing, also assign to '_' */
+    /* Before, set '_' to None to avoid recursion */
+    if (o == Py_None) {
+        Py_INCREF(Py_None);
+        return Py_None;
+    }
+    if (PyObject_SetAttrString(builtins, "_", Py_None) != 0)
+        return NULL;
+    if (Py_FlushLine() != 0)
+        return NULL;
+    outf = PySys_GetObject("stdout");
+    if (outf == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "lost sys.stdout");
+        return NULL;
+    }
+    if (PyFile_WriteObject(o, outf, 0) != 0)
+        return NULL;
+    PyFile_SoftSpace(outf, 1);
+    if (Py_FlushLine() != 0)
+        return NULL;
+    if (PyObject_SetAttrString(builtins, "_", o) != 0)
+        return NULL;
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+
 PyDoc_STRVAR(excepthook_doc, "excepthook(exctype, value, traceback) -> None\n"
                              "\n"
                              "Handle an exception by displaying it with a traceback on sys.stderr.\n");
 
+PyDoc_STRVAR(displayhook_doc, "displayhook(object) -> None\n"
+                              "\n"
+                              "Print an object to sys.stdout and also save it in __builtin__._\n");
+
 static PyMethodDef sys_methods[] = {
     { "excepthook", sys_excepthook, METH_VARARGS, excepthook_doc },
+    { "displayhook", sys_displayhook, METH_O, displayhook_doc },
 };
 
 void setupSys() {
@@ -507,6 +555,7 @@ void setupSys() {
         sys_module->giveAttr(md.ml_name, new BoxedCApiFunction(&md, sys_module));
     }
 
+    sys_module->giveAttr("__displayhook__", sys_module->getattr(internStringMortal("displayhook")));
     sys_module->giveAttr("flags", new BoxedSysFlags());
 }
 

src/runtime/inline/CMakeLists.txt

@@ -87,7 +87,7 @@ foreach(BC_SRC ${BC_INLINE_SRCS})
   # generate the bitcode for this file
   add_custom_command(OUTPUT ${BC_SRC_BASENAME}.bc
                      COMMAND ${BC_CXX} ${BC_DEFINES} ${BC_CXX_FLAGS} ${BC_INCLUDES} -c ${BC_SRC_FULLPATH} -o ${BC_SRC_BASENAME}.bc -emit-llvm
-                     DEPENDS ${BC_SRC_FULLPATH} clang ${PYSTON_HEADERS} ${FROM_CPYTHON_HEADERS}
+                     DEPENDS ${BC_SRC_FULLPATH} clang ${PYSTON_HEADERS} ${FROM_CPYTHON_HEADERS} libunwind
                      COMMENT "Building LLVM bitcode ${BC_SRC_BASENAME}.bc"
                      VERBATIM)
   set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${BC_SRC_BASENAME}.bc)

src/runtime/inline/link_forcer.cpp

@@ -15,6 +15,7 @@
 // This file is for forcing the inclusion of function declarations into the stdlib.
 // This is so that the types of the functions are available to the compiler.
 
+#include "codegen/ast_interpreter.h"
 #include "codegen/irgen/hooks.h"
 #include "core/ast.h"
 #include "core/threading.h"
@@ -111,6 +112,7 @@ void force() {
     FORCE(assertNameDefined);
     FORCE(assertFailDerefNameDefined);
     FORCE(assertFail);
+    FORCE(printExprHelper);
 
     FORCE(strOrUnicode);
     FORCE(printFloat);

src/runtime/objmodel.cpp

@@ -2046,6 +2046,7 @@ bool dataDescriptorSetSpecialCases(Box* obj, Box* val, Box* descr, SetattrRewrit
         }
 
         getset_descr->set(obj, val, getset_descr->closure);
+        checkAndThrowCAPIException();
 
         return true;
     } else if (descr->cls == member_descriptor_cls) {

src/runtime/stacktrace.cpp

@@ -27,6 +27,7 @@
 
 #define UNW_LOCAL_ONLY
 #include <libunwind.h>
+#undef UNW_LOCAL_ONLY
 
 namespace pyston {
 

src/runtime/types.cpp

@@ -1900,7 +1900,7 @@ static PyObject* typeModule(Box* _type, void* context) {
     PyObject* mod;
     const char* s;
 
-    if (type->tp_flags & Py_TPFLAGS_HEAPTYPE) {
+    if (type->tp_flags & Py_TPFLAGS_HEAPTYPE && type->is_user_defined) {
         static BoxedString* module_str = internStringImmortal("__module__");
         mod = type->getattr(module_str);
         if (!mod)
@@ -3197,6 +3197,38 @@ static void setupDefaultClassGCParticipation() {
     gc::invalidateOrderedFinalizerList();
 }
 
+static Box* getsetGet(Box* self, Box* obj, Box* type) {
+    // TODO: should call the full descr_check instead
+    if (obj == NULL || obj == None)
+        return self;
+
+    BoxedGetsetDescriptor* getset_descr = static_cast<BoxedGetsetDescriptor*>(self);
+    if (isSubclass(self->cls, pyston_getset_cls)) {
+        return getset_descr->get(obj, getset_descr->closure);
+    } else {
+        RELEASE_ASSERT(isSubclass(self->cls, capi_getset_cls), "");
+        Box* r = getset_descr->get(obj, getset_descr->closure);
+        if (!r)
+            throwCAPIException();
+        return r;
+    }
+}
+
+static Box* getsetSet(Box* self, Box* obj, Box* val) {
+    assert(obj != NULL && obj != None);
+
+    BoxedGetsetDescriptor* getset_descr = static_cast<BoxedGetsetDescriptor*>(self);
+    if (isSubclass(self->cls, pyston_getset_cls)) {
+        getset_descr->set(obj, val, getset_descr->closure);
+        return None;
+    } else {
+        RELEASE_ASSERT(isSubclass(self->cls, capi_getset_cls), "");
+        getset_descr->set(obj, val, getset_descr->closure);
+        checkAndThrowCAPIException();
+        return None;
+    }
+}
+
 bool TRACK_ALLOCATIONS = false;
 void setupRuntime() {
 
@@ -3275,7 +3307,7 @@ void setupRuntime() {
     list_cls = new (0) BoxedHeapClass(object_cls, &listGCHandler, 0, 0, sizeof(BoxedList), false, boxString("list"));
     list_cls->tp_flags |= Py_TPFLAGS_LIST_SUBCLASS;
     pyston_getset_cls = new (0)
-        BoxedHeapClass(object_cls, NULL, 0, 0, sizeof(BoxedGetsetDescriptor), false, boxString("getset"));
+        BoxedHeapClass(object_cls, NULL, 0, 0, sizeof(BoxedGetsetDescriptor), false, boxString("getset_descriptor"));
     attrwrapper_cls = new (0) BoxedHeapClass(object_cls, &AttrWrapper::gcHandler, 0, 0, sizeof(AttrWrapper), false,
                                              static_cast<BoxedString*>(boxString("attrwrapper")));
     dict_cls = new (0) BoxedHeapClass(object_cls, &dictGCHandler, 0, 0, sizeof(BoxedDict), false,
@@ -3446,6 +3478,10 @@ void setupRuntime() {
                                                  sizeof(AttrWrapperIter), false, "attrwrapperiter");
 
     // TODO: add explicit __get__ and __set__ methods to these
+    pyston_getset_cls->giveAttr("__get__", new BoxedFunction(boxRTFunction((void*)getsetGet, UNKNOWN, 3)));
+    capi_getset_cls->giveAttr("__get__", new BoxedFunction(boxRTFunction((void*)getsetGet, UNKNOWN, 3)));
+    pyston_getset_cls->giveAttr("__set__", new BoxedFunction(boxRTFunction((void*)getsetSet, UNKNOWN, 3)));
+    capi_getset_cls->giveAttr("__set__", new BoxedFunction(boxRTFunction((void*)getsetSet, UNKNOWN, 3)));
     pyston_getset_cls->freeze();
     capi_getset_cls->freeze();
 

test/CPYTHON_TEST_NOTES.md

@@ -17,12 +17,12 @@ The CPython tests I've included fail for various reasons. Recurring issues inclu
 ```
 FILE                    REASONS
 ------------------------------------------------------
-test_augassign          missing oldstyle-class __add__, __iadd__, etc
+test_abc                [unknown]
 test_bisect             somehow sys.modules['_bisect'] is getting set to 0
 test_builtin            execfile scoping issue
+test_class              needs ellipsis
 test_coercion           1**1L, divmod(1, 1L); some unknown bug
-test_collections        doctest (dies in inspect.getmodule())
-test_compare            "AssertionError: 2 != 0.0j"
+test_collections        assertion failed when doing vars(collections.namedtuple('Point', 'x y')(11, 12))
 test_complex            need complex.__nonzero__
 test_contextlib         lock.locked attributes
 test_datetime           needs _PyObject_GetDictPtr
@@ -30,31 +30,30 @@ test_decimal            I think we need to copy decimaltestdata from cpython
 test_decorators         decorator bug -- we evaluate decorator obj and its args in wrong order
 test_deque              couple unknown issues
 test_descr              wontfix: crashes at "self.__dict__ = self"
-test_descrtut           doctest (dies in inspect.getmodule())
-test_dict               need to handle repr of recursive structures (ie `d = {}; d['self'] = d; print d`)
+test_descrtut           `exec in DefaultDict()`
+test_dict               misc failures related to things like gc, abc, comparisons, detecting mutations during iterations
 test_dictcomps          we need to disallow assigning to dictcomps
 test_dictviews          various unique bugs
-test_doctest            doctest (dies in inspect.getmodule())
-test_doctest2           doctest (sys.displayhook)
+test_doctest            hard to know.  also missing some input files
 test_enumerate          assert instead of exception in BoxedEnumerate
 test_exceptions         we are missing recursion-depth checking
-test_extcall            doctest (syss.displayhook())
+test_extcall            f(**kw) crashes if kw isn't a dict
 test_file               wontfix: we don't destruct file objects when the test wants
 test_file2k             we abort when you try to open() a directory
 test_file_eintr         not sure
 test_float              float(long), a couple unknown things
-test_format             float(long)
 test_funcattrs          we don't allow changing numing of function defaults
 test_functools          unknown errors
-test_generators         doctest (sys.displayhook)
-test_genexps            doctest (sys.displayhook)
-test_getopt             doctest (sys.displayhook)
+test_generators         crash when sending non-None to a just-started generator
+test_genexps            parser not raising a SyntaxError when assigning to a genexp
 test_global             SyntaxWarnings for global statements after uses
 test_grammar            bug in our tokenizer
 test_hash               number of hash bugs (all representations of '1' don't have the same hash; empty string is supposed to have 0 hash, etc)
 test_index              slice.indices, old-styl-class __mul__
 test_int                we assert instead of throw exception
 test_io                 memory/gc issue?
+test_iterlen            [unknown]
+test_itertools          [unknown]
 test_json               'from test.script_helper import assert_python_ok' fails; sounds like it is trying to look at pycs
 test_list               longs as slice indices
 test_long               sys.long_info
@@ -66,28 +65,22 @@ test_optparse           assertion instead of exceptions for long("invalid number
 test_pep277             segfaults
 test_pep352             various unique bugs
 test_pkg                unknown bug
-test_pow                pow(3L, 3L, -8) fails
 test_random             long("invalid number")
 test_repr               complex.__hash__; some unknown issues
 test_richcmp            PyObject_Not
 test_scope              eval of code object from existing function (not currently supported)
 test_set                weird function-picking issue
-test_setcomps           doctest (sys.displayhook)
-test_sets               doctest (sys.displayhook)
-test_slice              segfault
+test_setcomps           parser not raising a SyntaxError when assigning to a setcomp
 test_sort               argument specification issue in listSort?
 test_str                memory leak?
 test_string             infinite loops in test_replace
 test_subprocess         exit code 141 [sigpipe?], no error message
-test_tuple              tuple features: ()*0L, tuple.count, tuple.__getslice__; "test_constructors" fails
 test_types              PyErr_WarnEx
-test_unary              objmodel.cpp: unaryop: Assertion `attr_func' failed: str.__pos__
 test_undocumented_details   function.func_closure
 test_unicode            argument passing issue?
 test_unicode_file       exit code 139, no error message
 test_unittest           serialize_ast assert
-test_unpack             doctest (sys.displayhook)
-test_urllib2            doctest (dies in inspect.getmodule())
+test_urllib2            segfault due to attrwrapper corruption
 test_userdict           segfault: repr of recursive dict?
 test_userlist           slice(1L, 1L)
 test_userstring         float(1L); hangs in test_replace

test/tests/classes.py

+class C(object):
+    pass
+class D:
+    pass
+for l in [type, set, int, list, tuple, C, D]:
+    print l, l.__module__
+
+print type.__dict__['__module__'].__get__(set, None)
+print type(type.__dict__['__module__'].__get__(None, set))

test/tests/compile_exec.py

@@ -30,3 +30,12 @@ print a, sorted(g.keys())
 print
 c = compile("a = 1; b = 2; a - b; b - a; c = a - b; c; print c", "test.py", "single")
 exec c
+
+exec compile("'hello world'", "test.py", "single")
+
+import sys
+def new_displayhook(arg):
+    print "my custom displayhook!"
+    print arg
+sys.displayhook = new_displayhook
+exec compile("'hello world'", "test.py", "single")

test/tests/doctest_test.py

-# expected: fail
 # requires:
 # - code.co_firstlineno
 # - sys.dysplayhook