Merge pull request #344 from rntz/tester-explicit-failure

Add should_error directive, for tests that should exit non-0

src/runtime/import.cpp

@@ -149,6 +149,7 @@ static Box* importSub(const std::string& name, const std::string& full_name, Box
         return module;
     }
 
+    // TODO: these are a crude hack to help our tests, find a better way
     if (name == "basic_test")
         return importTestExtension("basic_test");
     if (name == "descr_test")

test/tests/21.py

+# should_error
+
 # Expected failure: even though we could copy the "definedness" information in the assignment,
 # that's actually an error.
 

test/tests/22.py

+# should_error
 # Expected failure: stacktrace testing
-
 print 5 % 0

test/tests/23.py

+# should_error
 not_defined

test/tests/24.py

+# should_error
 # Expected failure: adding things that can't be added
 
 1 + ""

test/tests/39.py

+# should_error
 int.a = 1

test/tests/40.py

+# should_error
 i = 0
 i.a = 0

test/tests/46.py

@@ -15,4 +15,7 @@ print C.b
 print C.a
 print o.c
 print o.b
-print o.a # this should error
+try:
+    print o.a
+except AttributeError, e:
+    print e

test/tests/48.py

+# should_error
 # This should raise a python level error, not an assertion in the compiler
 
 x = 1

test/tests/49.py

 # This should raise a python level error, not an assertion in the compiler
 
-print int.doesnt_exist
+try:
+    print int.doesnt_exist
+except AttributeError, e:
+    print e

test/tests/60.py

@@ -22,5 +22,11 @@ class C(object):
         return self.n
 
 print len(C(1))
-print len(1)
-print len(C("hello world"))
+try:
+    print len(1)
+except TypeError, e:
+    print e
+try:
+    print len(C("hello world"))
+except TypeError, e:
+    print e

test/tests/66.py

+# should_error
 # Make sure that overriding __file__ doesn't change the traceback
 # TODO the tester doesn't currently check the traceback
 

test/tests/69.py

+# should_error
 class C(object):
     pass
 

test/tests/75.py

+# should_error
 def f():
     if 0:
         str = 0

test/tests/assert.py

+# should_error
 def msg():
     print "msg()"
     return "failure message"

test/tests/augassign_analysis.py

@@ -114,4 +114,7 @@ def f10():
     # This should error: the lhs is evaluated first
     x += [x for x in xrange(5)][0]
     print x
-f10()
+try:
+    f10()
+except UnboundLocalError, e:
+    print e

test/tests/augassign_reverse.py

@@ -9,5 +9,8 @@ def f():
     # Augassigns can change the type of the variable:
     i += IntLike()
     print i
-    i + 1
+    try:
+        i + 1
+    except TypeError, e:
+        print e
 f()

test/tests/basestring.py

@@ -4,4 +4,7 @@ print isinstance(3, basestring)
 print basestring.__doc__
 
 # should raise an exception
-t = basestring.__new__(basestring)
+try:
+    t = basestring.__new__(basestring)
+except TypeError, e:
+    print e

test/tests/break_outside_loop.py

+# should_error
 break

test/tests/capi_slots.py

@@ -157,4 +157,7 @@ try:
 except SystemError, e:
     print e
 
-print c.foo()
+try:
+    print c.foo()
+except SystemError, e:
+    print e

test/tests/class_noctor.py

+# should_error
 # skip-if: sys.version_info < (2, 7, 4)
 # - Error message changed in 2.7.4
 

test/tests/closure_name_error.py

@@ -6,5 +6,7 @@ def bad_addr3(_x):
     def g(y3):
         return x3 + y3
     return g
-print bad_addr3(1)(2)
-
+try:
+    print bad_addr3(1)(2)
+except NameError, e:
+    print e

test/tests/continue_outside_loop.py

+# should_error
 continue

test/tests/control_flow.py

@@ -5,4 +5,7 @@ def _escape():
     if code:
         return code
 
-_escape()
+try:
+    _escape()
+except NameError, e:
+    print e

test/tests/data_descriptors.py

@@ -34,8 +34,8 @@ C.ndd = 7
 #TODO it would be nice to print these out (once __dict__ is implemented)
 #print C.__dict__['dd']
 #print C.__dict__['ndd']
-print c.dd
-print c.ndd
+print C.dd
+print C.ndd
 
 # Repeat all of the above for subclasses of the descriptors
 

test/tests/del_name_scoping.py

@@ -4,4 +4,7 @@ def f():
         # the del marks 'x' as a name written to in this scope
         del x
     print x
-f()
+try:
+    f()
+except NameError, e:
+    print e

test/tests/exceptions_nonreprable.py

+# should_error
 class BadException(Exception):
     def __str__(self):
         print "str"
@@ -7,7 +8,7 @@ try:
     # This will raise:
     print BadException()
     assert 0
-except NotImplementedError:
-    pass
+except NotImplementedError, e:
+    print e
 
 raise BadException()

test/tests/expr_evaluation_order.py

@@ -18,4 +18,7 @@ def f2(y):
         return x
     return y
 
-print g2(f2(2))
+try:
+    print g2(f2(2))
+except NameError, e:
+    print e

test/tests/future_non_existent.py

+# should_error
 from __future__ import rvalue_references # should cause syntax error

test/tests/future_not_at_beginning.py

+# should_error
+
 "docstring"
 
 "not a docstring"

test/tests/future_not_at_beginning2.py

+# should_error
+
 "docstring"
 
 def f():

test/tests/getattr.py

@@ -2,4 +2,7 @@ l = range(5)
 print getattr(l, "pop")()
 
 print getattr([], "a", "default")
-print getattr([], "a")
+try:
+    print getattr([], "a")
+except AttributeError, e:
+    print e

test/tests/getattr_nonstring.py

+# should_error
 print getattr([], [])

test/tests/getitem.py

@@ -21,4 +21,7 @@ c = C()
 c.__getitem__ = gi
 print c[1]
 
-print 1[1]
+try:
+    print 1[1]
+except TypeError, e:
+    print e

test/tests/globals.py

@@ -4,7 +4,10 @@ def f():
     print True # builtin, redefined
     print False # builtin, not redefined
     print z # local
-    print y # non-builtin, not defined
+    try:
+        print y # non-builtin, not defined
+    except NameError, e:
+        print e
 
 x = 2
 z = 2

test/tests/init_must_return_none.py

+# should_error
 # As the test filename says, init functions must return None.
 # This file tests that; it also makes sure that it gets tested
 # when in a patchpoint.

test/tests/listcomp_assigning.py

@@ -40,5 +40,7 @@ def f2(n, b):
 f2(5, 0)
 print i
 f2(0, 1)
-f2(0, 0)
-
+try:
+    f2(0, 0)
+except UnboundLocalError, e:
+    print e

test/tests/object_new_arguments.py

+# should_error
 # skip-if: sys.version_info < (2, 7, 4)
 # - Error message changed in 2.7.4
 

test/tests/resource_test.py

@@ -5,7 +5,7 @@ for k in sorted(dir(resource)):
         continue
     print k, getattr(resource, k)
 
-TIME_LIMIT = 100
+TIME_LIMIT = 5
 resource.setrlimit(resource.RLIMIT_CPU, (TIME_LIMIT + 1, TIME_LIMIT + 1))
 
 MAX_MEM_MB = 100

test/tests/return_in_class.py

+# should_error
 class C(object):
     return

test/tests/return_outside_func.py

+# should_error
 return

test/tests/setitem.py

@@ -17,4 +17,7 @@ def si(k, v):
 c.__setitem__ = si
 c[3] = 4
 
-1[2] = 3
+try:
+    1[2] = 3
+except TypeError, e:
+    print e

test/tests/sys_excinfo.py

+# should_error
 # Different ways of nesting exceptions
 
 import sys

test/tests/tuple_unpacking_invalid.py

+# should_error
 # Int not iterable:
 a, b, c = 1

test/tests/varargs_protocol.py

@@ -17,4 +17,7 @@ class C(object):
 def f(a, b, c):
     print a, b, c
 
-f(*C())
+try:
+    f(*C())
+except TypeError, e:
+    print e

tools/tester.py

@@ -126,6 +126,7 @@ def run_test(fn, check_stats, run_memcheck):
     jit_args = ["-rq"] + EXTRA_JIT_ARGS
     collect_stats = True
     expected = "success"
+    should_error = False
     allow_warnings = []
     for l in open(fn):
         l = l.strip()
@@ -141,6 +142,8 @@ def run_test(fn, check_stats, run_memcheck):
             jit_args += l
         elif l.startswith("# expected:"):
             expected = l[len("# expected:"):].strip()
+        elif l.startswith("# should_error"):
+            should_error = True
         elif l.startswith("# fail-if:"):
             condition = l.split(':', 1)[1].strip()
             if eval(condition):
@@ -216,13 +219,29 @@ def run_test(fn, check_stats, run_memcheck):
         if expected == "fail":
             r += "    Expected failure (got code %d, should be %d)" % (code, expected_code)
             return r
+        elif KEEP_GOING:
+            r += "    \033[%dmFAILED\033[0m (%s)" % (color, msg)
+            failed.append(fn)
+            return r
         else:
-            if KEEP_GOING:
-                r += "    \033[%dmFAILED\033[0m (%s)" % (color, msg)
-                failed.append(fn)
-                return r
-            else:
-                raise Exception("%s\n%s\n%s" % (msg, err, stderr))
+            raise Exception("%s\n%s\n%s" % (msg, err, stderr))
+
+    elif should_error == (code == 0):
+        color = 31              # red
+        if code == 0:
+            msg = "Exited successfully; remove '# should_error' if this is expected"
+        else:
+            msg = "Exited with code %d; add '# should_error' if this is expected" % code
+
+        if KEEP_GOING:
+            r += "    \033[%dmFAILED\033[0m (%s)" % (color, msg)
+            failed.append(fn)
+            return r
+        else:
+            # show last line of stderr so we have some idea went wrong
+            print "Last line of stderr: " + last_stderr_line
+            raise Exception(msg)
+
     elif out != expected_out:
         if expected == "fail":
             r += "    Expected failure (bad output)"