Commit 321fe20a authored by Denis Bilenko's avatar Denis Bilenko

testrunner.py: report unexpected passes

parent 8fd25a87
...@@ -22,6 +22,7 @@ def run_many(tests, expected=None, failfast=False): ...@@ -22,6 +22,7 @@ def run_many(tests, expected=None, failfast=False):
start = time.time() start = time.time()
total = 0 total = 0
failed = {} failed = {}
passed = {}
NWORKERS = min(len(tests), NWORKERS) or 1 NWORKERS = min(len(tests), NWORKERS) or 1
pool = ThreadPool(NWORKERS) pool = ThreadPool(NWORKERS)
...@@ -33,6 +34,8 @@ def run_many(tests, expected=None, failfast=False): ...@@ -33,6 +34,8 @@ def run_many(tests, expected=None, failfast=False):
if failfast: if failfast:
sys.exit(1) sys.exit(1)
failed[result.name] = [cmd, kwargs] failed[result.name] = [cmd, kwargs]
else:
passed[result.name] = True
results = [] results = []
...@@ -73,7 +76,7 @@ def run_many(tests, expected=None, failfast=False): ...@@ -73,7 +76,7 @@ def run_many(tests, expected=None, failfast=False):
reap_all() reap_all()
except KeyboardInterrupt: except KeyboardInterrupt:
pool.terminate() pool.terminate()
report(total, failed, exit=False, took=time.time() - start, expected=expected) report(total, failed, passed, exit=False, took=time.time() - start, expected=expected)
log('(partial results)\n') log('(partial results)\n')
raise raise
except: except:
...@@ -82,7 +85,7 @@ def run_many(tests, expected=None, failfast=False): ...@@ -82,7 +85,7 @@ def run_many(tests, expected=None, failfast=False):
raise raise
reap_all() reap_all()
util.report(total, failed, took=time.time() - start, expected=expected) report(total, failed, passed, took=time.time() - start, expected=expected)
def discover(tests=None, ignore=None): def discover(tests=None, ignore=None):
...@@ -127,9 +130,18 @@ def load_list_from_file(filename): ...@@ -127,9 +130,18 @@ def load_list_from_file(filename):
return result return result
def matches(expected, command): def might_fail(expected, command):
for line in expected:
if command.endswith(' ' + line.replace('FLAKY ', '')):
return True
return False
def must_fail(expected, command):
for line in expected: for line in expected:
if command.endswith(' ' + line): if 'FLAKY' in line:
continue
if command.endswith(' ' + line.replace('FLAKY ', '')):
return True return True
return False return False
...@@ -143,7 +155,7 @@ def format_seconds(seconds): ...@@ -143,7 +155,7 @@ def format_seconds(seconds):
return seconds return seconds
def report(total, failed, exit=True, took=None, expected=None): def report(total, failed, passed, exit=True, took=None, expected=None):
runtimelog = util.runtimelog runtimelog = util.runtimelog
if runtimelog: if runtimelog:
log('\nLongest-running tests:') log('\nLongest-running tests:')
...@@ -159,34 +171,49 @@ def report(total, failed, exit=True, took=None, expected=None): ...@@ -159,34 +171,49 @@ def report(total, failed, exit=True, took=None, expected=None):
failed_expected = [] failed_expected = []
failed_unexpected = [] failed_unexpected = []
passed_unexpected = []
for name in passed:
if must_fail(expected, name):
passed_unexpected.append(name)
if passed_unexpected:
log('\n%s/%s unexpected passes', len(passed_unexpected), total)
print_list(passed_unexpected)
if failed: if failed:
log('\n%s/%s tests failed%s', len(failed), total, took) log('\n%s/%s tests failed%s', len(failed), total, took)
expected = set(expected or []) expected = set(expected or [])
for name in failed: for name in failed:
if matches(expected, name): if might_fail(expected, name):
failed_expected.append(name) failed_expected.append(name)
else: else:
failed_unexpected.append(name) failed_unexpected.append(name)
if failed_expected: if failed_expected:
log('\n%s/%s expected failures', len(failed_expected), total) log('\n%s/%s expected failures', len(failed_expected), total)
for name in failed_expected: print_list(failed_expected)
log(' - %s', name)
if failed_unexpected: if failed_unexpected:
log('\n%s/%s unexpected failures', len(failed_unexpected), total) log('\n%s/%s unexpected failures', len(failed_unexpected), total)
for name in failed_unexpected: print_list(failed_unexpected)
log(' - %s', name)
else: else:
log('\n%s tests passed%s', total, took) log('\n%s tests passed%s', total, took)
if exit: if exit:
if failed_unexpected: if failed_unexpected:
sys.exit(min(100, len(failed_unexpected))) sys.exit(min(100, len(failed_unexpected)))
if passed_unexpected:
sys.exit(101)
if total <= 0: if total <= 0:
sys.exit('No tests found.') sys.exit('No tests found.')
def print_list(lst):
for name in lst:
log(' - %s', name)
def main(): def main():
import optparse import optparse
parser = optparse.OptionParser() parser = optparse.OptionParser()
......
# This is a list of known failures (=bugs). # This is a list of known failures (=bugs).
# The tests listed there must fail (or testrunner.py will report error) unless they are prefixed with FLAKY
# in which cases the result of them is simply ignored
import os import os
import sys import sys
...@@ -9,7 +11,7 @@ PYPY = hasattr(sys, 'pypy_version_info') ...@@ -9,7 +11,7 @@ PYPY = hasattr(sys, 'pypy_version_info')
FAILING_TESTS = [ FAILING_TESTS = [
# needs investigating # needs investigating
'test__issue6.py', 'FLAKY test__issue6.py',
# bunch of SSLError: [Errno 1] _ssl.c:504: error:14090086:SSL routines:SSL3_GET_SERVER_CERTIFICATE:certificate verify failed # bunch of SSLError: [Errno 1] _ssl.c:504: error:14090086:SSL routines:SSL3_GET_SERVER_CERTIFICATE:certificate verify failed
# seems to be Python/OpenSSL problem, not gevent's # seems to be Python/OpenSSL problem, not gevent's
...@@ -21,8 +23,8 @@ FAILING_TESTS = [ ...@@ -21,8 +23,8 @@ FAILING_TESTS = [
if os.environ.get('GEVENT_RESOLVER') == 'ares': if os.environ.get('GEVENT_RESOLVER') == 'ares':
# XXX fix this # XXX fix this
FAILING_TESTS += [ FAILING_TESTS += [
'test__socket_dns.py', 'FLAKY test__socket_dns.py',
'test__socket_dns6.py', 'FLAKY test__socket_dns6.py',
] ]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment