Commit 1561b3a0 authored by Kirill Smelkov's avatar Kirill Smelkov

Clarify meaning of error vs failure

failure is assertion failure.
error is any other error.

Source: https://docs.python.org/3/library/unittest.html#organizing-test-code
-> "If the test fails, an exception will be raised ... identify the test
case as a failure. Any other exceptions will be treated as errors"

While searching ERP5 code for failure_count to clarify its meaning, I've
also discovered expected_failures and unexpected_successes (*) so they
are also added to the table.

(*) see e.g. https://lab.nexedi.com/nexedi/erp5/blob/1e31c091/product/ERP5Type/tests/ERP5TypeTestSuite.py#L6-8
parent 74a7853c
...@@ -267,10 +267,16 @@ class LocalTestResultLine: ...@@ -267,10 +267,16 @@ class LocalTestResultLine:
_ = v('error_count') _ = v('error_count')
if _ == '?': if _ == '?':
st = '?' st = '?'
elif _ == 0: elif _ != 0:
st = 'ok' st = 'error'
else: else:
_ = v('failure_count')
if _ == '?':
st = '?'
elif _ != 0:
st = 'fail' st = 'fail'
else:
st = 'ok'
print('%s\t%s\t%.3fs\t# %st %se %sf %ss' % (st, self.name, kw['duration'], v('test_count'), v('error_count'), v('failure_count'), v('skip_count'))) print('%s\t%s\t%.3fs\t# %st %se %sf %ss' % (st, self.name, kw['duration'], v('test_count'), v('error_count'), v('failure_count'), v('skip_count')))
# XXX + dump .json ? # XXX + dump .json ?
...@@ -290,17 +296,19 @@ class PyTest: ...@@ -290,17 +296,19 @@ class PyTest:
return default return default
return int(m.group(1)) return int(m.group(1))
stat = {} stat = {'test_count': 0}
def stat_set(stat_key, from_name): def stat_set(stat_key, from_name):
v = get(from_name) v = get(from_name)
if v is None: if v is None:
return return
stat[stat_key] = v stat[stat_key] = v
stat['test_count'] += v
stat_set('skip_count', 'skipped') stat_set('skip_count', 'skipped')
stat_set('error_count', 'failed') stat_set('failure_count', 'failed')
stat_set('failure_count', 'xfailed') # XXX ok? stat_set('expected_failures', 'xfailed')
npass = get('passed', 0) stat_set('unexpected_successes', 'xpassed')
stat['test_count'] = npass + stat.get('failure_count', 0) + stat.get('skip_count', 0) + stat.get('error_count', 0) # don't set error_count - everything goes to failure_count
stat['test_count'] += get('passed', 0)
return stat return stat
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment