Commit 435dee38 authored by Jérome Perrin's avatar Jérome Perrin

test_result: Retry failed test result lines

If a test result finish and its title match the test suite pattern allowed to
restart defined on test suite, it will be retried at most once.

Note about the test: The current behavior is to use PASS/FAIL for test result
and PASSED/FAILED for test result line. The test checks the actual values, if
we ever fix this inconsistency, we can safely update the test I believe.
parent f1c6e03d
......@@ -1418,6 +1418,90 @@ class TestTaskDistribution(TaskDistributionTestCase):
self.assertEqual(None, test_suite.getAlarmDate())
class TestRetryFailedTest(TaskDistributionTestCase):
"""Test how failed tests can be automatically retried.
"""
def afterSetUp(self):
super(TestRetryFailedTest, self).afterSetUp()
self.test_suite, = self.test_suite_module.objectValues()
self._createTestNode()
self.tic()
def test_failed_test_not_retried_by_default(self):
test_result_path, _ = self._createTestResult(test_list=['testFoo', ])
test_result = self.portal.unrestrictedTraverse(test_result_path)
line_url, _ = self.tool.startUnitTest(test_result_path)
test_result_line = self.portal.restrictedTraverse(line_url)
status_dict = {
'test_count': 100,
'error_count': 2,
'failure_count': 3,
}
self.tool.stopUnitTest(line_url, status_dict)
self.tic()
self.assertEqual(test_result_line.getStringIndex(), 'FAILED')
self.assertEqual(test_result_line.getSimulationState(), 'stopped')
self.assertEqual(test_result.getStringIndex(), 'FAIL')
self.assertEqual(test_result.getSimulationState(), 'stopped')
def test_failed_retried_once_then_fail(self):
self.test_suite.setRetryTestPattern('testF.*')
test_result_path, _ = self._createTestResult(test_list=['testFoo', ])
test_result = self.portal.unrestrictedTraverse(test_result_path)
line_url, _ = self.tool.startUnitTest(test_result_path)
test_result_line = self.portal.restrictedTraverse(line_url)
status_dict = {
'test_count': 100,
'error_count': 2,
'failure_count': 3,
}
self.tool.stopUnitTest(line_url, status_dict)
self.tic()
# test failed, but it will be retried
self.assertEqual(test_result_line.getStringIndex(), 'RETRYING')
self.assertEqual(test_result_line.getSimulationState(), 'draft')
# if it fails again ...
self.tool.stopUnitTest(line_url, status_dict)
self.tic()
# ... the test result will be fail.
self.assertEqual(test_result_line.getStringIndex(), 'FAILED')
self.assertEqual(test_result_line.getSimulationState(), 'stopped')
self.assertEqual(test_result.getStringIndex(), 'FAIL')
self.assertEqual(test_result.getSimulationState(), 'stopped')
self.assertEqual(test_result.getProperty('errors'), 2)
self.assertEqual(test_result.getProperty('failures'), 3)
self.assertEqual(test_result.getProperty('test_result_retry_count'), 1)
def test_failed_retried_once_then_pass(self):
self.test_suite.setRetryTestPattern('testF.*')
test_result_path, _ = self._createTestResult(test_list=['testFoo', ])
test_result = self.portal.unrestrictedTraverse(test_result_path)
line_url, _ = self.tool.startUnitTest(test_result_path)
test_result_line = self.portal.restrictedTraverse(line_url)
status_dict = {
'test_count': 100,
'error_count': 2,
'failure_count': 3,
}
self.tool.stopUnitTest(line_url, status_dict)
self.tic()
# test failed, but it will be retried
self.assertEqual(test_result_line.getStringIndex(), 'RETRYING')
self.assertEqual(test_result_line.getSimulationState(), 'draft')
# if it succeed next time ...
status_dict['error_count'] = 0
status_dict['failure_count'] = 0
self.tool.stopUnitTest(line_url, status_dict)
self.tic()
# ... the test result will be successful.
self.assertEqual(test_result_line.getStringIndex(), 'PASSED')
self.assertEqual(test_result_line.getSimulationState(), 'stopped')
self.assertEqual(test_result.getStringIndex(), 'PASS')
self.assertEqual(test_result.getSimulationState(), 'stopped')
self.assertEqual(test_result.getProperty('test_result_retry_count'), 1)
class TestGitlabRESTConnectorInterface(ERP5TypeTestCase):
"""Tests for Gitlab commits annotations.
"""
......
import re
test_result = sci['object']
kw = sci['kwargs']
test_result.setStopDate(kw.get('date') or DateTime())
......@@ -8,6 +9,30 @@ def unexpected(test_result):
# passed if there's no unexpected failures.
return test_result.getSourceProjectTitle() != "NEO R&D"
def shouldRetry(test_result_line):
# type: (erp5.portal_type.TestResultLine,) -> bool
"""Should the test result line be retried ?
We retry test result line once for tests matching pattern defined on test suite.
Unless if there's already another failed test result line, in that case we don't retry.
"""
if test_result_line.getProperty('test_result_retry_count') or 0:
return False
test_result = test_result_line.getParentValue()
for other_test_result_line in test_result.contentValues(portal_type='Test Result Line'):
if test_result_line != other_test_result_line and other_test_result_line.getStringIndex() in ('UNKNOWN', 'FAILED'):
return False
test_suite_data = test_result.TestResult_getTestSuiteData()
if not test_suite_data:
return False
if not test_suite_data['retry_test_pattern']:
return False
return re.search(test_suite_data['retry_test_pattern'], test_result_line.getTitle() or '')
if test_result.getPortalType() == 'Test Result':
has_unknown_result = False
edit_kw = dict(duration=0,
......@@ -48,12 +73,12 @@ elif test_result.getPortalType() == 'Test Result Line':
duration = kw.get('duration')
if duration is None:
duration = (test_result.getStopDate() - test_result.getStartDate()) * (24*60*60)
cmdline = kw.get('command', getattr(test_result, 'cmdline', ''))
cmdline = kw.get('command', '')
if same_type(cmdline, []):
cmdline = ' '.join(map(repr, cmdline))
stdout = kw.get('stdout', getattr(test_result, 'stdout', ''))
stderr = kw.get('stderr', getattr(test_result, 'stderr', ''))
html_test_result = kw.get('html_test_result', getattr(test_result, 'html_test_result', ''))
stdout = kw.get('stdout', '')
stderr = kw.get('stderr', '')
html_test_result = kw.get('html_test_result', '')
test_result.edit(cmdline=cmdline,
stdout=stdout,
stderr=stderr,
......@@ -64,5 +89,11 @@ elif test_result.getPortalType() == 'Test Result Line':
failures=failures,
skips=skips,
html_test_result=html_test_result)
if status == 'FAILED' and shouldRetry(test_result):
test_result.edit(
test_result_retry_count=1 + (test_result.getProperty('test_result_retry_count') or 0),
string_index='RETRYING',
)
test_result.redraft(comment="Retried after a first failure")
else:
raise NotImplementedError("unknown type : %r" % test_result.getPortalType())
......@@ -32,6 +32,7 @@
<string>cancel_action</string>
<string>fail</string>
<string>publish_stopped</string>
<string>redraft</string>
</tuple>
</value>
</item>
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment