Commit 32859fc4 authored by Sebastien Robin's avatar Sebastien Robin

task distribution: avoid test nodes stuck with test suite having failing runTestSuite

parent d2dd961f
...@@ -493,6 +493,12 @@ class TestTaskDistribution(ERP5TypeTestCase): ...@@ -493,6 +493,12 @@ class TestTaskDistribution(ERP5TypeTestCase):
['testFailing', 'testSlow', 'testFast'], ['testFailing', 'testSlow', 'testFast'],
[self.tool.startUnitTest(test_result_path)[1] for _ in range(3)]) [self.tool.startUnitTest(test_result_path)[1] for _ in range(3)])
def checkTestResultLine(self, test_result, expected):
line_list = test_result.objectValues(portal_type="Test Result Line")
found_list = [(x.getTitle(), x.getSimulationState()) for x in line_list]
found_list.sort(key=lambda x: x[0])
self.assertEqual(expected, found_list)
def test_06b_restartStuckTest(self): def test_06b_restartStuckTest(self):
""" """
Check if a test result line is not stuck in 'started', if so, redraft Check if a test result line is not stuck in 'started', if so, redraft
...@@ -503,18 +509,13 @@ class TestTaskDistribution(ERP5TypeTestCase): ...@@ -503,18 +509,13 @@ class TestTaskDistribution(ERP5TypeTestCase):
test_result = self.portal.unrestrictedTraverse(test_result_path) test_result = self.portal.unrestrictedTraverse(test_result_path)
line_url, _ = self.tool.startUnitTest(test_result_path) line_url, _ = self.tool.startUnitTest(test_result_path)
now = DateTime() now = DateTime()
def checkTestResultLine(expected): self.checkTestResultLine(test_result, [('testBar', 'started'), ('testFoo', 'draft')])
line_list = test_result.objectValues(portal_type="Test Result Line")
found_list = [(x.getTitle(), x.getSimulationState()) for x in line_list]
found_list.sort(key=lambda x: x[0])
self.assertEqual(expected, found_list)
checkTestResultLine([('testBar', 'started'), ('testFoo', 'draft')])
self._callRestartStuckTestResultAlarm() self._callRestartStuckTestResultAlarm()
checkTestResultLine([('testBar', 'started'), ('testFoo', 'draft')]) self.checkTestResultLine(test_result, [('testBar', 'started'), ('testFoo', 'draft')])
line_url, _ = self.tool.startUnitTest(test_result_path) line_url, _ = self.tool.startUnitTest(test_result_path)
checkTestResultLine([('testBar', 'started'), ('testFoo', 'started')]) self.checkTestResultLine(test_result, [('testBar', 'started'), ('testFoo', 'started')])
self._callRestartStuckTestResultAlarm() self._callRestartStuckTestResultAlarm()
checkTestResultLine([('testBar', 'started'), ('testFoo', 'started')]) self.checkTestResultLine(test_result, [('testBar', 'started'), ('testFoo', 'started')])
# now let change history to do like if a test result line was started # now let change history to do like if a test result line was started
# a long time ago # a long time ago
line = self.portal.restrictedTraverse(line_url) line = self.portal.restrictedTraverse(line_url)
...@@ -522,9 +523,9 @@ class TestTaskDistribution(ERP5TypeTestCase): ...@@ -522,9 +523,9 @@ class TestTaskDistribution(ERP5TypeTestCase):
if history_line['action'] == 'start': if history_line['action'] == 'start':
history_line['time'] = now - 1 history_line['time'] = now - 1
self._callRestartStuckTestResultAlarm() self._callRestartStuckTestResultAlarm()
checkTestResultLine([('testBar', 'started'), ('testFoo', 'draft')]) self.checkTestResultLine(test_result, [('testBar', 'started'), ('testFoo', 'draft')])
self.tool.stopUnitTest(line_url, {}) self.tool.stopUnitTest(line_url, {})
checkTestResultLine([('testBar', 'started'), ('testFoo', 'stopped')]) self.checkTestResultLine(test_result, [('testBar', 'started'), ('testFoo', 'stopped')])
def test_07_reportTaskFailure(self): def test_07_reportTaskFailure(self):
""" """
...@@ -594,6 +595,50 @@ class TestTaskDistribution(ERP5TypeTestCase): ...@@ -594,6 +595,50 @@ class TestTaskDistribution(ERP5TypeTestCase):
finally: finally:
self.unpinDateTime() self.unpinDateTime()
def test_07c_reportTaskFailureWithRedraftedTestResultLine(self):
"""
Similar to above test.
But on the other hand, if a test result line is started many times (due to
automatic redraft), then this might just means we have issue of runTestSuite unable
to finish tests, or we might have just tests that can never be executed within timeout time.
In such case, it's better to mark test result as failed to give a chance to other test
suites to be executed
"""
now = DateTime()
try:
self.pinDateTime(now - 1.0/24*8)
test_result_path, _ = self._createTestResult(test_list=['testFoo'])
test_result = self.getPortalObject().unrestrictedTraverse(test_result_path)
self.checkTestResultLine(test_result, [('testFoo', 'draft')])
self.assertEqual("started", test_result.getSimulationState())
node, = test_result.objectValues(portal_type="Test Result Node",
sort_on=[("title", "ascending")])
self.assertEqual("started", node.getSimulationState())
self.tool.startUnitTest(test_result_path)
self.checkTestResultLine(test_result, [('testFoo', 'started')])
# We have a failure but with recent activities on tests
self.pinDateTime(now - 1.0/24*7.5)
self.tool.reportTaskFailure(test_result_path, {}, "Node0")
self.assertEqual("failed", node.getSimulationState())
self.assertEqual("started", test_result.getSimulationState())
self.checkTestResultLine(test_result, [('testFoo', 'started')])
# some hours later, test line is redrafted
self.pinDateTime(now - 1.0/24*3)
self._callRestartStuckTestResultAlarm()
self.checkTestResultLine(test_result, [('testFoo', 'draft')])
# Test is then relaunched
self.tool.startUnitTest(test_result_path)
self.checkTestResultLine(test_result, [('testFoo', 'started')])
# We have another failure but remains only test result line that was already
# redrafted, so we have to mark the test result as failed
self.pinDateTime(now - 1.0/24*2.5)
self.tool.reportTaskFailure(test_result_path, {}, "Node0")
self.assertEqual("failed", node.getSimulationState())
self.assertEqual("failed", test_result.getSimulationState())
finally:
self.unpinDateTime()
def test_08_checkWeCanNotCreateTwoTestResultInParallel(self): def test_08_checkWeCanNotCreateTwoTestResultInParallel(self):
""" """
To avoid duplicates of test result when several testnodes works on the To avoid duplicates of test result when several testnodes works on the
......
...@@ -285,7 +285,13 @@ class TaskDistributionTool(BaseTool): ...@@ -285,7 +285,13 @@ class TaskDistributionTool(BaseTool):
for test_result_line in test_result.objectValues( for test_result_line in test_result.objectValues(
portal_type="Test Result Line"): portal_type="Test Result Line"):
if test_result_line.getModificationDate() > recent_time: if test_result_line.getModificationDate() > recent_time:
break # do not take into account redrafted lines, this means we already
# add issues with them
if len([x for x in portal.portal_workflow.getInfoFor(
ob=test_result_line,
name='history',
wf_id='test_result_workflow') if x['action']=='redraft']) == 0:
break
else: else:
if test_result.getSimulationState() not in ('failed', 'cancelled'): if test_result.getSimulationState() not in ('failed', 'cancelled'):
test_result.fail() test_result.fail()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment