Commit 820e661b authored by Sebastien Robin's avatar Sebastien Robin

stop using a local cache on TaskDistributionTool

A dictionary was used as cached to store test result path and
duration of tests. This was quite convenient, however this
dictionary was growing, and since this data was commited thouzands
time per day, this was making the zodb growing faster and faster.

So we now rely on catalog + tags to retrieve test results. We
also use int_index on test result line to order them by duration
time. We will surely increase the load of sql database (we might
need to add specific index), but this will avoid zodb using too
much space.
parent 0e571301
...@@ -48,11 +48,6 @@ class TaskDistributionTool(BaseTool): ...@@ -48,11 +48,6 @@ class TaskDistributionTool(BaseTool):
def __init__(self, *args, **kw): def __init__(self, *args, **kw):
BaseTool.__init__(self, *args, **kw) BaseTool.__init__(self, *args, **kw)
# XXX Cache information about running test results, because the protocol
# is synchronous and we can't rely on the catalog.
# This is a hack until a better (and asynchronous) protocol is
# implemented.
self.test_result_dict = {}
security.declarePublic('getProtocolRevision') security.declarePublic('getProtocolRevision')
def getProtocolRevision(self): def getProtocolRevision(self):
...@@ -89,9 +84,19 @@ class TaskDistributionTool(BaseTool): ...@@ -89,9 +84,19 @@ class TaskDistributionTool(BaseTool):
portal = self.getPortalObject() portal = self.getPortalObject()
if test_title is None: if test_title is None:
test_title = name test_title = name
test_result_path, line_dict = self.test_result_dict.get( tag = "test_result_creation_%s" % test_title
test_title, ('', {})) # If there is already pending creation, we raise error, the testnode
duration_dict = {} # will automatically retry later
assert portal.portal_activities.countMessageWithTag(tag) == 0, \
"There is already a test result under creation"
test_result_path = None
result_list = portal.test_result_module.searchFolder(
portal_type="Test Result",
simulation_state="started",
title=test_title,
sort_on=[("creation_date","descending")])
if len(result_list):
test_result_path = result_list[0].getRelativeUrl()
def createNode(test_result, node_title): def createNode(test_result, node_title):
if node_title is not None: if node_title is not None:
node = self._getTestResultNode(test_result, node_title) node = self._getTestResultNode(test_result, node_title)
...@@ -99,7 +104,8 @@ class TaskDistributionTool(BaseTool): ...@@ -99,7 +104,8 @@ class TaskDistributionTool(BaseTool):
node = test_result.newContent(portal_type='Test Result Node', node = test_result.newContent(portal_type='Test Result Node',
title=node_title) title=node_title)
node.start() node.start()
def createTestResultLineList(test_result, test_name_list, line_dict): def createTestResultLineList(test_result, test_name_list):
duration_list = []
previous_test_result_list = portal.test_result_module.searchFolder( previous_test_result_list = portal.test_result_module.searchFolder(
title='=%s' % test_result.getTitle(), title='=%s' % test_result.getTitle(),
sort_on=[('creation_date','descending')], sort_on=[('creation_date','descending')],
...@@ -109,11 +115,19 @@ class TaskDistributionTool(BaseTool): ...@@ -109,11 +115,19 @@ class TaskDistributionTool(BaseTool):
previous_test_result = previous_test_result_list[0].getObject() previous_test_result = previous_test_result_list[0].getObject()
for line in previous_test_result.objectValues(): for line in previous_test_result.objectValues():
if line.getSimulationState() == 'stopped': if line.getSimulationState() == 'stopped':
duration_dict[line.getTitle()] = line.getProperty('duration') duration_list.append((line.getTitle(),line.getProperty('duration')))
duration_list.sort(key=lambda x: -x[1])
sorted_test_list = [x[0] for x in duration_list]
for test_name in test_name_list: for test_name in test_name_list:
index = 0
if sorted_test_list:
try:
index = sorted_test_list.index(test_name)
except ValueError:
pass
line = test_result.newContent(portal_type='Test Result Line', line = test_result.newContent(portal_type='Test Result Line',
title=test_name) title=test_name,
line_dict[line.getId()] = duration_dict.get(test_name) int_index=index)
reference_list_string = None reference_list_string = None
if type(revision) is str and '=' in revision: if type(revision) is str and '=' in revision:
reference_list_string = revision reference_list_string = revision
...@@ -129,8 +143,7 @@ class TaskDistributionTool(BaseTool): ...@@ -129,8 +143,7 @@ class TaskDistributionTool(BaseTool):
test_result = portal.unrestrictedTraverse(test_result_path, None) test_result = portal.unrestrictedTraverse(test_result_path, None)
if test_result is None or test_result.getSimulationState() in \ if test_result is None or test_result.getSimulationState() in \
('cancelled', 'failed'): ('cancelled', 'failed'):
del self.test_result_dict[test_title] pass
line_dict = {}
else: else:
last_state = test_result.getSimulationState() last_state = test_result.getSimulationState()
last_revision = str(test_result.getIntIndex()) last_revision = str(test_result.getIntIndex())
...@@ -141,9 +154,10 @@ class TaskDistributionTool(BaseTool): ...@@ -141,9 +154,10 @@ class TaskDistributionTool(BaseTool):
last_revision = reference last_revision = reference
elif reference: elif reference:
last_revision = last_revision, reference last_revision = last_revision, reference
if len(line_dict) == 0 and len(test_name_list): if len(test_result.objectValues(portal_type="Test Result Line")) == 0 \
self._p_changed = 1 and len(test_name_list):
createTestResultLineList(test_result, test_name_list, line_dict) self._p_changed = 1 # avoid two parallel creation
createTestResultLineList(test_result, test_name_list)
return test_result_path, last_revision return test_result_path, last_revision
if last_state == 'stopped': if last_state == 'stopped':
if reference_list_string is not None: if reference_list_string is not None:
...@@ -155,7 +169,8 @@ class TaskDistributionTool(BaseTool): ...@@ -155,7 +169,8 @@ class TaskDistributionTool(BaseTool):
portal_type='Test Result', portal_type='Test Result',
title=test_title, title=test_title,
reference=reference, reference=reference,
predecessor=test_result_path) predecessor=test_result_path,
activate_kw={"tag": tag})
if int_index is not None: if int_index is not None:
test_result.setIntIndex(int_index) test_result.setIntIndex(int_index)
if project_title is not None: if project_title is not None:
...@@ -168,10 +183,9 @@ class TaskDistributionTool(BaseTool): ...@@ -168,10 +183,9 @@ class TaskDistributionTool(BaseTool):
([x.path for x in project_list], project_title)) ([x.path for x in project_list], project_title))
test_result.updateLocalRolesOnSecurityGroups() # XXX test_result.updateLocalRolesOnSecurityGroups() # XXX
test_result_path = test_result.getRelativeUrl() test_result_path = test_result.getRelativeUrl()
self.test_result_dict[test_title] = test_result_path, line_dict
self._p_changed = 1 self._p_changed = 1
test_result.start() test_result.start()
createTestResultLineList(test_result, test_name_list, line_dict) createTestResultLineList(test_result, test_name_list)
createNode(test_result, node_title) createNode(test_result, node_title)
return test_result_path, revision return test_result_path, revision
...@@ -188,12 +202,9 @@ class TaskDistributionTool(BaseTool): ...@@ -188,12 +202,9 @@ class TaskDistributionTool(BaseTool):
test_result = portal.restrictedTraverse(test_result_path) test_result = portal.restrictedTraverse(test_result_path)
if test_result.getSimulationState() != 'started': if test_result.getSimulationState() != 'started':
return return
path, line_dict = self.test_result_dict[test_result.getTitle()]
assert path == test_result_path
started_list = [] started_list = []
for line_id, duration in sorted(line_dict.iteritems(), for line in test_result.objectValues(portal_type="Test Result Line",
key=lambda x: x[1], reverse=1): sort_on=[("int_index","ascending")]):
line = test_result[line_id]
test = line.getTitle() test = line.getTitle()
if test not in exclude_list: if test not in exclude_list:
state = line.getSimulationState() state = line.getSimulationState()
...@@ -218,15 +229,11 @@ class TaskDistributionTool(BaseTool): ...@@ -218,15 +229,11 @@ class TaskDistributionTool(BaseTool):
portal = self.getPortalObject() portal = self.getPortalObject()
line = portal.restrictedTraverse(test_path) line = portal.restrictedTraverse(test_path)
test_result = line.getParentValue() test_result = line.getParentValue()
path, line_dict = self.test_result_dict[test_result.getTitle()]
if test_result.getSimulationState() == 'started': if test_result.getSimulationState() == 'started':
assert path == test_result.getRelativeUrl() if line.getSimulationState() == "started":
line_id = line.getId()
if line_id in line_dict:
line.stop(**status_dict) line.stop(**status_dict)
del line_dict[line_id] if set([x.getSimulationState() for x in test_result.objectValues(
self._p_changed = 1 portal_type="Test Result Line")]) == set(["stopped"]):
if not line_dict:
test_result.stop() test_result.stop()
def _extractXMLRPCDict(self, xmlrpc_dict): def _extractXMLRPCDict(self, xmlrpc_dict):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment