Commit c2205acc authored by Roque's avatar Roque

erp5_test_result: ERP5ScalabilityDistributor refactoring

- code for generateConfigurationList moved to scalability distributor
- general code refactoring and cleanup
parent 35cadb66
...@@ -32,6 +32,8 @@ from zLOG import LOG,ERROR ...@@ -32,6 +32,8 @@ from zLOG import LOG,ERROR
from AccessControl import ClassSecurityInfo from AccessControl import ClassSecurityInfo
from Products.ERP5Type import Permissions from Products.ERP5Type import Permissions
import json import json
import jinja2
from Products.ERP5.Tool.TaskDistributionTool import TaskDistributionTool
class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor): class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor):
security = ClassSecurityInfo() security = ClassSecurityInfo()
...@@ -42,7 +44,6 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor): ...@@ -42,7 +44,6 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor):
""" """
Do master testnode selection and associate testsuites. Do master testnode selection and associate testsuites.
""" """
# Get modules # Get modules
test_node_module = self._getTestNodeModule() test_node_module = self._getTestNodeModule()
test_suite_module = self._getTestSuiteModule() test_suite_module = self._getTestSuiteModule()
...@@ -50,22 +51,25 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor): ...@@ -50,22 +51,25 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor):
# Get lists of test node wich belong to the current distributor # Get lists of test node wich belong to the current distributor
all_test_node_list = test_node_module.searchFolder( all_test_node_list = test_node_module.searchFolder(
portal_type="Test Node", specialise_uid=self.getUid()) portal_type="Test Node", specialise_uid=self.getUid())
test_node_list = [ x.getObject() for x in all_test_node_list if x.getValidationState() == 'validated' ] test_node_list = [ x.getObject() for x in all_test_node_list
invalid_test_node_list = [ x.getObject() for x in all_test_node_list if x.getValidationState() != 'validated' ] if x.getValidationState() == 'validated' ]
invalid_test_node_list = [ x.getObject() for x in all_test_node_list
if x.getValidationState() != 'validated' ]
# Set master property to False for all invalid testnode # Set master property to False for all invalid testnode
for node in invalid_test_node_list: for node in invalid_test_node_list:
node.setMaster(False) node.setMaster(False)
# Get all valid slave testnodes # Get all valid slave testnodes
slave_test_node_list = [ x.getObject() for x in test_node_list if x.getMaster()!=True ] slave_test_node_list = [ x.getObject() for x in test_node_list
master_test_node_list = [ x.getObject() for x in test_node_list if x.getMaster()==True ] if x.getMaster()!=True ]
master_test_node_list = [ x.getObject() for x in test_node_list
if x.getMaster()==True ]
# Set to 'False' slaves # Set to 'False' slaves
for node in slave_test_node_list: for node in slave_test_node_list:
node.setMaster(False) node.setMaster(False)
# if there is validated testnodes # if there is validated testnodes
if len(test_node_list) > 0: if len(test_node_list) > 0:
# Only one testnode must be the master # Only one testnode must be the master
...@@ -80,8 +84,10 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor): ...@@ -80,8 +84,10 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor):
node.setMaster(False) node.setMaster(False)
# Update testnode lists # Update testnode lists
slave_test_node_list = [ x.getObject() for x in test_node_list if x.getMaster()!=True ] slave_test_node_list = [ x.getObject() for x in test_node_list
master_test_node_list = [ x.getObject() for x in test_node_list if x.getMaster()==True ] if x.getMaster()!=True ]
master_test_node_list = [ x.getObject() for x in test_node_list
if x.getMaster()==True ]
# Get test suites and aggregate them to all valid testnode # Get test suites and aggregate them to all valid testnode
test_suite_list = [ x.getRelativeUrl() for x in test_suite_module.searchFolder( test_suite_list = [ x.getRelativeUrl() for x in test_suite_module.searchFolder(
...@@ -97,8 +103,9 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor): ...@@ -97,8 +103,9 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor):
""" """
test_node_module = self._getTestNodeModule() test_node_module = self._getTestNodeModule()
distributor_title = self.getTitle() distributor_title = self.getTitle()
involved_nodes = [] involved_nodes = [ x.getObject() for x in test_node_module.searchFolder(
involved_nodes = involved_nodes + [ x.getObject() for x in test_node_module.searchFolder(validation_state='validated') if ( x.getSpecialiseTitle() == distributor_title ) ] validation_state='validated')
if ( x.getSpecialiseTitle() == distributor_title ) ]
return involved_nodes return involved_nodes
security.declarePublic("getTestNode") security.declarePublic("getTestNode")
...@@ -130,10 +137,12 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor): ...@@ -130,10 +137,12 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor):
security.declarePublic("isMasterTestnode") security.declarePublic("isMasterTestnode")
def isMasterTestnode(self,title, batch_mode=0): def isMasterTestnode(self,title, batch_mode=0):
""" """
isMasterTestnode : return True if the node given in parameter exists and is a validated master isMasterTestnode : return True if the node given in
parameter exists and is a validated master
""" """
test_node_module = self._getTestNodeModule() test_node_module = self._getTestNodeModule()
test_node_master = [ node for node in test_node_module.searchFolder(portal_type="Test Node", test_node_master = [ node for node in test_node_module.searchFolder(
portal_type="Test Node",
title=SimpleQuery(comparison_operator='=', title=title), title=SimpleQuery(comparison_operator='=', title=title),
specialise_uid=self.getUid(), specialise_uid=self.getUid(),
validation_state="validated") if node.getMaster() == 1 ] validation_state="validated") if node.getMaster() == 1 ]
...@@ -142,7 +151,6 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor): ...@@ -142,7 +151,6 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor):
else: else:
return False return False
security.declarePublic("getTestType") security.declarePublic("getTestType")
def getTestType(self, batch_mode=0): def getTestType(self, batch_mode=0):
""" """
...@@ -150,21 +158,34 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor): ...@@ -150,21 +158,34 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor):
""" """
return 'ScalabilityTest' return 'ScalabilityTest'
security.declarePublic("createTestResult")
def createTestResult(self, name, revision, test_name_list, allow_restart,
test_title=None, node_title=None, project_title=None):
"""
Here this is only a proxy to the task distribution tool
"""
LOG('[ERP5ScalabilityDistributor] createTestResult', 0, (node_title, test_title))
portal = self.getPortalObject()
return portal.portal_task_distribution.createTestResult(name,
revision, test_name_list, allow_restart,
test_title=test_title, node_title=node_title,
project_title=project_title)
security.declarePublic("startTestSuite") security.declarePublic("startTestSuite")
def startTestSuite(self,title, computer_guid='unknown', batch_mode=0): def startTestSuite(self,title, computer_guid='unknown', batch_mode=0):
""" """
startTestSuite : subscribe node + return testsuite list to the master startTestSuite : subscribe node + return testsuite list to the master
""" """
super(ERP5ScalabilityDistributor, super(ERP5ScalabilityDistributor, self).subscribeNode(
self).subscribeNode(title=title, computer_guid=computer_guid, batch_mode=batch_mode) title=title, computer_guid=computer_guid, batch_mode=batch_mode)
test_node_module = self._getTestNodeModule() test_node_module = self._getTestNodeModule()
# If the testnode wich request testsuites is not the master # If the testnode wich request testsuites is not the master
# he does not have to receive any testsuites # he does not have to receive any testsuites
master_test_node_title_list = [x.getTitle() for x in test_node_module.searchFolder( master_test_node_title_list = [x.getTitle() for x in test_node_module.searchFolder(
validation_state = 'validated') if (x.getMaster() == True) ] validation_state = 'validated',
specialise_uid=self.getUid())
if (x.getMaster() == True) ]
if len(master_test_node_title_list) == 1 and title == master_test_node_title_list[0]: if len(master_test_node_title_list) == 1 and title == master_test_node_title_list[0]:
return super(ERP5ScalabilityDistributor, return super(ERP5ScalabilityDistributor,
self).startTestSuite(title=title, self).startTestSuite(title=title,
...@@ -174,13 +195,13 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor): ...@@ -174,13 +195,13 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor):
return [] return []
return json.dumps([]) return json.dumps([])
security.declarePublic("generateConfiguration") security.declarePublic("generateConfiguration")
def generateConfiguration(self, test_suite_title, batch_mode=0): def generateConfiguration(self, test_suite_title, batch_mode=0):
""" """
generateConfiguration : this is just a proxy to an external method generateConfiguration : this is just a proxy to an external method
""" """
generated_configuration = self.ScalabilityTestSuiteUtils_generateConfigurationList(self, test_suite_title) portal = self.getPortalObject()
generated_configuration = self._generateConfigurationList(test_suite_title, portal)
if batch_mode: if batch_mode:
return generated_configuration return generated_configuration
return json.dumps(generated_configuration) return json.dumps(generated_configuration)
...@@ -201,14 +222,13 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor): ...@@ -201,14 +222,13 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor):
if state == "started": if state == "started":
relative_path = test_result_line.getRelativeUrl() relative_path = test_result_line.getRelativeUrl()
title = test_result_line.getTitle() title = test_result_line.getTitle()
break; break
else: else:
return None return None
next_test = {"relative_path": relative_path, next_test = {"relative_path": relative_path,
"title": title, "count" : count, "title": title, "count" : count,
} }
if batch_mode: if batch_mode:
return next_test return next_test
return json.dumps(next_test) return json.dumps(next_test)
...@@ -221,3 +241,113 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor): ...@@ -221,3 +241,113 @@ class ERP5ScalabilityDistributor(ERP5ProjectUnitTestDistributor):
portal = self.getPortalObject() portal = self.getPortalObject()
test_result_line = portal.restrictedTraverse(test_result_line_path) test_result_line = portal.restrictedTraverse(test_result_line_path)
return test_result_line.getSimulationState() == "started" return test_result_line.getSimulationState() == "started"
security.declarePublic("_unvalidateConfig")
def _unvalidateConfig(self, my_dict):
my_dict['launchable'] = False
if 'launcher_nodes_computer_guid' in my_dict:
my_dict['launcher_nodes_computer_guid'] = {}
if 'involved_nodes_computer_guid' in my_dict:
my_dict['involved_nodes_computer_guid'] = {}
if 'configuration_list' in my_dict:
my_dict['configuration_list'] = []
security.declarePublic("_getInvolvedNodes")
def _getInvolvedNodes(self, available_nodes, return_dict):
"""
Get the list of all nodes involved in the test
"""
def _isContained(my_value, my_container):
def _isInMyDictOrList(current_container):
if current_container == my_value :
return True
elif isinstance(current_container, dict):
for k,v in current_container.items():
if str(my_value) in str(k):
return True
if _isInMyDictOrList(current_container[k]) :
return True
elif isinstance(current_container, list):
for sub_container in current_container:
if _isInMyDictOrList(sub_container) :
return True
return False
return _isInMyDictOrList(my_container)
involved_nodes_computer_guid = []
for node in available_nodes:
if _isContained(node, return_dict):
involved_nodes_computer_guid.append(node)
return involved_nodes_computer_guid
security.declarePublic("_getAvailableNodes")
def _getAvailableNodes(self, test_suite_specialise_title, portal):
"""
gets testnodes available for this distributor
"""
distributor_uid = self.getUid()
test_node_module = portal.test_node_module
available_nodes = test_node_module.searchFolder(
portal_type="Test Node", validation_state="validated",
specialise_uid=distributor_uid)
available_nodes = [ node.getComputerGuid() for node in available_nodes ]
return available_nodes
security.declarePublic("_generateConfigurationList")
def _generateConfigurationList(self, test_suite_title, portal):
"""
generateConfigurationList : generate a dict wich contains a list
of cluster configurations for testnodes, using a test_suite and
available testnodes.
If it is not possible to generate a configuration, the 'launchable'
entry is False, otherwise it is True.
"""
return_dict = {}
# Get test_suite informations from his title given in parameter
try:
test_suite = portal.test_suite_module.searchFolder(title=test_suite_title)[0]
cluster_configuration = test_suite.getClusterConfiguration()
number_configuration_list = test_suite.getGraphCoordinate()
randomized_path = test_suite.getRandomizedPath()
except Exception as e:
return_dict['error_message'] = 'Error getting test suite information. \
Bad test suite title? ' + str(e)
self._unvalidateConfig(return_dict)
return return_dict
try:
available_nodes = self._getAvailableNodes(test_suite.getSpecialiseTitle(), portal)
remaining_nodes = list(available_nodes)
launcher_nodes = [ remaining_nodes.pop() ]
if len(remaining_nodes) == 0: # if there is only one computer, use launcher
remaining_nodes.append(launcher_nodes[0])
except Exception:
return_dict['error_message'] = 'Error getting available nodes. \
No test nodes for test suite?'
self._unvalidateConfig(return_dict)
return return_dict
configuration_list_json = []
return_dict['error_message'] = 'No error.'
return_dict['randomized_path'] = randomized_path
try:
template = jinja2.Template(cluster_configuration)
for index in xrange(0, len(number_configuration_list)):
template_vars = { "count" : number_configuration_list[index],
"comp" : remaining_nodes }
configuration_list_json.append( json.loads(
template.render(template_vars)
)
)
return_dict['launchable'] = True
return_dict['configuration_list'] = configuration_list_json
return_dict['launcher_nodes_computer_guid'] = launcher_nodes
except Exception:
return_dict['error_message'] = 'Bad json cluster_configuration?'
self._unvalidateConfig(return_dict)
return return_dict
return_dict['involved_nodes_computer_guid'] = self._getInvolvedNodes(available_nodes,
return_dict)
return return_dict
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment