Commit 169ecf6f authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼 Committed by Lu Xu

erp5.util: add a new test type to do real requests to the slapos master

This was previously known as "SlapOS Agent".
parent a57e0e2f
...@@ -489,6 +489,12 @@ class TaskDistributor(RPCRetry): ...@@ -489,6 +489,12 @@ class TaskDistributor(RPCRetry):
""" """
return self._retryRPC('getSlaposHateoasUrl') return self._retryRPC('getSlaposHateoasUrl')
def getServiceList(self, test_suite_title):
"""
Returns the list of services needed to run the test
"""
return self._retryRPC('getServiceList', (test_suite_title,))
def createTestResult(self, revision, test_name_list, node_title, def createTestResult(self, revision, test_name_list, node_title,
allow_restart=False, test_title=None, project_title=None): allow_restart=False, test_title=None, project_title=None):
""" """
......
This diff is collapsed.
...@@ -386,10 +386,11 @@ Require valid-user ...@@ -386,10 +386,11 @@ Require valid-user
self.error_message = test_configuration['error_message'] self.error_message = test_configuration['error_message']
self.randomized_path = test_configuration['randomized_path'] self.randomized_path = test_configuration['randomized_path']
if not self.launchable: if not self.launchable:
logger.info("Test suite %s is not actually launchable" error_message = ("Test suite %s is not actually launchable"
" with the current cluster configuration.", node_test_suite.test_suite_title) " with the current cluster configuration.\n"
logger.info("ERP5 Master indicates : %s", self.error_message) "ERP5 Master indicates : %s" % (node_test_suite.test_suite_title, self.error_message))
return {'status_code' : 1} logger.info(error_message)
return {'status_code' : 1, 'error_message' : error_message}
configuration_list = test_configuration['configuration_list'] configuration_list = test_configuration['configuration_list']
configuration = configuration_list[0] configuration = configuration_list[0]
...@@ -451,8 +452,9 @@ Require valid-user ...@@ -451,8 +452,9 @@ Require valid-user
self._comeBackFromDummySlapOS() self._comeBackFromDummySlapOS()
if self.remainSoftwareToInstall() : if self.remainSoftwareToInstall() :
# All softwares are not installed, however maxtime is elapsed, that's a failure. # All softwares are not installed, however maxtime is elapsed, that's a failure.
logger.error("All softwares are not installed.") error_message = "All softwares are not installed."
return {'status_code' : 1} logger.error(error_message)
return {'status_code' : 1, 'error_message' : error_message}
logger.debug("All software installed.") logger.debug("All software installed.")
# even if we re-use existing setup we need proper configuration applied # even if we re-use existing setup we need proper configuration applied
...@@ -469,11 +471,12 @@ Require valid-user ...@@ -469,11 +471,12 @@ Require valid-user
purge_previous_instance = not self.use_existing_setup) purge_previous_instance = not self.use_existing_setup)
logger.debug("Scalability instance requested.") logger.debug("Scalability instance requested.")
except Exception as e: except Exception as e:
logger.error("Error creating instance: " + str(e)) error_message = "Error creating instance: " + str(e)
return {'status_code' : 1} logger.error(error_message)
return {'status_code' : 1, 'error_message' : error_message}
return {'status_code' : 0} return {'status_code' : 0}
return {'status_code' : 1} return {'status_code' : 1, 'error_message' : "Software installation too long or error(s) are present during SR install."}
def makeSuite(self, test_suite, location_list, **kwargs): def makeSuite(self, test_suite, location_list, **kwargs):
import imp import imp
......
from __future__ import print_function from __future__ import print_function
import datetime import datetime
import feedparser
import json import json
import traceback import traceback
import time import time
...@@ -71,6 +72,8 @@ class SlapOSMasterCommunicator(object): ...@@ -71,6 +72,8 @@ class SlapOSMasterCommunicator(object):
self.slap_order = slap_order self.slap_order = slap_order
self.slap_supply = slap_supply self.slap_supply = slap_supply
self.hateoas_navigator = self.slap._hateoas_navigator self.hateoas_navigator = self.slap._hateoas_navigator
self.message_history = []
self.computer_guid = ""
if url is not None and \ if url is not None and \
url.startswith(SOFTWARE_PRODUCT_NAMESPACE): url.startswith(SOFTWARE_PRODUCT_NAMESPACE):
...@@ -79,7 +82,7 @@ class SlapOSMasterCommunicator(object): ...@@ -79,7 +82,7 @@ class SlapOSMasterCommunicator(object):
try: try:
url = product.__getattr__(url[len(SOFTWARE_PRODUCT_NAMESPACE):]) url = product.__getattr__(url[len(SOFTWARE_PRODUCT_NAMESPACE):])
except AttributeError as e: except AttributeError as e:
logger.warning('Error on get software release: %s ', e.message) logger.warning("Error on get software release: {}".format(e))
self.url = url self.url = url
...@@ -115,6 +118,7 @@ class SlapOSMasterCommunicator(object): ...@@ -115,6 +118,7 @@ class SlapOSMasterCommunicator(object):
partition_reference=self.name, partition_reference=self.name,
shared=shared, shared=shared,
state=state, state=state,
software_type=software_type,
**self.request_kw) **self.request_kw)
@retryOnNetworkFailure @retryOnNetworkFailure
...@@ -183,15 +187,17 @@ class SlapOSMasterCommunicator(object): ...@@ -183,15 +187,17 @@ class SlapOSMasterCommunicator(object):
message_list = [] message_list = []
try: try:
for instance in self.getInstanceUrlList(): for instance in self.getInstanceUrlList():
# we need to explicitly encode as utf-8 the unicode string we get logger.info('in _getInstanceState, viewing instance')
instance["text_content"] = instance["text_content"].encode('utf8') logger.info(instance)
news = instance['SoftwareInstance_getNewsDict'] news = instance['SoftwareInstance_getNewsDict']
state = INSTANCE_STATE_UNKNOWN state = INSTANCE_STATE_UNKNOWN
monitor_information_dict = {} monitor_information_dict = {}
is_slave = instance['portal_type'] == "Slave Instance" is_slave = instance['portal_type'] == "Slave Instance"
if is_slave: if is_slave:
if len(instance['getConnectionXmlAsDict']) > 0: # XXX for now consider a slave as always ready because in ORS software
# there is no information published in the slave
#if len(instance['getConnectionXmlAsDict']) > 0:
state = INSTANCE_STATE_STARTED state = INSTANCE_STATE_STARTED
else: else:
# not slave # not slave
...@@ -300,7 +306,6 @@ class SlapOSTester(SlapOSMasterCommunicator): ...@@ -300,7 +306,6 @@ class SlapOSTester(SlapOSMasterCommunicator):
self.request_kw = json.loads(request_kw) self.request_kw = json.loads(request_kw)
else: else:
self.request_kw = request_kw self.request_kw = request_kw
self.message_history = []
def getInfo(self): def getInfo(self):
info = "" info = ""
...@@ -323,12 +328,13 @@ class SlapOSTester(SlapOSMasterCommunicator): ...@@ -323,12 +328,13 @@ class SlapOSTester(SlapOSMasterCommunicator):
def requestInstanceStop(self, instance_title=None, request_kw=None, shared=False, software_type="RootSoftwareInstance"): def requestInstanceStop(self, instance_title=None, request_kw=None, shared=False, software_type="RootSoftwareInstance"):
self.instance = self._request(INSTANCE_STATE_STOPPED, instance_title, request_kw, shared, software_type) self.instance = self._request(INSTANCE_STATE_STOPPED, instance_title, request_kw, shared, software_type)
def requestInstanceDestroy(self, instance_title=None, request_kw=None, shared=False): def requestInstanceDestroy(self):
# TODO remove this function
self.destroyInstance()
def waitInstanceStarted(self, instance_title=None):
if not instance_title: if not instance_title:
instance_title = self.name instance_title = self.name
self.destroyInstance(instance_title)
def waitInstanceStarted(self, instance_title):
error_message = self._waitInstance(instance_title, INSTANCE_STATE_STARTED)["error_message"] error_message = self._waitInstance(instance_title, INSTANCE_STATE_STARTED)["error_message"]
if error_message is not None: if error_message is not None:
logger.error(error_message) logger.error(error_message)
...@@ -349,6 +355,13 @@ class SlapOSTester(SlapOSMasterCommunicator): ...@@ -349,6 +355,13 @@ class SlapOSTester(SlapOSMasterCommunicator):
logger.error(error_message) logger.error(error_message)
raise ValueError(error_message) raise ValueError(error_message)
def getInstanceParameterDict(self):
for instance in self.getInstanceUrlList():
if instance["title"] == self.name:
return instance["getConnectionXmlAsDict"]
return {}
def getMasterFrontendDict(self): def getMasterFrontendDict(self):
def getInstanceGuid(): def getInstanceGuid():
try: try:
...@@ -365,7 +378,7 @@ class SlapOSTester(SlapOSMasterCommunicator): ...@@ -365,7 +378,7 @@ class SlapOSTester(SlapOSMasterCommunicator):
pass pass
start_time = time.time() start_time = time.time()
while not getInstanceGuid() and time.time()-start_time < 60*5: while not getInstanceGuid() and time.time()-start_time < 60*5:
sleep(60) time.sleep(60)
return {'instance_guid' : getInstanceGuid(), 'frontend_master_ipv6' : frontend_master_ipv6} return {'instance_guid' : getInstanceGuid(), 'frontend_master_ipv6' : frontend_master_ipv6}
# XXX TODO # XXX TODO
...@@ -406,16 +419,15 @@ class SlapOSTester(SlapOSMasterCommunicator): ...@@ -406,16 +419,15 @@ class SlapOSTester(SlapOSMasterCommunicator):
'frontend-url-list' : frontend_url_list, \ 'frontend-url-list' : frontend_url_list, \
'balancer-user-v6': balancer_user_v6 } 'balancer-user-v6': balancer_user_v6 }
def destroyInstance(self, instance_title): def destroyInstance(self):
self.name = instance_title
instance_url_list = self.getInstanceUrlList() instance_url_list = self.getInstanceUrlList()
if instance_url_list: if instance_url_list:
for instance in instance_url_list: for instance in instance_url_list:
if instance["title"] != instance_title: if instance["title"] != self.name:
self._request(INSTANCE_STATE_DESTROYED, instance["title"]) self._request(INSTANCE_STATE_DESTROYED, instance["title"])
else: else:
root_instance = instance root_instance = instance
logger.info("Going to destroy root partition: " + str(instance_title)) logger.info("Going to destroy root partition: " + str(self.name))
self._request(INSTANCE_STATE_DESTROYED, root_instance["title"]) self._request(INSTANCE_STATE_DESTROYED, root_instance["title"])
else: else:
logger.info("Instance not found") logger.info("Instance not found")
...@@ -537,7 +549,7 @@ class SoftwareReleaseTester(SlapOSTester): ...@@ -537,7 +549,7 @@ class SoftwareReleaseTester(SlapOSTester):
deadline = self.deadline deadline = self.deadline
if deadline < now and deadline is not None: if deadline < now and deadline is not None:
raise TestTimeout(self.state) raise Exception("Test timeout (current state is {}).".format(self.state))
_, _, next_state, software_state, instance_state = self.transition_dict[ _, _, next_state, software_state, instance_state = self.transition_dict[
self.state] self.state]
......
...@@ -32,14 +32,10 @@ import logging ...@@ -32,14 +32,10 @@ import logging
from . import logger from . import logger
from .ProcessManager import SubprocessError, format_command from .ProcessManager import SubprocessError, format_command
from .SlapOSControler import SlapOSControler from .SlapOSControler import SlapOSControler
from .Utils import createFolder from .Utils import createFolder, dealShebang
from slapos.grid.utils import md5digest from slapos.grid.utils import md5digest
def dealShebang(run_test_suite_path):
with open(run_test_suite_path) as f:
if f.read(2) == '#!':
return f.readline().split(None, 1)
return []
class UnitTestRunner(object): class UnitTestRunner(object):
...@@ -203,6 +199,7 @@ class UnitTestRunner(object): ...@@ -203,6 +199,7 @@ class UnitTestRunner(object):
log_prefix='runTestSuite', log_prefix='runTestSuite',
output_replacers=(hide_distributor_url,), output_replacers=(hide_distributor_url,),
get_output=False) get_output=False)
return {'status_code' : 0}
def getRelativePathUsage(self): def getRelativePathUsage(self):
""" """
......
...@@ -10,6 +10,12 @@ def createFolder(folder, clean=False): ...@@ -10,6 +10,12 @@ def createFolder(folder, clean=False):
rmtree(folder) rmtree(folder)
os.mkdir(folder) os.mkdir(folder)
def dealShebang(run_test_suite_path):
with open(run_test_suite_path) as f:
if f.read(2) == '#!':
return f.readline().split(None, 1)
return []
if six.PY3: if six.PY3:
def deunicodeData(data): def deunicodeData(data):
return data return data
......
...@@ -38,6 +38,7 @@ from subprocess import CalledProcessError ...@@ -38,6 +38,7 @@ from subprocess import CalledProcessError
from .Updater import Updater from .Updater import Updater
from .NodeTestSuite import NodeTestSuite, SlapOSInstance from .NodeTestSuite import NodeTestSuite, SlapOSInstance
from .ScalabilityTestRunner import ScalabilityTestRunner from .ScalabilityTestRunner import ScalabilityTestRunner
from .RealRequestRunner import RealRequestRunner
from .UnitTestRunner import UnitTestRunner from .UnitTestRunner import UnitTestRunner
from .Utils import deunicodeData from .Utils import deunicodeData
from .. import taskdistribution from .. import taskdistribution
...@@ -49,6 +50,7 @@ PROFILE_PATH_KEY = 'profile_path' ...@@ -49,6 +50,7 @@ PROFILE_PATH_KEY = 'profile_path'
test_type_registry = { test_type_registry = {
'UnitTest': UnitTestRunner, 'UnitTest': UnitTestRunner,
'ScalabilityTest': ScalabilityTestRunner, 'ScalabilityTest': ScalabilityTestRunner,
'SlapOSAgentTest': RealRequestRunner
} }
class TestNode(object): class TestNode(object):
...@@ -106,13 +108,13 @@ class TestNode(object): ...@@ -106,13 +108,13 @@ class TestNode(object):
# Absolute path to relative path # Absolute path to relative path
software_config_path = os.path.join(repository_path, profile_path) software_config_path = os.path.join(repository_path, profile_path)
if use_relative_path : if use_relative_path:
from_path = os.path.join(self.working_directory, from_path = os.path.join(self.working_directory,
node_test_suite.reference) node_test_suite.reference)
software_config_path = os.path.relpath(software_config_path, from_path) software_config_path = os.path.relpath(software_config_path, from_path)
# Construct sections # Construct sections
if not(buildout_section_id is None): if buildout_section_id is not None:
# Absolute path to relative # Absolute path to relative
if use_relative_path: if use_relative_path:
from_path = os.path.join(self.working_directory, from_path = os.path.join(self.working_directory,
...@@ -320,7 +322,7 @@ shared = true ...@@ -320,7 +322,7 @@ shared = true
testnode_software_successfully_built = True testnode_software_successfully_built = True
logger.info("Will now skip build of testnode software") logger.info("Will now skip build of testnode software")
# Clean-up test suites # Clean-up test suites
self.purgeOldTestSuite(test_suite_data) #self.purgeOldTestSuite(test_suite_data)
for test_suite in test_suite_data: for test_suite in test_suite_data:
node_test_suite = self.getNodeTestSuite( node_test_suite = self.getNodeTestSuite(
test_suite.pop("test_suite_reference")) test_suite.pop("test_suite_reference"))
...@@ -354,6 +356,8 @@ shared = true ...@@ -354,6 +356,8 @@ shared = true
generated_config = taskdistributor.generateConfiguration( generated_config = taskdistributor.generateConfiguration(
node_test_suite.test_suite_title) node_test_suite.test_suite_title)
json_data = json.loads(generated_config) json_data = json.loads(generated_config)
logger.info("DEBUG JSON")
logger.info(json_data)
cluster_configuration = deunicodeData(json_data['configuration_list'][0]) cluster_configuration = deunicodeData(json_data['configuration_list'][0])
node_test_suite.edit(cluster_configuration=cluster_configuration) node_test_suite.edit(cluster_configuration=cluster_configuration)
# Now prepare the installation of SlapOS and create instance # Now prepare the installation of SlapOS and create instance
...@@ -364,35 +368,26 @@ shared = true ...@@ -364,35 +368,26 @@ shared = true
# should be at the same revision, so it is safe to prune orphan # should be at the same revision, so it is safe to prune orphan
# objects now. # objects now.
git_gc_auto() git_gc_auto()
def report_error(error_message):
test_result.reportFailure(
stdout=error_message
)
logger.error(error_message)
raise ValueError(error_message)
if status_dict['status_code'] == 1:
report_error(status_dict.get('error_message') or "Error during prepareSlapOSForTestSuite")
# Give some time so computer partitions may start # Give some time so computer partitions may start
# as partitions can be of any kind we have and likely will never have # as partitions can be of any kind we have and likely will never have
# a reliable way to check if they are up or not ... # a reliable way to check if they are up or not ...
time.sleep(20) time.sleep(20)
# XXX: Do not switch according to the test type. IOW, the
# following code must be moved to the test type class.
if my_test_type == 'UnitTest':
runner.runTestSuite(node_test_suite, portal_url) runner.runTestSuite(node_test_suite, portal_url)
elif my_test_type == 'ScalabilityTest':
error_message = None
# A problem is appeared during runTestSuite
if status_dict['status_code'] == 1:
error_message = "Software installation too long or error(s) are present during SR install."
else:
status_dict = runner.runTestSuite(node_test_suite, portal_url) status_dict = runner.runTestSuite(node_test_suite, portal_url)
# A problem is appeared during runTestSuite # A problem is appeared during runTestSuite
if status_dict['status_code'] == 1: if status_dict['status_code'] == 1:
error_message = status_dict['error_message'] report_error(status_dict.get('error_message') or 'Error during runTestSuite')
# If an error is appeared
if error_message:
test_result.reportFailure(
stdout=error_message
)
logger.error(error_message)
raise ValueError(error_message)
else:
raise NotImplementedError
# break the loop to get latest priorities from master
break break
except (SubprocessError, CalledProcessError, ConnectionError) as e: except (SubprocessError, CalledProcessError, ConnectionError) as e:
logger.exception("") logger.exception("")
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment