Commit 2b4a5b6b authored by Benjamin Blanc's avatar Benjamin Blanc

testnode: SlapOSMasterCommunicator: Correct bugs and accelerate connections.

parent b03d9784
...@@ -502,6 +502,8 @@ revision = %(revision2)s ...@@ -502,6 +502,8 @@ revision = %(revision2)s
return (grade == 'master') return (grade == 'master')
def patch_isHostingSubscriptionCorrectly(self, *args, **kw): def patch_isHostingSubscriptionCorrectly(self, *args, **kw):
return True return True
def patch_isRegisteredHostingSubscription(self, *args, **kw):
return True
test_self = self test_self = self
test_result_path_root = os.path.join(test_self._temp_dir,'test/results') test_result_path_root = os.path.join(test_self._temp_dir,'test/results')
os.makedirs(test_result_path_root) os.makedirs(test_result_path_root)
...@@ -568,6 +570,7 @@ revision = %(revision2)s ...@@ -568,6 +570,7 @@ revision = %(revision2)s
original_isMasterTestnode = TaskDistributor.isMasterTestnode original_isMasterTestnode = TaskDistributor.isMasterTestnode
original_updateInstanceXML = RunnerClass._updateInstanceXML original_updateInstanceXML = RunnerClass._updateInstanceXML
original_isHostingSubscriptionCorrectly = SlapOSMasterCommunicator.isHostingSubscriptionCorrectly original_isHostingSubscriptionCorrectly = SlapOSMasterCommunicator.isHostingSubscriptionCorrectly
original_isRegisteredHostingSubscription = SlapOSMasterCommunicator.isRegisteredHostingSubscription
original_SlapOSMasterCommunicator__init__ = SlapOSMasterCommunicator.__init__ original_SlapOSMasterCommunicator__init__ = SlapOSMasterCommunicator.__init__
TaskDistributor.getSlaposAccountKey = patch_getSlaposAccountKey TaskDistributor.getSlaposAccountKey = patch_getSlaposAccountKey
TaskDistributor.getSlaposAccountCertificate = patch_getSlaposAccountCertificate TaskDistributor.getSlaposAccountCertificate = patch_getSlaposAccountCertificate
...@@ -575,6 +578,7 @@ revision = %(revision2)s ...@@ -575,6 +578,7 @@ revision = %(revision2)s
TaskDistributor.isMasterTestnode = patch_isMasterTestnode TaskDistributor.isMasterTestnode = patch_isMasterTestnode
RunnerClass._updateInstanceXML = doNothing RunnerClass._updateInstanceXML = doNothing
SlapOSMasterCommunicator.isHostingSubscriptionCorrectly = patch_isHostingSubscriptionCorrectly SlapOSMasterCommunicator.isHostingSubscriptionCorrectly = patch_isHostingSubscriptionCorrectly
SlapOSMasterCommunicator.isRegisteredHostingSubscription = patch_isRegisteredHostingSubscription
SlapOSMasterCommunicator.__init__ = doNothing SlapOSMasterCommunicator.__init__ = doNothing
original_startTestSuite = TaskDistributor.startTestSuite original_startTestSuite = TaskDistributor.startTestSuite
original_subscribeNode = TaskDistributor.subscribeNode original_subscribeNode = TaskDistributor.subscribeNode
...@@ -605,6 +609,7 @@ revision = %(revision2)s ...@@ -605,6 +609,7 @@ revision = %(revision2)s
TaskDistributor.isMasterTestnode = original_isMasterTestnode TaskDistributor.isMasterTestnode = original_isMasterTestnode
RunnerClass._updateInstanceXML = original_updateInstanceXML RunnerClass._updateInstanceXML = original_updateInstanceXML
SlapOSMasterCommunicator.isHostingSubscriptionCorrectly = original_isHostingSubscriptionCorrectly SlapOSMasterCommunicator.isHostingSubscriptionCorrectly = original_isHostingSubscriptionCorrectly
SlapOSMasterCommunicator.isRegisteredHostingSubscription = original_isRegisteredHostingSubscription
SlapOSMasterCommunicator.__init__ = original_SlapOSMasterCommunicator__init__ SlapOSMasterCommunicator.__init__ = original_SlapOSMasterCommunicator__init__
TaskDistributor.startTestSuite = original_startTestSuite TaskDistributor.startTestSuite = original_startTestSuite
TaskDistributionTool.createTestResult = original_createTestResult TaskDistributionTool.createTestResult = original_createTestResult
...@@ -668,6 +673,8 @@ revision = %(revision2)s ...@@ -668,6 +673,8 @@ revision = %(revision2)s
return grade == 'master' return grade == 'master'
def patch_isHostingSubscriptionCorrectly(self, *args, **kw): def patch_isHostingSubscriptionCorrectly(self, *args, **kw):
return True return True
def patch_isRegisteredHostingSubscription(self, *args, **kw):
return True
test_self = self test_self = self
test_result_path_root = os.path.join(test_self._temp_dir,'test/results') test_result_path_root = os.path.join(test_self._temp_dir,'test/results')
os.makedirs(test_result_path_root) os.makedirs(test_result_path_root)
...@@ -722,6 +729,7 @@ revision = %(revision2)s ...@@ -722,6 +729,7 @@ revision = %(revision2)s
original_request = SlapOSControler.request original_request = SlapOSControler.request
original_updateInstanceXML = RunnerClass._updateInstanceXML original_updateInstanceXML = RunnerClass._updateInstanceXML
original_isHostingSubscriptionCorrectly = SlapOSMasterCommunicator.isHostingSubscriptionCorrectly original_isHostingSubscriptionCorrectly = SlapOSMasterCommunicator.isHostingSubscriptionCorrectly
original_isRegisteredHostingSubscription = SlapOSMasterCommunicator.isRegisteredHostingSubscription
original_SlapOSMasterCommunicator__init__ = SlapOSMasterCommunicator.__init__ original_SlapOSMasterCommunicator__init__ = SlapOSMasterCommunicator.__init__
TaskDistributor.getSlaposAccountKey = patch_getSlaposAccountKey TaskDistributor.getSlaposAccountKey = patch_getSlaposAccountKey
TaskDistributor.getSlaposAccountCertificate = patch_getSlaposAccountCertificate TaskDistributor.getSlaposAccountCertificate = patch_getSlaposAccountCertificate
...@@ -731,6 +739,7 @@ revision = %(revision2)s ...@@ -731,6 +739,7 @@ revision = %(revision2)s
SlapOSControler.request = doNothing SlapOSControler.request = doNothing
RunnerClass._updateInstanceXML = doNothing RunnerClass._updateInstanceXML = doNothing
SlapOSMasterCommunicator.isHostingSubscriptionCorrectly = patch_isHostingSubscriptionCorrectly SlapOSMasterCommunicator.isHostingSubscriptionCorrectly = patch_isHostingSubscriptionCorrectly
SlapOSMasterCommunicator.isRegisteredHostingSubscription = patch_isRegisteredHostingSubscription
SlapOSMasterCommunicator.__init__ = doNothing SlapOSMasterCommunicator.__init__ = doNothing
original_startTestSuite = TaskDistributor.startTestSuite original_startTestSuite = TaskDistributor.startTestSuite
original_subscribeNode = TaskDistributor.subscribeNode original_subscribeNode = TaskDistributor.subscribeNode
...@@ -761,6 +770,7 @@ revision = %(revision2)s ...@@ -761,6 +770,7 @@ revision = %(revision2)s
SlapOSControler.request = original_request SlapOSControler.request = original_request
SlapOSControler.updateInstanceXML = original_updateInstanceXML SlapOSControler.updateInstanceXML = original_updateInstanceXML
SlapOSMasterCommunicator.isHostingSubscriptionCorrectly = original_isHostingSubscriptionCorrectly SlapOSMasterCommunicator.isHostingSubscriptionCorrectly = original_isHostingSubscriptionCorrectly
SlapOSMasterCommunicator.isRegisteredHostingSubscription = original_isRegisteredHostingSubscription
SlapOSMasterCommunicator.__init__ = original_SlapOSMasterCommunicator__init__ SlapOSMasterCommunicator.__init__ = original_SlapOSMasterCommunicator__init__
TaskDistributor.startTestSuite = original_startTestSuite TaskDistributor.startTestSuite = original_startTestSuite
TaskDistributionTool.createTestResult = original_createTestResult TaskDistributionTool.createTestResult = original_createTestResult
...@@ -938,6 +948,8 @@ revision = %(revision2)s ...@@ -938,6 +948,8 @@ revision = %(revision2)s
return "ScalabilityTest" return "ScalabilityTest"
def patch_isHostingSubscriptionCorrectly(self, *args, **kw): def patch_isHostingSubscriptionCorrectly(self, *args, **kw):
return True return True
def patch_isRegisteredHostingSubscription(self, *args, **kw):
return True
test_self = self test_self = self
test_result_path_root = os.path.join(test_self._temp_dir,'test/results') test_result_path_root = os.path.join(test_self._temp_dir,'test/results')
os.makedirs(test_result_path_root) os.makedirs(test_result_path_root)
...@@ -960,6 +972,7 @@ revision = %(revision2)s ...@@ -960,6 +972,7 @@ revision = %(revision2)s
original_request = SlapOSControler.request original_request = SlapOSControler.request
original_updateInstanceXML = SlapOSControler.updateInstanceXML original_updateInstanceXML = SlapOSControler.updateInstanceXML
original_isHostingSubscriptionCorrectly = SlapOSMasterCommunicator.isHostingSubscriptionCorrectly original_isHostingSubscriptionCorrectly = SlapOSMasterCommunicator.isHostingSubscriptionCorrectly
original_isRegisteredHostingSubscription = SlapOSMasterCommunicator.isRegisteredHostingSubscription
original_SlapOSMasterCommunicator__init__ = SlapOSMasterCommunicator.__init__ original_SlapOSMasterCommunicator__init__ = SlapOSMasterCommunicator.__init__
# #
...@@ -978,6 +991,7 @@ revision = %(revision2)s ...@@ -978,6 +991,7 @@ revision = %(revision2)s
SlapOSControler.request = doNothing SlapOSControler.request = doNothing
SlapOSControler.updateInstanceXML = doNothing SlapOSControler.updateInstanceXML = doNothing
SlapOSMasterCommunicator.isHostingSubscriptionCorrectly = patch_isHostingSubscriptionCorrectly SlapOSMasterCommunicator.isHostingSubscriptionCorrectly = patch_isHostingSubscriptionCorrectly
SlapOSMasterCommunicator.isRegisteredHostingSubscription = patch_isRegisteredHostingSubscription
SlapOSMasterCommunicator.__init__ = doNothing SlapOSMasterCommunicator.__init__ = doNothing
# Run # Run
test_node = self.getTestNode() test_node = self.getTestNode()
...@@ -997,5 +1011,6 @@ revision = %(revision2)s ...@@ -997,5 +1011,6 @@ revision = %(revision2)s
SlapOSControler.request = original_request SlapOSControler.request = original_request
SlapOSControler.updateInstanceXML = original_updateInstanceXML SlapOSControler.updateInstanceXML = original_updateInstanceXML
SlapOSMasterCommunicator.isHostingSubscriptionCorrectly = original_isHostingSubscriptionCorrectly SlapOSMasterCommunicator.isHostingSubscriptionCorrectly = original_isHostingSubscriptionCorrectly
SlapOSMasterCommunicator.isRegisteredHostingSubscription = original_isRegisteredHostingSubscription
SlapOSMasterCommunicator.__init__ = original_SlapOSMasterCommunicator__init__ SlapOSMasterCommunicator.__init__ = original_SlapOSMasterCommunicator__init__
time.sleep =original_sleep time.sleep =original_sleep
...@@ -47,6 +47,7 @@ from erp5.util import taskdistribution ...@@ -47,6 +47,7 @@ from erp5.util import taskdistribution
import signal import signal
MAX_INSTANCE_TIME = 60*60 # 1 hour MAX_INSTANCE_TIME = 60*60 # 1 hour
MAX_CREATION_INSTANCE_TIME = 60*5 # 5 minutes
class ScalabilityTestRunner(): class ScalabilityTestRunner():
def __init__(self, testnode): def __init__(self, testnode):
...@@ -177,13 +178,6 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),)) ...@@ -177,13 +178,6 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
# this simulate a SlapOS answer # this simulate a SlapOS answer
return self.simulateSlapOSAnswer() return self.simulateSlapOSAnswer()
def isInstanceReady(self, instance_title, state):
"""
Return true if the specified instance is ready.
This method should communicates with SlapOS Master.
"""
return self.slapos_communicator.isHostingSubscriptionCorrectly(instance_title, state)
def remainSoftwareToInstall(self): def remainSoftwareToInstall(self):
""" """
Return True if it remains softwares to install, otherwise return False Return True if it remains softwares to install, otherwise return False
...@@ -207,16 +201,31 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),)) ...@@ -207,16 +201,31 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
self.slapos_controler.updateInstanceXML(instance_title, {"_" : config}) self.slapos_controler.updateInstanceXML(instance_title, {"_" : config})
return {'status_code' : 0} return {'status_code' : 0}
# Used to simulate slapOS answer def _waitInstance(self, instance_title, state, max_time=MAX_INSTANCE_TIME):
def _waitInstance(self, instance_title, state): """
Wait for 'max_time' an instance specific state
"""
self.log("Wait for instance state: %s" %state) self.log("Wait for instance state: %s" %state)
#TODO: add a time limit start_time = time.time()
max_time = MAX_INSTANCE_TIME while (not self.slapos_communicator.isHostingSubscriptionCorrectly(instance_title, state)
start_time = 0
while (not self.isInstanceReady(instance_title, state)
and (max_time > (time.time()-start_time))): and (max_time > (time.time()-start_time))):
time.sleep(15) time.sleep(15)
if (time.time()-start_time) > max_time:
raise ErrorValue("Instance '%s' not '%s' after %s seconds" %(title, state))
self.log("Instance correctly '%s' after %s seconds." %(state, str(time.time()-start_time)))
def _waitInstanceCreation(self, title, max_time=MAX_CREATION_INSTANCE_TIME):
"""
Wait for 'max_time' the instance creation
"""
self.log("Wait for instance creation")
start_time = time.time()
while ( not self.slapos_communicator.isRegisteredHostingSubscription(title) \
and (max_time > (time.time()-start_time)) ):
time.sleep(5)
if (time.time()-start_time) > max_time:
raise ErrorValue("Instance '%s' not found after %s seconds" %(title, max_time))
self.log("Instance found on slapOSMaster")
def prepareSlapOSForTestSuite(self, node_test_suite): def prepareSlapOSForTestSuite(self, node_test_suite):
""" """
...@@ -303,9 +312,9 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),)) ...@@ -303,9 +312,9 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
self.instance_title = self._generateInstancetitle(node_test_suite.test_suite_title) self.instance_title = self._generateInstancetitle(node_test_suite.test_suite_title)
self._createInstance(self.reachable_profile, configuration_list[0], self._createInstance(self.reachable_profile, configuration_list[0],
self.instance_title, node_test_suite.test_result, node_test_suite.test_suite) self.instance_title, node_test_suite.test_result, node_test_suite.test_suite)
self.log("Waiting for instance creation..")
self._waitInstance(self.instance_title, 'started')
self.log("Scalability instance requested") self.log("Scalability instance requested")
self.log("Waiting for instance creation..")
self._waitInstanceCreation(self.instance_title)
""" except: """ except:
self.log("Unable to launch instance") self.log("Unable to launch instance")
raise ValueError("Unable to launch instance") raise ValueError("Unable to launch instance")
...@@ -330,16 +339,14 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),)) ...@@ -330,16 +339,14 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
count = 0 count = 0
for configuration in configuration_list: for configuration in configuration_list:
# Stop instance # Stop instance
self.log("Instance state: %s", self.slapos_controler.getInstanceState(self.instance_title)) if count > 0:
self.slapos_controler.stopInstance(self.instance_title) self.slapos_controler.stopInstance(self.instance_title)
self._waitInstance(self.instance_title, 'stopped') self._waitInstance(self.instance_title, 'stopped')
# Update instance XML configuration # Update instance XML configuration
self.log("Instance state: %s", self.slapos_controler.getInstanceState(self.instance_title))
self._updateInstanceXML(configuration, self.instance_title, self._updateInstanceXML(configuration, self.instance_title,
node_test_suite.test_result, node_test_suite.test_suite) node_test_suite.test_result, node_test_suite.test_suite)
self._waitInstance(self.instance_title, 'started') self._waitInstance(self.instance_title, 'started')
# Start instance # Start instance
self.log("Instance state: %s", self.slapos_controler.getInstanceState(self.instance_title))
self.slapos_controler.startInstance(self.instance_title) self.slapos_controler.startInstance(self.instance_title)
self._waitInstance(self.instance_title, 'started') self._waitInstance(self.instance_title, 'started')
...@@ -387,10 +394,8 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),)) ...@@ -387,10 +394,8 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
error = ValueError("Test case is in an undeterminated state") error = ValueError("Test case is in an undeterminated state")
break; break;
self.slapos_controler.destroyInstance(self.instance_title) self.slapos_controler.destroyInstance(self.instance_title)
if error: if error:
test_result_proxy.fail() test_result_proxy.fail()
raise error raise error
......
...@@ -136,13 +136,6 @@ class SlapOSControler(object): ...@@ -136,13 +136,6 @@ class SlapOSControler(object):
except: except:
raise ValueError("Instance '%s' not exist" %self.instance_config[reference]) raise ValueError("Instance '%s' not exist" %self.instance_config[reference])
def getInstanceState(self, reference):
try:
return self.instance_config[reference]['partition'].getState()
except:
raise ValueError("Impossible to get the instance state, instance "
"'%s' may not exist" %self.instance_config[reference])
def request(self, reference, software_url, software_type=None, def request(self, reference, software_url, software_type=None,
software_configuration=None, computer_guid=None, state='started'): software_configuration=None, computer_guid=None, state='started'):
""" """
......
import json import json
import httplib import httplib
import urlparse import urlparse
import time
# TODO: News-> look list to get last news... (and not the first of the list) # TODO: News-> look list to get last news... (and not the first of the list)
...@@ -25,15 +26,16 @@ class SlapOSMasterCommunicator(object): ...@@ -25,15 +26,16 @@ class SlapOSMasterCommunicator(object):
for instance_link in instance_link_list: for instance_link in instance_link_list:
news = communicator.getNewsFromInstanceLink(instance_link) news = communicator.getNewsFromInstanceLink(instance_link)
print news['news'] print news['news']
""" """
def __init__(self, certificate_path, key_path, log, def __init__(self, certificate_path, key_path, log,
url): url):
# Create connection # Create connection
api_scheme, api_netloc, api_path, api_query, api_fragment = urlparse.urlsplit(url) api_scheme, api_netloc, api_path, api_query, api_fragment = urlparse.urlsplit(url)
self.log = log self.log = log
self.log("HTTPS Connection with: %s, cert=%s, key=%s" %(api_netloc,key_path,certificate_path)) self.certificate_path = certificate_path
self.connection = httplib.HTTPSConnection(api_netloc, key_file=key_path, cert_file=certificate_path) self.key_path = key_path
self.url = url
self.connection = self._getConnection(self.certificate_path, self.key_path, self.url)
# Get master # Get master
master_link = {'href':api_path,'type':"application/vnd.slapos.org.hal+json; class=slapos.org.master"} master_link = {'href':api_path,'type':"application/vnd.slapos.org.hal+json; class=slapos.org.master"}
master = self._curl(master_link) master = self._curl(master_link)
...@@ -51,19 +53,29 @@ class SlapOSMasterCommunicator(object): ...@@ -51,19 +53,29 @@ class SlapOSMasterCommunicator(object):
"it may take several time...") "it may take several time...")
self._update_hosting_subscription_informations() self._update_hosting_subscription_informations()
def _getConnection(self,certificate_path, key_path, url):
api_scheme, api_netloc, api_path, api_query, api_fragment = urlparse.urlsplit(url)
self.log("HTTPS Connection with: %s, cert=%s, key=%s" %(api_netloc,key_path,certificate_path))
return httplib.HTTPSConnection(api_netloc, key_file=key_path, cert_file=certificate_path)
def _curl(self, link): def _curl(self, link):
""" """
'link' must look like : {'href':url,'type':content_type} 'link' must look like : {'href':url,'type':content_type}
""" """
self.log("_curl with: url:%s content_type:%s" %(link['href'], link['type'])) self.log("_curl with: url:%s content_type:%s" %(link['href'], link['type']))
api_scheme, api_netloc, api_path, api_query, api_fragment = urlparse.urlsplit(link['href']) api_scheme, api_netloc, api_path, api_query, api_fragment = urlparse.urlsplit(link['href'])
self.connection.request( # Try to use existing conection
method='GET', try:
url=api_path, self.connection.request(method='GET', url=api_path, headers={'Accept': link['type']}, body="")
headers={'Accept': link['type']}, response = self.connection.getresponse()
body="" except:
) try:
# Try to update and use the connection
self.connection = self._getConnection(self.certificate_path, self.key_path, self.url)
self.connection.request(method='GET', url=api_path, headers={'Accept': link['type']}, body="")
response = self.connection.getresponse() response = self.connection.getresponse()
except:
raise ValueError("Impossible to use connection")
return json.loads(response.read()) return json.loads(response.read())
def _update_hosting_subscription_informations(self): def _update_hosting_subscription_informations(self):
...@@ -77,8 +89,8 @@ class SlapOSMasterCommunicator(object): ...@@ -77,8 +89,8 @@ class SlapOSMasterCommunicator(object):
# For each hosting_subcription present in the collection # For each hosting_subcription present in the collection
for hosting_subscription_link in collection['_links']['item']: for hosting_subscription_link in collection['_links']['item']:
if hosting_subscription_link not in self.visited_hosting_subcriptions_link_list: if hosting_subscription_link not in self.visited_hosting_subcriptions_link_list:
title = self._curl(hosting_subscription_link)['title'] hosting_subscription = self._curl(hosting_subscription_link)
self.hosting_subcriptions_dict.update({title:hosting_subscription_link}) self.hosting_subcriptions_dict.update({hosting_subscription['title']:hosting_subscription_link})
self.visited_hosting_subcriptions_link_list.append(hosting_subscription_link) self.visited_hosting_subcriptions_link_list.append(hosting_subscription_link)
def _getRelatedInstanceLink(self, hosting_subscription_title): def _getRelatedInstanceLink(self, hosting_subscription_title):
...@@ -160,8 +172,16 @@ class SlapOSMasterCommunicator(object): ...@@ -160,8 +172,16 @@ class SlapOSMasterCommunicator(object):
""" """
instance_link_list = self._getRelatedInstanceLink(hosting_subscription_title) instance_link_list = self._getRelatedInstanceLink(hosting_subscription_title)
for instance_link in instance_link_list: for instance_link in instance_link_list:
if not communicator.isInstanceCorrectly(instance_link, status): if not self.isInstanceCorrectly(instance_link, status):
return False return False
return len(instance_link_list) > 0 return len(instance_link_list) > 0
def isRegisteredHostingSubscription(self, hosting_subscription_title):
"""
Return True if the specified hosting_subscription is present on SlapOSMaster
"""
self._update_hosting_subscription_informations()
if self.hosting_subcriptions_dict.get(hosting_subscription_title):
return True
return False
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment