Commit 38441c3f authored by Benjamin Blanc's avatar Benjamin Blanc

Good update, but still not working

parent 146b188a
from unittest import TestCase from unittest import TestCase
from erp5.util.testnode.testnode import TestNode from erp5.util.testnode.testnode import TestNode
from erp5.util.testnode.NodeTestSuite import SlapOSInstance from erp5.util.testnode.NodeTestSuite import SlapOSInstance, NodeTestSuite
from erp5.util.testnode.ProcessManager import ProcessManager, SubprocessError from erp5.util.testnode.ProcessManager import ProcessManager, SubprocessError
from erp5.util.testnode.SlapOSControler import SlapOSControler from erp5.util.testnode.SlapOSControler import SlapOSControler
from erp5.util.testnode.UnitTestRunner import UnitTestRunner
from erp5.util.testnode.ScalabilityTestRunner import ScalabilityTestRunner
from erp5.util.testnode.SlapOSControler import createFolder from erp5.util.testnode.SlapOSControler import createFolder
from erp5.util.taskdistribution import TaskDistributor from erp5.util.taskdistribution import TaskDistributor
from erp5.util.taskdistribution import TaskDistributionTool from erp5.util.taskdistribution import TaskDistributionTool
from erp5.util.taskdistribution import TestResultProxy from erp5.util.taskdistribution import TestResultProxy
...@@ -54,7 +57,8 @@ class ERP5TestNode(TestCase): ...@@ -54,7 +57,8 @@ class ERP5TestNode(TestCase):
# XXX how to get property the git path ? # XXX how to get property the git path ?
config = {} config = {}
config["git_binary"] = "git" config["git_binary"] = "git"
config["slapos_directory"] = config["working_directory"] = self.working_directory config["slapos_directory"] = self.working_directory
config["working_directory"] = self.working_directory
config["node_quantity"] = 3 config["node_quantity"] = 3
config["test_suite_directory"] = self.test_suite_directory config["test_suite_directory"] = self.test_suite_directory
config["environment"] = self.environment config["environment"] = self.environment
...@@ -90,6 +94,9 @@ class ERP5TestNode(TestCase): ...@@ -90,6 +94,9 @@ class ERP5TestNode(TestCase):
def updateNodeTestSuiteData(self, node_test_suite, def updateNodeTestSuiteData(self, node_test_suite,
add_third_repository=False): add_third_repository=False):
"""
Update from zero/Regenerate the testsuite
"""
node_test_suite.edit(working_directory=self.working_directory, node_test_suite.edit(working_directory=self.working_directory,
**self.getTestSuiteData(add_third_repository=add_third_repository)[0]) **self.getTestSuiteData(add_third_repository=add_third_repository)[0])
...@@ -346,36 +353,49 @@ branch = foo ...@@ -346,36 +353,49 @@ branch = foo
original_getSupportedParameter = ProcessManager.getSupportedParameterSet original_getSupportedParameter = ProcessManager.getSupportedParameterSet
original_spawn = ProcessManager.spawn original_spawn = ProcessManager.spawn
try: try:
# Create a file
def _createPath(path_to_create, end_path): def _createPath(path_to_create, end_path):
os.makedirs(path_to_create) os.makedirs(path_to_create)
return os.close(os.open(os.path.join(path_to_create, return os.close(os.open(os.path.join(path_to_create,
end_path),os.O_CREAT)) end_path),os.O_CREAT))
def get_parameters(self, *args, **kw): def get_parameters(self, *args, **kw):
call_parameter_list.append({'args': [x for x in args], 'kw':kw}) call_parameter_list.append({'args': [x for x in args], 'kw':kw})
def patch_getSupportedParameterSet(self, run_test_suite_path, parameter_list,): def patch_getSupportedParameterSet(self, run_test_suite_path, parameter_list,):
if '--firefox_bin' and '--xvfb_bin' in parameter_list: if '--firefox_bin' and '--xvfb_bin' in parameter_list:
return set(['--firefox_bin','--xvfb_bin']) return set(['--firefox_bin','--xvfb_bin'])
else: else:
return [] return []
test_node = self.getTestNode() test_node = self.getTestNode()
test_node.slapos_controler = SlapOSControler(self.working_directory, test_node.slapos_controler = SlapOSControler(
test_node.config, self.log) self.working_directory,
test_node.config, self.log)
# Create and initialise/regenerate a nodetestsuite
node_test_suite = test_node.getNodeTestSuite('foo') node_test_suite = test_node.getNodeTestSuite('foo')
self.updateNodeTestSuiteData(node_test_suite) self.updateNodeTestSuiteData(node_test_suite)
node_test_suite.revision = 'dummy' node_test_suite.revision = 'dummy'
# Path to the dummy runable
run_test_suite_path = _createPath( run_test_suite_path = _createPath(
os.path.join(test_node.slapos_controler.instance_root,'a/bin'),'runTestSuite') os.path.join(test_node.slapos_controler.instance_root,'a/bin'),'runTestSuite')
def checkRunTestSuiteParameters(additional_parameter_list=None): def checkRunTestSuiteParameters(additional_parameter_list=None):
ProcessManager.getSupportedParameterSet = patch_getSupportedParameterSet ProcessManager.getSupportedParameterSet = patch_getSupportedParameterSet
ProcessManager.spawn = get_parameters ProcessManager.spawn = get_parameters
test_node.runTestSuite(node_test_suite,"http://foo.bar") runner = UnitTestRunner(test_node)
runner.runTestSuite(node_test_suite,"http://foo.bar")
expected_parameter_list = ['%s/a/bin/runTestSuite' expected_parameter_list = ['%s/a/bin/runTestSuite'
%(test_node.slapos_controler.instance_root), '--test_suite', 'Foo', '--revision', %(runner.testnode.slapos_controler.instance_root), '--test_suite', 'Foo', '--revision',
'dummy', '--test_suite_title', 'Foo-Test', '--node_quantity', 3, '--master_url', 'dummy', '--test_suite_title', 'Foo-Test', '--node_quantity', 3, '--master_url',
'http://foo.bar'] 'http://foo.bar']
if additional_parameter_list: if additional_parameter_list:
expected_parameter_list.extend(additional_parameter_list) expected_parameter_list.extend(additional_parameter_list)
self.assertEqual(call_parameter_list[0]['args'], expected_parameter_list) self.assertEqual(call_parameter_list[0]['args'], expected_parameter_list)
call_parameter_list = [] call_parameter_list = []
checkRunTestSuiteParameters() checkRunTestSuiteParameters()
_createPath(os.path.join(test_node.config['slapos_directory'], 'soft/a/parts/firefox'),'firefox-slapos') _createPath(os.path.join(test_node.config['slapos_directory'], 'soft/a/parts/firefox'),'firefox-slapos')
...@@ -394,6 +414,7 @@ branch = foo ...@@ -394,6 +414,7 @@ branch = foo
def test_10_prepareSlapOS(self): def test_10_prepareSlapOS(self):
test_node = self.getTestNode() test_node = self.getTestNode()
test_node_slapos = SlapOSInstance() test_node_slapos = SlapOSInstance()
runner = UnitTestRunner(test_node)
node_test_suite = test_node.getNodeTestSuite('foo') node_test_suite = test_node.getNodeTestSuite('foo')
node_test_suite.edit(working_directory=self.working_directory) node_test_suite.edit(working_directory=self.working_directory)
status_dict = {"status_code" : 0} status_dict = {"status_code" : 0}
...@@ -412,17 +433,18 @@ branch = foo ...@@ -412,17 +433,18 @@ branch = foo
SlapOSControler.initializeSlapOSControler = Patch("initializeSlapOSControler") SlapOSControler.initializeSlapOSControler = Patch("initializeSlapOSControler")
SlapOSControler.runSoftwareRelease = Patch("runSoftwareRelease") SlapOSControler.runSoftwareRelease = Patch("runSoftwareRelease")
SlapOSControler.runComputerPartition = Patch("runComputerPartition") SlapOSControler.runComputerPartition = Patch("runComputerPartition")
test_node.prepareSlapOSForTestNode(test_node_slapos)
runner.prepareSlapOSForTestNode(test_node_slapos)
self.assertEquals(["initializeSlapOSControler", "runSoftwareRelease"], self.assertEquals(["initializeSlapOSControler", "runSoftwareRelease"],
[x["method_name"] for x in call_list]) [x["method_name"] for x in call_list])
call_list = [] call_list = []
test_node.prepareSlapOSForTestSuite(node_test_suite) runner.prepareSlapOSForTestSuite(node_test_suite)
self.assertEquals(["initializeSlapOSControler", "runSoftwareRelease", self.assertEquals(["initializeSlapOSControler", "runSoftwareRelease",
"runComputerPartition"], "runComputerPartition"],
[x["method_name"] for x in call_list]) [x["method_name"] for x in call_list])
call_list = [] call_list = []
SlapOSControler.runSoftwareRelease = Patch("runSoftwareRelease", status_code=1) SlapOSControler.runSoftwareRelease = Patch("runSoftwareRelease", status_code=1)
self.assertRaises(SubprocessError, test_node.prepareSlapOSForTestSuite, self.assertRaises(SubprocessError, runner.prepareSlapOSForTestSuite,
node_test_suite) node_test_suite)
def test_11_run(self): def test_11_run(self):
...@@ -438,10 +460,10 @@ branch = foo ...@@ -438,10 +460,10 @@ branch = foo
config_list = [] config_list = []
def _checkExistingTestSuite(reference_set): def _checkExistingTestSuite(reference_set):
test_self.assertEquals(set(reference_set), test_self.assertEquals(set(reference_set),
set(os.listdir(test_node.config["working_directory"]))) set(os.listdir(test_node.working_directory)))
for x in reference_set: for x in reference_set:
test_self.assertTrue(os.path.exists(os.path.join( test_self.assertTrue(os.path.exists(os.path.join(
test_node.config["working_directory"],x)),True) test_node.working_directory,x)),True)
if counter == 0: if counter == 0:
config_list.append(test_self.getTestSuiteData(reference='foo')[0]) config_list.append(test_self.getTestSuiteData(reference='foo')[0])
config_list.append(test_self.getTestSuiteData(reference='bar')[0]) config_list.append(test_self.getTestSuiteData(reference='bar')[0])
...@@ -483,18 +505,22 @@ branch = foo ...@@ -483,18 +505,22 @@ branch = foo
original_createTestResult = TaskDistributionTool.createTestResult original_createTestResult = TaskDistributionTool.createTestResult
TaskDistributionTool.createTestResult = patch_createTestResult TaskDistributionTool.createTestResult = patch_createTestResult
test_node = self.getTestNode() test_node = self.getTestNode()
original_prepareSlapOS = test_node._prepareSlapOS test_node_slapos = SlapOSInstance()
test_node._prepareSlapOS = doNothing
original_runTestSuite = test_node.runTestSuite runner = UnitTestRunner(test_node)
test_node.runTestSuite = doNothing
original_prepareSlapOS = runner._prepareSlapOS
runner._prepareSlapOS = doNothing
original_runTestSuite = runner.runTestSuite
runner.runTestSuite = doNothing
SlapOSControler.initializeSlapOSControler = doNothing SlapOSControler.initializeSlapOSControler = doNothing
test_node.run() runner.testnode.run()
self.assertEquals(5, counter) self.assertEquals(5, counter)
time.sleep = original_sleep time.sleep = original_sleep
TaskDistributor.startTestSuite = original_startTestSuite TaskDistributor.startTestSuite = original_startTestSuite
TaskDistributionTool.createTestResult = original_createTestResult TaskDistributionTool.createTestResult = original_createTestResult
test_node._prepareSlapOS = original_prepareSlapOS runner._prepareSlapOS = original_prepareSlapOS
test_node.runTestSuite = original_runTestSuite runner.runTestSuite = original_runTestSuite
def test_12_spawn(self): def test_12_spawn(self):
def _checkCorrectStatus(expected_status,*args): def _checkCorrectStatus(expected_status,*args):
...@@ -585,10 +611,11 @@ branch = foo ...@@ -585,10 +611,11 @@ branch = foo
original_createTestResult = TaskDistributionTool.createTestResult original_createTestResult = TaskDistributionTool.createTestResult
TaskDistributionTool.createTestResult = patch_createTestResult TaskDistributionTool.createTestResult = patch_createTestResult
test_node = self.getTestNode() test_node = self.getTestNode()
original_prepareSlapOS = test_node._prepareSlapOS runner = UnitTestRunner(test_node)
test_node._prepareSlapOS = doNothing original_prepareSlapOS = runner._prepareSlapOS
original_runTestSuite = test_node.runTestSuite runner._prepareSlapOS = doNothing
test_node.runTestSuite = doNothing original_runTestSuite = runner.runTestSuite
runner.runTestSuite = doNothing
SlapOSControler.initializeSlapOSControler = doNothing SlapOSControler.initializeSlapOSControler = doNothing
test_node.run() test_node.run()
self.assertEquals(counter, 3) self.assertEquals(counter, 3)
...@@ -596,8 +623,8 @@ branch = foo ...@@ -596,8 +623,8 @@ branch = foo
time.sleep = original_sleep time.sleep = original_sleep
TaskDistributor.startTestSuite = original_startTestSuite TaskDistributor.startTestSuite = original_startTestSuite
TaskDistributionTool.createTestResult = original_createTestResult TaskDistributionTool.createTestResult = original_createTestResult
test_node._prepareSlapOS = original_prepareSlapOS runner._prepareSlapOS = original_prepareSlapOS
test_node.runTestSuite = original_runTestSuite runner.runTestSuite = original_runTestSuite
def test_16_cleanupLogDirectory(self): def test_16_cleanupLogDirectory(self):
# Make sure that we are able to cleanup old log folders # Make sure that we are able to cleanup old log folders
...@@ -650,6 +677,8 @@ branch = foo ...@@ -650,6 +677,8 @@ branch = foo
SlapOSControler.initializeSlapOSControler SlapOSControler.initializeSlapOSControler
initial_runSoftwareRelease = SlapOSControler.runSoftwareRelease initial_runSoftwareRelease = SlapOSControler.runSoftwareRelease
test_node = self.getTestNode() test_node = self.getTestNode()
runner = UnitTestRunner(test_node)
node_test_suite = test_node.getNodeTestSuite('foo') node_test_suite = test_node.getNodeTestSuite('foo')
init_call_kw_list = [] init_call_kw_list = []
def initializeSlapOSControler(self, **kw): def initializeSlapOSControler(self, **kw):
...@@ -659,7 +688,7 @@ branch = foo ...@@ -659,7 +688,7 @@ branch = foo
SlapOSControler.initializeSlapOSControler = initializeSlapOSControler SlapOSControler.initializeSlapOSControler = initializeSlapOSControler
SlapOSControler.runSoftwareRelease = runSoftwareRelease SlapOSControler.runSoftwareRelease = runSoftwareRelease
def callPrepareSlapOS(): def callPrepareSlapOS():
test_node._prepareSlapOS(self.working_directory, node_test_suite, runner._prepareSlapOS(self.working_directory, node_test_suite,
test_node.log, create_partition=0) test_node.log, create_partition=0)
def callRaisingPrepareSlapos(): def callRaisingPrepareSlapos():
self.assertRaises(SubprocessError, callPrepareSlapOS) self.assertRaises(SubprocessError, callPrepareSlapOS)
......
...@@ -42,8 +42,6 @@ from subprocess import CalledProcessError ...@@ -42,8 +42,6 @@ from subprocess import CalledProcessError
from Updater import Updater from Updater import Updater
from erp5.util import taskdistribution from erp5.util import taskdistribution
class ScalabilityTestRunner(object): class ScalabilityTestRunner():
def __init__(self): def __init__(self, testnode):
pass self.testnode = testnode
...@@ -43,16 +43,108 @@ from NodeTestSuite import SlapOSInstance ...@@ -43,16 +43,108 @@ from NodeTestSuite import SlapOSInstance
from Updater import Updater from Updater import Updater
from erp5.util import taskdistribution from erp5.util import taskdistribution
class UnitTestRunner(object): class UnitTestRunner():
def __init__(self, testnode): def __init__(self, testnode):
self.testnode = testnode self.testnode = testnode
self.test_node_slapos = SlapOSInstance() self.slapos_controler = SlapOSControler.SlapOSControler(
self.test_node_slapos.edit(working_directory=self.testnode.config['slapos_directory']) self.testnode.working_directory,
self.testnode.config,
self.testnode.log)
def _prepareSlapOS(self, working_directory, slapos_instance, log,
create_partition=1, software_path_list=None, **kw):
"""
Launch slapos to build software and partitions
"""
slapproxy_log = os.path.join(self.testnode.config['log_directory'],
'slapproxy.log')
log('Configured slapproxy log to %r' % slapproxy_log)
reset_software = slapos_instance.retry_software_count > 10
if reset_software:
slapos_instance.retry_software_count = 0
log('testnode, retry_software_count : %r' % \
slapos_instance.retry_software_count)
self.slapos_controler.initializeSlapOSControler(slapproxy_log=slapproxy_log,
process_manager=self.testnode.process_manager, reset_software=reset_software,
software_path_list=software_path_list)
self.testnode.process_manager.supervisord_pid_file = os.path.join(\
self.slapos_controler.instance_root, 'var', 'run', 'supervisord.pid')
method_list= ["runSoftwareRelease"]
if create_partition:
method_list.append("runComputerPartition")
for method_name in method_list:
slapos_method = getattr(self.slapos_controler, method_name)
status_dict = slapos_method(self.testnode.config,
environment=self.testnode.config['environment'],
)
if status_dict['status_code'] != 0:
slapos_instance.retry = True
slapos_instance.retry_software_count += 1
raise SubprocessError(status_dict)
else:
slapos_instance.retry_software_count = 0
return status_dict
def prepareSlapOSForTestNode(self, test_node_slapos):
"""
We will build slapos software needed by the testnode itself,
like the building of selenium-runner by default
"""
return self._prepareSlapOS(self.testnode.config['slapos_directory'],
test_node_slapos, self.testnode.log, create_partition=0,
software_path_list=self.testnode.config.get("software_list"))
def prepareSlapOSForTestSuite(self, node_test_suite):
log = self.testnode.log
if log is None:
log = self.testnode.log
return self._prepareSlapOS(node_test_suite.working_directory,
node_test_suite, log,
software_path_list=[node_test_suite.custom_profile_path])
def runTestSuite(self, node_test_suite, portal_url, log=None):
config = self.testnode.config
parameter_list = []
run_test_suite_path_list = glob.glob("%s/*/bin/runTestSuite" % \
self.slapos_controler.instance_root)
if not len(run_test_suite_path_list):
raise ValueError('No runTestSuite provided in installed partitions.')
run_test_suite_path = run_test_suite_path_list[0]
run_test_suite_revision = node_test_suite.revision
# Deal with Shebang size limitation
invocation_list = self.testnode._dealShebang(run_test_suite_path)
invocation_list.extend([run_test_suite_path,
'--test_suite', node_test_suite.test_suite,
'--revision', node_test_suite.revision,
'--test_suite_title', node_test_suite.test_suite_title,
'--node_quantity', config['node_quantity'],
'--master_url', portal_url])
firefox_bin_list = glob.glob("%s/soft/*/parts/firefox/firefox-slapos" % \
config["slapos_directory"])
if len(firefox_bin_list):
parameter_list.append('--firefox_bin')
xvfb_bin_list = glob.glob("%s/soft/*/parts/xserver/bin/Xvfb" % \
config["slapos_directory"])
if len(xvfb_bin_list):
parameter_list.append('--xvfb_bin')
supported_paramater_set = self.testnode.process_manager.getSupportedParameterSet(
run_test_suite_path, parameter_list)
if '--firefox_bin' in supported_paramater_set:
invocation_list.extend(["--firefox_bin", firefox_bin_list[0]])
if '--xvfb_bin' in supported_paramater_set:
invocation_list.extend(["--xvfb_bin", xvfb_bin_list[0]])
bt5_path_list = config.get("bt5_path")
if bt5_path_list not in ('', None,):
invocation_list.extend(["--bt5_path", bt5_path_list])
# From this point, test runner becomes responsible for updating test
# result. We only do cleanup if the test runner itself is not able
# to run.
SlapOSControler.createFolder(node_test_suite.test_suite_directory,
clean=True)
self.testnode.process_manager.spawn(*invocation_list,
cwd=node_test_suite.test_suite_directory,
log_prefix='runTestSuite', get_output=False)
......
...@@ -88,6 +88,7 @@ class BaseTestNode(object): ...@@ -88,6 +88,7 @@ class BaseTestNode(object):
self.log = log self.log = log
self.config = config or {} self.config = config or {}
self.process_manager = ProcessManager(log) self.process_manager = ProcessManager(log)
self.working_directory = config['working_directory']
self.node_test_suite_dict = {} self.node_test_suite_dict = {}
self.file_handler = None self.file_handler = None
self.max_log_time = max_log_time self.max_log_time = max_log_time
...@@ -96,12 +97,12 @@ class BaseTestNode(object): ...@@ -96,12 +97,12 @@ class BaseTestNode(object):
def checkOldTestSuite(self,test_suite_data): def checkOldTestSuite(self,test_suite_data):
config = self.config config = self.config
installed_reference_set = set(os.listdir(config['working_directory'])) installed_reference_set = set(os.listdir(self.working_directory))
wished_reference_set = set([x['test_suite_reference'] for x in test_suite_data]) wished_reference_set = set([x['test_suite_reference'] for x in test_suite_data])
to_remove_reference_set = installed_reference_set.difference( to_remove_reference_set = installed_reference_set.difference(
wished_reference_set) wished_reference_set)
for y in to_remove_reference_set: for y in to_remove_reference_set:
fpath = os.path.join(config['working_directory'],y) fpath = os.path.join(self.working_directory,y)
self.delNodeTestSuite(y) self.delNodeTestSuite(y)
self.log("testnode.checkOldTestSuite, DELETING : %r" % (fpath,)) self.log("testnode.checkOldTestSuite, DELETING : %r" % (fpath,))
if os.path.isdir(fpath): if os.path.isdir(fpath):
...@@ -149,7 +150,7 @@ class BaseTestNode(object): ...@@ -149,7 +150,7 @@ class BaseTestNode(object):
# Absolute path to relative path # Absolute path to relative path
software_config_path = os.path.join(repository_path, profile_path) software_config_path = os.path.join(repository_path, profile_path)
if use_relative_path : if use_relative_path :
from_path = os.path.join(self.config['working_directory'], from_path = os.path.join(self.working_directory,
node_test_suite.reference) node_test_suite.reference)
software_config_path = os.path.relpath(software_config_path, from_path) software_config_path = os.path.relpath(software_config_path, from_path)
...@@ -163,7 +164,7 @@ extends = %(software_config_path)s ...@@ -163,7 +164,7 @@ extends = %(software_config_path)s
if not(buildout_section_id is None): if not(buildout_section_id is None):
# Absolute path to relative # Absolute path to relative
if use_relative_path: if use_relative_path:
from_path = os.path.join(self.config['working_directory'], from_path = os.path.join(self.working_directory,
node_test_suite.reference) node_test_suite.reference)
repository_path = os.path.relpath(repository_path, from_path) repository_path = os.path.relpath(repository_path, from_path)
...@@ -272,6 +273,38 @@ branch = %(branch)s ...@@ -272,6 +273,38 @@ branch = %(branch)s
self.log("deleting log directory %r" % (folder_path,)) self.log("deleting log directory %r" % (folder_path,))
shutil.rmtree(folder_path) shutil.rmtree(folder_path)
def _cleanupTemporaryFiles(self):
"""
buildout seems letting files under /tmp. To avoid regular error of
missing disk space, remove old logs
"""
temp_directory = self.config["system_temp_folder"]
now = time.time()
user_id = os.geteuid()
for temp_folder in os.listdir(temp_directory):
folder_path = os.path.join(temp_directory, temp_folder)
if (temp_folder.startswith("tmp") or
temp_folder.startswith("buildout")):
try:
stat = os.stat(folder_path)
if stat.st_uid == user_id and \
(now - stat.st_mtime)/86400 > self.max_temp_time:
self.log("deleting temp directory %r" % (folder_path,))
if os.path.isdir(folder_path):
shutil.rmtree(folder_path)
else:
os.remove(folder_path)
except OSError:
self.log("_cleanupTemporaryFiles exception", exc_info=sys.exc_info())
def cleanUp(self,test_result):
log = self.log
log('Testnode.cleanUp')
self.process_manager.killPreviousRun()
self._cleanupLog()
self._cleanupTemporaryFiles()
class ScalabilityTestNode(BaseTestNode): class ScalabilityTestNode(BaseTestNode):
def __init__(self, log, config, max_log_time=MAX_LOG_TIME, def __init__(self, log, config, max_log_time=MAX_LOG_TIME,
...@@ -331,152 +364,32 @@ class ScalabilityTestNode(BaseTestNode): ...@@ -331,152 +364,32 @@ class ScalabilityTestNode(BaseTestNode):
# Merge BaseTestNode and TestNode
class TestNode(BaseTestNode): class TestNode(BaseTestNode):
def __init__(self, log, config, max_log_time=MAX_LOG_TIME, def __init__(self, log, config, max_log_time=MAX_LOG_TIME,
max_temp_time=MAX_TEMP_TIME): max_temp_time=MAX_TEMP_TIME):
BaseTestNode.__init__(self, log, config, max_log_time, max_temp_time) BaseTestNode.__init__(self, log, config, max_log_time, max_temp_time)
def _prepareSlapOS(self, working_directory, slapos_instance, log,
create_partition=1, software_path_list=None, **kw):
"""
Launch slapos to build software and partitions
"""
slapproxy_log = os.path.join(self.config['log_directory'],
'slapproxy.log')
log('Configured slapproxy log to %r' % slapproxy_log)
reset_software = slapos_instance.retry_software_count > 10
if reset_software:
slapos_instance.retry_software_count = 0
log('testnode, retry_software_count : %r' % \
slapos_instance.retry_software_count)
self.slapos_controler = SlapOSControler.SlapOSControler(
working_directory, self.config, log)
self.slapos_controler.initializeSlapOSControler(slapproxy_log=slapproxy_log,
process_manager=self.process_manager, reset_software=reset_software,
software_path_list=software_path_list)
self.process_manager.supervisord_pid_file = os.path.join(\
self.slapos_controler.instance_root, 'var', 'run', 'supervisord.pid')
method_list= ["runSoftwareRelease"]
if create_partition:
method_list.append("runComputerPartition")
for method_name in method_list:
slapos_method = getattr(self.slapos_controler, method_name)
status_dict = slapos_method(self.config,
environment=self.config['environment'],
)
if status_dict['status_code'] != 0:
slapos_instance.retry = True
slapos_instance.retry_software_count += 1
raise SubprocessError(status_dict)
else:
slapos_instance.retry_software_count = 0
return status_dict
def prepareSlapOSForTestNode(self, test_node_slapos):
"""
We will build slapos software needed by the testnode itself,
like the building of selenium-runner by default
"""
return self._prepareSlapOS(self.config['slapos_directory'],
test_node_slapos, self.log, create_partition=0,
software_path_list=self.config.get("software_list"))
def prepareSlapOSForTestSuite(self, node_test_suite):
log = self.log
if log is None:
log = self.log
return self._prepareSlapOS(node_test_suite.working_directory,
node_test_suite, log,
software_path_list=[node_test_suite.custom_profile_path])
def runTestSuite(self, node_test_suite, portal_url, log=None):
config = self.config
parameter_list = []
run_test_suite_path_list = glob.glob("%s/*/bin/runTestSuite" % \
self.slapos_controler.instance_root)
if not len(run_test_suite_path_list):
raise ValueError('No runTestSuite provided in installed partitions.')
run_test_suite_path = run_test_suite_path_list[0]
run_test_suite_revision = node_test_suite.revision
# Deal with Shebang size limitation
invocation_list = self._dealShebang(run_test_suite_path)
invocation_list.extend([run_test_suite_path,
'--test_suite', node_test_suite.test_suite,
'--revision', node_test_suite.revision,
'--test_suite_title', node_test_suite.test_suite_title,
'--node_quantity', config['node_quantity'],
'--master_url', portal_url])
firefox_bin_list = glob.glob("%s/soft/*/parts/firefox/firefox-slapos" % \
config["slapos_directory"])
if len(firefox_bin_list):
parameter_list.append('--firefox_bin')
xvfb_bin_list = glob.glob("%s/soft/*/parts/xserver/bin/Xvfb" % \
config["slapos_directory"])
if len(xvfb_bin_list):
parameter_list.append('--xvfb_bin')
supported_paramater_set = self.process_manager.getSupportedParameterSet(
run_test_suite_path, parameter_list)
if '--firefox_bin' in supported_paramater_set:
invocation_list.extend(["--firefox_bin", firefox_bin_list[0]])
if '--xvfb_bin' in supported_paramater_set:
invocation_list.extend(["--xvfb_bin", xvfb_bin_list[0]])
bt5_path_list = config.get("bt5_path")
if bt5_path_list not in ('', None,):
invocation_list.extend(["--bt5_path", bt5_path_list])
# From this point, test runner becomes responsible for updating test
# result. We only do cleanup if the test runner itself is not able
# to run.
SlapOSControler.createFolder(node_test_suite.test_suite_directory,
clean=True)
self.process_manager.spawn(*invocation_list,
cwd=node_test_suite.test_suite_directory,
log_prefix='runTestSuite', get_output=False)
def _cleanupTemporaryFiles(self):
"""
buildout seems letting files under /tmp. To avoid regular error of
missing disk space, remove old logs
"""
temp_directory = self.config["system_temp_folder"]
now = time.time()
user_id = os.geteuid()
for temp_folder in os.listdir(temp_directory):
folder_path = os.path.join(temp_directory, temp_folder)
if (temp_folder.startswith("tmp") or
temp_folder.startswith("buildout")):
try:
stat = os.stat(folder_path)
if stat.st_uid == user_id and \
(now - stat.st_mtime)/86400 > self.max_temp_time:
self.log("deleting temp directory %r" % (folder_path,))
if os.path.isdir(folder_path):
shutil.rmtree(folder_path)
else:
os.remove(folder_path)
except OSError:
self.log("_cleanupTemporaryFiles exception", exc_info=sys.exc_info())
def cleanUp(self,test_result):
log = self.log
log('Testnode.cleanUp')
self.process_manager.killPreviousRun()
self._cleanupLog()
self._cleanupTemporaryFiles()
def run(self): def run(self):
## BLOCK OK
log = self.log log = self.log
config = self.config config = self.config
slapgrid = None slapgrid = None
previous_revision_dict = {} previous_revision_dict = {}
revision_dict = {} revision_dict = {}
test_result = None test_result = None
test_node_slapos = SlapOSInstance()
runner = UnitTestRunner(self) test_node_slapos.edit(working_directory=self.config['slapos_directory'])
## /BLOCK OK
try: try:
while True: while True:
try: try:
##BLOCK OK
node_test_suite = None node_test_suite = None
self.log = self.process_manager.log = self.testnode_log self.log = self.process_manager.log = self.testnode_log
self.cleanUp(None) self.cleanUp(None)
...@@ -489,14 +402,28 @@ class TestNode(BaseTestNode): ...@@ -489,14 +402,28 @@ class TestNode(BaseTestNode):
test_suite_data = deunicodeData(json.loads(test_suite_json)) test_suite_data = deunicodeData(json.loads(test_suite_json))
log("Got following test suite data from master : %r" % \ log("Got following test suite data from master : %r" % \
(test_suite_data,)) (test_suite_data,))
##/BLOCK OK
# Here we know what we are (sclability or unit test) # Here we know what we are (sclability or unit test)
# change the line below to runner.prepareSlapOSForTestNode ..
self.prepareSlapOSForTestNode(runner.test_node_slapos) # self.prepareSlapOSForTestNode(test_node_slapos)
# Select the good runner
if True :
runner = UnitTestRunner(self)
elif False :
runner = ScalabilityTestRunner(self)
else :
runner = UnitTestRunner(self)
runner.prepareSlapOSForTestNode(test_node_slapos)
#Clean-up test suites #Clean-up test suites
self.checkOldTestSuite(test_suite_data) self.checkOldTestSuite(test_suite_data)
for test_suite in test_suite_data: for test_suite in test_suite_data:
## BLOCK OK
remote_test_result_needs_cleanup = False remote_test_result_needs_cleanup = False
node_test_suite = self.getNodeTestSuite( node_test_suite = self.getNodeTestSuite(
test_suite["test_suite_reference"]) test_suite["test_suite_reference"])
...@@ -517,16 +444,28 @@ class TestNode(BaseTestNode): ...@@ -517,16 +444,28 @@ class TestNode(BaseTestNode):
node_test_suite.project_title) node_test_suite.project_title)
remote_test_result_needs_cleanup = True remote_test_result_needs_cleanup = True
log("testnode, test_result : %r" % (test_result, )) log("testnode, test_result : %r" % (test_result, ))
## /BLOCK OK
if test_result is not None: if test_result is not None:
## BLOCK OK
self.registerSuiteLog(test_result, node_test_suite) self.registerSuiteLog(test_result, node_test_suite)
self.checkRevision(test_result,node_test_suite) self.checkRevision(test_result,node_test_suite)
## /BLOCK OK
# Now prepare the installation of SlapOS and create instance # Now prepare the installation of SlapOS and create instance
status_dict = self.prepareSlapOSForTestSuite(node_test_suite) # status_dict = self.prepareSlapOSForTestSuite(node_test_suite)
status_dict = runner.prepareSlapOSForTestSuite(node_test_suite)
# Give some time so computer partitions may start # Give some time so computer partitions may start
# as partitions can be of any kind we have and likely will never have # as partitions can be of any kind we have and likely will never have
# a reliable way to check if they are up or not ... # a reliable way to check if they are up or not ...
time.sleep(20) # time.sleep(20)
self.runTestSuite(node_test_suite,portal_url) # self.runTestSuite(node_test_suite,portal_url)
# For scalability test runTestSuite is a big part
runner.runTestSuite(node_test_suite,portal_url)
# break the loop to get latest priorities from master # break the loop to get latest priorities from master
break break
self.cleanUp(test_result) self.cleanUp(test_result)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment