Commit 0a2145dd authored by Jérome Perrin's avatar Jérome Perrin

Testnode environment variables and log URL

- use environment variables to pass SLAPOS_TEST_LOG_DIRECTORY and SLAPOS_TEST_SHARED_PART_LIST, as discussed on nexedi/nxdtest!2
- Fix #20200514-218C705 - [testnode] frontend for log access

See merge request nexedi/erp5!1304
parents 7924d351 daf3d133
...@@ -95,6 +95,7 @@ class ERP5TestNode(TestCase): ...@@ -95,6 +95,7 @@ class ERP5TestNode(TestCase):
config["httpd_ip"] = "ff:ff:ff:ff:ff:ff:ff:ff" config["httpd_ip"] = "ff:ff:ff:ff:ff:ff:ff:ff"
config["httpd_software_access_port"] = "9080" config["httpd_software_access_port"] = "9080"
config["frontend_url"] = "http://frontend/" config["frontend_url"] = "http://frontend/"
config["log_frontend_url"] = "http://log-frontend/"
config["software_list"] = ["foo", "bar"] config["software_list"] = ["foo", "bar"]
config["partition_reference"] = "part" config["partition_reference"] = "part"
config["ipv4_address"] = "1.2.3.4" config["ipv4_address"] = "1.2.3.4"
...@@ -620,6 +621,7 @@ shared = true ...@@ -620,6 +621,7 @@ shared = true
def test_distributor_url_obfuscated_in_logs(self): def test_distributor_url_obfuscated_in_logs(self):
test_node = self.getTestNode() test_node = self.getTestNode()
del test_node.suiteLog
runner = test_type_registry['UnitTest'](test_node) runner = test_type_registry['UnitTest'](test_node)
node_test_suite = test_node.getNodeTestSuite('foo') node_test_suite = test_node.getNodeTestSuite('foo')
...@@ -635,7 +637,8 @@ shared = true ...@@ -635,7 +637,8 @@ shared = true
""".format(temp_file.name)) """.format(temp_file.name))
os.chmod(path, 0o700) os.chmod(path, 0o700)
with mock.patch.object(logger, '_log') as http_logger,\ with test_node.suiteLog(node_test_suite),\
mock.patch.object(logger, '_log') as http_logger,\
mock.patch.object(logging.getLogger(), '_log') as root_logger,\ mock.patch.object(logging.getLogger(), '_log') as root_logger,\
mock.patch.object(logging.getLogger(), 'isEnabledFor', return_value=True): mock.patch.object(logging.getLogger(), 'isEnabledFor', return_value=True):
runner.runTestSuite(node_test_suite, "https://user:secret@example.com/portal_distributions") runner.runTestSuite(node_test_suite, "https://user:secret@example.com/portal_distributions")
...@@ -676,7 +679,8 @@ shared = true ...@@ -676,7 +679,8 @@ shared = true
call_parameter_list.append(args) call_parameter_list.append(args)
test_node = self.getTestNode() test_node = self.getTestNode()
with mock.patch.object(test_node.process_manager, 'spawn', side_effect=spawn): del test_node.suiteLog
with mock.patch.object(test_node.process_manager, 'spawn', side_effect=spawn) as spawn_mock:
runner = test_type_registry[my_test_type](test_node) runner = test_type_registry[my_test_type](test_node)
# Create and initialise/regenerate a nodetestsuite # Create and initialise/regenerate a nodetestsuite
node_test_suite = test_node.getNodeTestSuite('foo') node_test_suite = test_node.getNodeTestSuite('foo')
...@@ -695,9 +699,16 @@ shared = true ...@@ -695,9 +699,16 @@ shared = true
'--test_suite_title', 'Foo-Test', '--test_suite_title', 'Foo-Test',
] ]
def checkRunTestSuiteParameters(): def checkRunTestSuiteParameters():
with test_node.suiteLog(node_test_suite) as suite_log:
runner.runTestSuite(node_test_suite, "http://foo.bar") runner.runTestSuite(node_test_suite, "http://foo.bar")
self.assertEqual(list(call_parameter_list.pop()), expected_parameter_list) self.assertEqual(list(call_parameter_list.pop()), expected_parameter_list)
self.assertFalse(call_parameter_list) self.assertFalse(call_parameter_list)
self.assertEqual(
spawn_mock.call_args[-1]['SLAPOS_TEST_SHARED_PART_LIST'],
"/not/exists:/not/exists_either:%s/shared" % node_test_suite.working_directory)
self.assertEqual(
spawn_mock.call_args[-1]['SLAPOS_TEST_LOG_DIRECTORY'],
os.path.join(test_node.config['log_directory'], suite_log))
checkRunTestSuiteParameters() checkRunTestSuiteParameters()
...@@ -1005,7 +1016,7 @@ shared = true ...@@ -1005,7 +1016,7 @@ shared = true
logger, test_result_path, node_title, revision) logger, test_result_path, node_title, revision)
def patch_runTestSuite(self,*argv, **kw): def patch_runTestSuite(self,*argv, **kw):
return {'status_code': 0} return {'status_code': 0}
def checkTestSuite(test_node): def checkTestSuite(test_node, reportTaskStatus_mock):
test_node.node_test_suite_dict test_node.node_test_suite_dict
rand_part_set = set() rand_part_set = set()
self.assertEqual(2, len(test_node.node_test_suite_dict)) self.assertEqual(2, len(test_node.node_test_suite_dict))
...@@ -1015,6 +1026,14 @@ shared = true ...@@ -1015,6 +1026,14 @@ shared = true
"Incorrect suite log path : %r" % suite.suite_log_path) "Incorrect suite log path : %r" % suite.suite_log_path)
m = re.search('-(.*)/suite.log$', suite.suite_log_path) m = re.search('-(.*)/suite.log$', suite.suite_log_path)
rand_part = m.groups()[0] rand_part = m.groups()[0]
reportTaskStatus_mock.assert_has_calls(
[mock.call(
mock.ANY,
{'command': 'LOG url',
'stdout': "http://log-frontend/{}-{}".format(suite.reference, rand_part),
'stderr': ''},
"Foo-Test-Node")])
self.assertEqual(len(rand_part), 10) self.assertEqual(len(rand_part), 10)
self.assertNotIn(rand_part, rand_part_set) self.assertNotIn(rand_part, rand_part_set)
rand_part_set.add(rand_part) rand_part_set.add(rand_part)
...@@ -1071,9 +1090,11 @@ shared = true ...@@ -1071,9 +1090,11 @@ shared = true
return {'status_code': 0} return {'status_code': 0}
RunnerClass._prepareSlapOS = patch_prepareSlapOS RunnerClass._prepareSlapOS = patch_prepareSlapOS
SlapOSControler.initializeSlapOSControler = doNothing SlapOSControler.initializeSlapOSControler = doNothing
with mock.patch('erp5.util.taskdistribution.DummyTaskDistributor.reportTaskStatus') as reportTaskStatus:
test_node.run() test_node.run()
self.assertEqual(counter, 3) self.assertEqual(counter, 3)
checkTestSuite(test_node) checkTestSuite(test_node, reportTaskStatus)
time.sleep = original_sleep time.sleep = original_sleep
# Restore old class methods # Restore old class methods
if my_test_type == "ScalabilityTest": if my_test_type == "ScalabilityTest":
......
...@@ -150,6 +150,12 @@ class UnitTestRunner(object): ...@@ -150,6 +150,12 @@ class UnitTestRunner(object):
software_list = [soft + md5digest(x) for x in config['software_list']] software_list = [soft + md5digest(x) for x in config['software_list']]
PATH = os.getenv('PATH', '') PATH = os.getenv('PATH', '')
PATH = ':'.join(x + '/bin' for x in software_list) + (PATH and ':' + PATH) PATH = ':'.join(x + '/bin' for x in software_list) + (PATH and ':' + PATH)
SLAPOS_TEST_SHARED_PART_LIST = os.pathsep.join(
self._getSlapOSControler(
node_test_suite.working_directory,
True
).shared_part_list)
SLAPOS_TEST_LOG_DIRECTORY = node_test_suite.log_folder_path
supported_parameter_set = set(self.testnode.process_manager supported_parameter_set = set(self.testnode.process_manager
.getSupportedParameterList(run_test_suite_path)) .getSupportedParameterList(run_test_suite_path))
def path(name, compat): # BBB def path(name, compat): # BBB
...@@ -163,12 +169,8 @@ class UnitTestRunner(object): ...@@ -163,12 +169,8 @@ class UnitTestRunner(object):
('--node_quantity', lambda: config['node_quantity']), ('--node_quantity', lambda: config['node_quantity']),
('--xvfb_bin', lambda: path('xvfb', 'xserver/bin/Xvfb')), ('--xvfb_bin', lambda: path('xvfb', 'xserver/bin/Xvfb')),
('--project_title', lambda: node_test_suite.project_title), ('--project_title', lambda: node_test_suite.project_title),
('--shared_part_list', lambda: os.pathsep.join( ('--shared_part_list', lambda: SLAPOS_TEST_SHARED_PART_LIST),
self._getSlapOSControler( ('--log_directory', lambda: SLAPOS_TEST_LOG_DIRECTORY),
node_test_suite.working_directory,
True
).shared_part_list)),
('--log_directory', lambda: node_test_suite.log_folder_path),
): ):
if option in supported_parameter_set: if option in supported_parameter_set:
invocation_list += option, value() invocation_list += option, value()
...@@ -195,6 +197,8 @@ class UnitTestRunner(object): ...@@ -195,6 +197,8 @@ class UnitTestRunner(object):
return s.replace(portal_url.encode('utf-8'), b'$DISTRIBUTOR_URL') return s.replace(portal_url.encode('utf-8'), b'$DISTRIBUTOR_URL')
self.testnode.process_manager.spawn(*invocation_list, PATH=PATH, self.testnode.process_manager.spawn(*invocation_list, PATH=PATH,
SLAPOS_TEST_SHARED_PART_LIST=SLAPOS_TEST_SHARED_PART_LIST,
SLAPOS_TEST_LOG_DIRECTORY=SLAPOS_TEST_LOG_DIRECTORY,
cwd=node_test_suite.test_suite_directory, cwd=node_test_suite.test_suite_directory,
log_prefix='runTestSuite', log_prefix='runTestSuite',
output_replacers=(hide_distributor_url,), output_replacers=(hide_distributor_url,),
......
...@@ -74,7 +74,8 @@ def main(*args): ...@@ -74,7 +74,8 @@ def main(*args):
'proxy_port', 'git_binary','zip_binary','node_quantity', 'proxy_port', 'git_binary','zip_binary','node_quantity',
'test_node_title', 'ipv4_address','ipv6_address','test_suite_master_url', 'test_node_title', 'ipv4_address','ipv6_address','test_suite_master_url',
'slapos_binary', 'httpd_ip', 'httpd_port', 'httpd_software_access_port', 'slapos_binary', 'httpd_ip', 'httpd_port', 'httpd_software_access_port',
'computer_id', 'server_url', 'shared_part_list', 'keep_log_days'): 'computer_id', 'server_url', 'shared_part_list', 'keep_log_days',
'frontend_url', 'log_frontend_url', ):
CONFIG[key] = config.get('testnode',key) CONFIG[key] = config.get('testnode',key)
for key in ('slapos_directory', 'working_directory', 'test_suite_directory', for key in ('slapos_directory', 'working_directory', 'test_suite_directory',
...@@ -87,7 +88,6 @@ def main(*args): ...@@ -87,7 +88,6 @@ def main(*args):
CONFIG['httpd_url'] = 'https://[%s]:%s' % (CONFIG['httpd_ip'], CONFIG['httpd_url'] = 'https://[%s]:%s' % (CONFIG['httpd_ip'],
CONFIG['httpd_port']) CONFIG['httpd_port'])
CONFIG['system_temp_folder'] = "/tmp" CONFIG['system_temp_folder'] = "/tmp"
CONFIG['frontend_url'] = config.get('testnode', 'frontend_url')
# generate vcs_repository_list # generate vcs_repository_list
if 'bot_environment' in config.sections(): if 'bot_environment' in config.sections():
......
...@@ -28,6 +28,7 @@ import os ...@@ -28,6 +28,7 @@ import os
import json import json
import time import time
import logging import logging
from six.moves.urllib.parse import urljoin
from contextlib import contextmanager from contextlib import contextmanager
from slapos.slap.slap import ConnectionError from slapos.slap.slap import ConnectionError
from . import logger, log_formatter from . import logger, log_formatter
...@@ -341,8 +342,8 @@ shared = true ...@@ -341,8 +342,8 @@ shared = true
self.cleanUp() # XXX not a good place to do that self.cleanUp() # XXX not a good place to do that
continue continue
with self.suiteLog(node_test_suite) as suite_log_folder_name: with self.suiteLog(node_test_suite) as suite_log_folder_name:
test_result.reportStatus('LOG url', "%s/%s" % ( test_result.reportStatus(
config.get('httpd_url'), suite_log_folder_name), '') 'LOG url', urljoin(config['log_frontend_url'], suite_log_folder_name), '')
git_gc_auto = self.checkRevision(test_result, node_test_suite) git_gc_auto = self.checkRevision(test_result, node_test_suite)
node_test_suite.edit(test_result=test_result) node_test_suite.edit(test_result=test_result)
# get cluster configuration for this test suite, this is needed to # get cluster configuration for this test suite, this is needed to
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment