Testnode environment variables and log URL

- use environment variables to pass SLAPOS_TEST_LOG_DIRECTORY and SLAPOS_TEST_SHARED_PART_LIST, as discussed on nexedi/nxdtest!2
- Fix #20200514-218C705 - [testnode] frontend for log access

See merge request nexedi/erp5!1304

erp5/tests/testERP5TestNode.py

@@ -95,6 +95,7 @@ class ERP5TestNode(TestCase):
     config["httpd_ip"] = "ff:ff:ff:ff:ff:ff:ff:ff"
     config["httpd_software_access_port"] = "9080"
     config["frontend_url"] = "http://frontend/"
+    config["log_frontend_url"] = "http://log-frontend/"
     config["software_list"] = ["foo", "bar"]
     config["partition_reference"] = "part"
     config["ipv4_address"] = "1.2.3.4"
@@ -620,6 +621,7 @@ shared = true
 
   def test_distributor_url_obfuscated_in_logs(self):
     test_node = self.getTestNode()
+    del test_node.suiteLog
 
     runner = test_type_registry['UnitTest'](test_node)
     node_test_suite = test_node.getNodeTestSuite('foo')
@@ -635,7 +637,8 @@ shared = true
         """.format(temp_file.name))
       os.chmod(path, 0o700)
 
-      with mock.patch.object(logger, '_log') as http_logger,\
+      with test_node.suiteLog(node_test_suite),\
+          mock.patch.object(logger, '_log') as http_logger,\
           mock.patch.object(logging.getLogger(), '_log') as root_logger,\
           mock.patch.object(logging.getLogger(), 'isEnabledFor', return_value=True):
         runner.runTestSuite(node_test_suite, "https://user:secret@example.com/portal_distributions")
@@ -676,7 +679,8 @@ shared = true
       call_parameter_list.append(args)
 
     test_node = self.getTestNode()
-    with mock.patch.object(test_node.process_manager, 'spawn', side_effect=spawn):
+    del test_node.suiteLog
+    with mock.patch.object(test_node.process_manager, 'spawn', side_effect=spawn) as spawn_mock:
       runner = test_type_registry[my_test_type](test_node)
       # Create and initialise/regenerate a nodetestsuite
       node_test_suite = test_node.getNodeTestSuite('foo')
@@ -695,9 +699,16 @@ shared = true
         '--test_suite_title', 'Foo-Test',
       ]
       def checkRunTestSuiteParameters():
+        with test_node.suiteLog(node_test_suite) as suite_log:
           runner.runTestSuite(node_test_suite, "http://foo.bar")
         self.assertEqual(list(call_parameter_list.pop()), expected_parameter_list)
         self.assertFalse(call_parameter_list)
+        self.assertEqual(
+            spawn_mock.call_args[-1]['SLAPOS_TEST_SHARED_PART_LIST'],
+            "/not/exists:/not/exists_either:%s/shared" % node_test_suite.working_directory)
+        self.assertEqual(
+            spawn_mock.call_args[-1]['SLAPOS_TEST_LOG_DIRECTORY'],
+            os.path.join(test_node.config['log_directory'], suite_log))
 
       checkRunTestSuiteParameters()
 
@@ -1005,7 +1016,7 @@ shared = true
                 logger, test_result_path, node_title, revision)
     def patch_runTestSuite(self,*argv, **kw):
       return {'status_code': 0}
-    def checkTestSuite(test_node):
+    def checkTestSuite(test_node, reportTaskStatus_mock):
       test_node.node_test_suite_dict
       rand_part_set = set()
       self.assertEqual(2, len(test_node.node_test_suite_dict))
@@ -1015,6 +1026,14 @@ shared = true
                          "Incorrect suite log path : %r" % suite.suite_log_path)
         m = re.search('-(.*)/suite.log$', suite.suite_log_path)
         rand_part = m.groups()[0]
+        reportTaskStatus_mock.assert_has_calls(
+            [mock.call(
+                mock.ANY,
+                {'command': 'LOG url',
+                'stdout': "http://log-frontend/{}-{}".format(suite.reference, rand_part),
+                'stderr': ''},
+                "Foo-Test-Node")])
+
         self.assertEqual(len(rand_part), 10)
         self.assertNotIn(rand_part, rand_part_set)
         rand_part_set.add(rand_part)
@@ -1071,9 +1090,11 @@ shared = true
       return {'status_code': 0}
     RunnerClass._prepareSlapOS = patch_prepareSlapOS
     SlapOSControler.initializeSlapOSControler = doNothing
+    with mock.patch('erp5.util.taskdistribution.DummyTaskDistributor.reportTaskStatus') as reportTaskStatus:
       test_node.run()
+
     self.assertEqual(counter, 3)
-    checkTestSuite(test_node)
+    checkTestSuite(test_node, reportTaskStatus)
     time.sleep = original_sleep
     # Restore old class methods
     if my_test_type == "ScalabilityTest":

erp5/util/testnode/UnitTestRunner.py

@@ -150,6 +150,12 @@ class UnitTestRunner(object):
     software_list = [soft + md5digest(x) for x in config['software_list']]
     PATH = os.getenv('PATH', '')
     PATH = ':'.join(x + '/bin' for x in software_list) + (PATH and ':' + PATH)
+    SLAPOS_TEST_SHARED_PART_LIST = os.pathsep.join(
+        self._getSlapOSControler(
+            node_test_suite.working_directory,
+            True
+        ).shared_part_list)
+    SLAPOS_TEST_LOG_DIRECTORY = node_test_suite.log_folder_path
     supported_parameter_set = set(self.testnode.process_manager
       .getSupportedParameterList(run_test_suite_path))
     def path(name, compat): # BBB
@@ -163,12 +169,8 @@ class UnitTestRunner(object):
         ('--node_quantity', lambda: config['node_quantity']),
         ('--xvfb_bin', lambda: path('xvfb', 'xserver/bin/Xvfb')),
         ('--project_title', lambda: node_test_suite.project_title),
-        ('--shared_part_list', lambda: os.pathsep.join(
-            self._getSlapOSControler(
-                node_test_suite.working_directory,
-                True
-            ).shared_part_list)),
-        ('--log_directory', lambda: node_test_suite.log_folder_path),
+        ('--shared_part_list', lambda: SLAPOS_TEST_SHARED_PART_LIST),
+        ('--log_directory', lambda: SLAPOS_TEST_LOG_DIRECTORY),
         ):
       if option in supported_parameter_set:
         invocation_list += option, value()
@@ -195,6 +197,8 @@ class UnitTestRunner(object):
       return s.replace(portal_url.encode('utf-8'), b'$DISTRIBUTOR_URL')
 
     self.testnode.process_manager.spawn(*invocation_list, PATH=PATH,
+                          SLAPOS_TEST_SHARED_PART_LIST=SLAPOS_TEST_SHARED_PART_LIST,
+                          SLAPOS_TEST_LOG_DIRECTORY=SLAPOS_TEST_LOG_DIRECTORY,
                           cwd=node_test_suite.test_suite_directory,
                           log_prefix='runTestSuite',
                           output_replacers=(hide_distributor_url,),

erp5/util/testnode/__init__.py

@@ -74,7 +74,8 @@ def main(*args):
               'proxy_port', 'git_binary','zip_binary','node_quantity',
               'test_node_title', 'ipv4_address','ipv6_address','test_suite_master_url',
               'slapos_binary', 'httpd_ip', 'httpd_port', 'httpd_software_access_port',
-              'computer_id', 'server_url', 'shared_part_list', 'keep_log_days'):
+              'computer_id', 'server_url', 'shared_part_list', 'keep_log_days',
+              'frontend_url', 'log_frontend_url', ):
     CONFIG[key] = config.get('testnode',key)
 
   for key in ('slapos_directory', 'working_directory', 'test_suite_directory',
@@ -87,7 +88,6 @@ def main(*args):
   CONFIG['httpd_url'] = 'https://[%s]:%s' % (CONFIG['httpd_ip'],
         CONFIG['httpd_port'])
   CONFIG['system_temp_folder'] = "/tmp"
-  CONFIG['frontend_url'] = config.get('testnode', 'frontend_url')
 
   # generate vcs_repository_list
   if 'bot_environment' in config.sections():

erp5/util/testnode/testnode.py

@@ -28,6 +28,7 @@ import os
 import json
 import time
 import logging
+from six.moves.urllib.parse import urljoin
 from contextlib import contextmanager
 from slapos.slap.slap import ConnectionError
 from . import logger, log_formatter
@@ -341,8 +342,8 @@ shared = true
               self.cleanUp() # XXX not a good place to do that
               continue
             with self.suiteLog(node_test_suite) as suite_log_folder_name:
-              test_result.reportStatus('LOG url', "%s/%s" % (
-                config.get('httpd_url'), suite_log_folder_name), '')
+              test_result.reportStatus(
+                  'LOG url', urljoin(config['log_frontend_url'], suite_log_folder_name), '')
               git_gc_auto = self.checkRevision(test_result, node_test_suite)
               node_test_suite.edit(test_result=test_result)
               # get cluster configuration for this test suite, this is needed to