Testnode: Expose log folder

Test nodes already share a folder with a `suite.log` (and a quick and dirty application to view the log live).

The path of this folder is now passed to test runner so that they can also put some log files there. 

ERP5 test runner uses it by writing the `Z2.log` and `zLOG.log` on the running tests there. 

Slapos SR tests also uses it.

See merge request !948

erp5/util/testnode/NodeTestSuite.py

@@ -75,9 +75,9 @@ class NodeTestSuite(SlapOSInstance):
     while 1:
       log_folder_name = '%s-%s' % (self.reference,
         ''.join(random.choice(alphabets) for i in range(10)))
-      log_folder_path = os.path.join(self.log_directory, log_folder_name)
+      self.log_folder_path = os.path.join(self.log_directory, log_folder_name)
       try:
-        os.makedirs(log_folder_path)
+        os.makedirs(self.log_folder_path)
       except OSError as e:
         if e.errno != errno.EEXIST:
           raise
@@ -85,8 +85,8 @@ class NodeTestSuite(SlapOSInstance):
         break
     # XXX copy the whole content of the log viewer app
     for fname in glob.glob(os.path.join(os.path.dirname(__file__), 'js-logtail', '*')):
-      shutil.copy(fname, log_folder_path)
-    self.suite_log_path = os.path.join(log_folder_path, 'suite.log')
+      shutil.copy(fname, self.log_folder_path)
+    self.suite_log_path = os.path.join(self.log_folder_path, 'suite.log')
     return self.suite_log_path, log_folder_name
 
   @property

erp5/util/testnode/UnitTestRunner.py

@@ -165,6 +165,7 @@ class UnitTestRunner(object):
                 node_test_suite.working_directory,
                 True
             ).shared_part_list)),
+        ('--log_directory', lambda: node_test_suite.log_folder_path),
         ):
       if option in supported_parameter_set:
         invocation_list += option, value()

erp5/util/testnode/__init__.py

@@ -74,7 +74,7 @@ def main(*args):
               'proxy_port', 'git_binary','zip_binary','node_quantity',
               'test_node_title', 'ipv4_address','ipv6_address','test_suite_master_url',
               'slapos_binary', 'httpd_ip', 'httpd_port', 'httpd_software_access_port',
-              'computer_id', 'server_url', 'shared_part_list'):
+              'computer_id', 'server_url', 'shared_part_list', 'keep_log_days'):
     CONFIG[key] = config.get('testnode',key)
 
   for key in ('slapos_directory', 'working_directory', 'test_suite_directory',

erp5/util/testnode/js-logtail/logtail.html

@@ -14,6 +14,7 @@
             <a href="logtail.html?noreverse">chronological</a> view.
             <a id="pause" href='#'>Pause</a>.
             <a href="suite.log">Full log file</a> or
+            <a href="./">snapshots directory</a>
         </div>
         <pre id="data" style="font-size: xx-small;">Loading...</pre>
     </body>

erp5/util/testnode/js-logtail/logtail.js

@@ -120,10 +120,10 @@ function get_log() {
                 log_data = "";
                 show_log();
 
-                setTimeout(get_log, poll);
             } else {
-                throw "Unknown AJAX Error (status " + xhr.status + ")";
+                console.error("Unknown AJAX Error (status " + xhr.status + ")");
             }
+            setTimeout(get_log, poll);
         }
     });
 }

erp5/util/testnode/testnode.py

@@ -41,8 +41,6 @@ from .Utils import deunicodeData
 from .Utils import rmtree
 from .. import taskdistribution
 
-MAX_LOG_TIME = 15 # time in days we should keep logs that we can see through
-                  # httd
 MAX_TEMP_TIME = 0.01 # time in days we should keep temp files
 
 PROFILE_PATH_KEY = 'profile_path'
@@ -54,14 +52,13 @@ test_type_registry = {
 
 class TestNode(object):
 
-  def __init__(self, config, max_log_time=MAX_LOG_TIME,
-               max_temp_time=MAX_TEMP_TIME):
+  def __init__(self, config, max_temp_time=MAX_TEMP_TIME):
     self.config = config or {}
     self.process_manager = ProcessManager()
     self.working_directory = config['working_directory']
     self.node_test_suite_dict = {}
     self.file_handler = None
-    self.max_log_time = max_log_time
+    self.max_log_time = float(config.get('keep_log_days', 15))
     self.max_temp_time = max_temp_time
     self.url_access = "https://[0::0]:0123" # Ipv6 + port of the node
 

erp5/util/testsuite/__init__.py

@@ -248,7 +248,8 @@ class EggTestSuite(TestSuite):
       status_dict = self.spawn(
           self.python_interpreter, 'setup.py', 'test',
           cwd=self.egg_test_path_dict[test],
-          SLAPOS_TEST_SHARED_PART_LIST=self.shared_part_list)
+          SLAPOS_TEST_SHARED_PART_LIST=self.shared_part_list,
+          SLAPOS_TEST_LOG_DIRECTORY=self.log_directory)
     except SubprocessError as e:
       status_dict = e.status_dict
     test_log = status_dict['stderr']
@@ -298,6 +299,9 @@ def runTestSuite():
   parser.add_argument('--frontend_url',
                       help='The url of the frontend of this test node',
                       default=None)
+  parser.add_argument('--log_directory',
+                      help='Directory to store logs',
+                      default=None)
   parser.add_argument('--python_interpreter',
                       help='Path to python interpreter used to run the test suite',
                       default='python')
@@ -329,7 +333,8 @@ def runTestSuite():
                     revision=revision,
                     python_interpreter=args.python_interpreter,
                     egg_test_path_dict=egg_test_path_dict,
-                    shared_part_list=args.shared_part_list
+                    shared_part_list=args.shared_part_list,
+                    log_directory=args.log_directory,
                     )
 
   test_result = master.createTestResult(revision, suite.getTestList(),

product/ERP5Type/tests/ERP5TypeTestSuite.py

@@ -22,7 +22,13 @@ class ERP5TypeTestSuite(TestSuite):
     instance_home = self.instance and 'unit_test.%u' % self.instance \
                                    or 'unit_test'
     if self.instance:
-      args = ('--instance_home=' + instance_home, ) + args
+      instance_home = 'unit_test.%u' % self.instance
+      args = ('--instance_home', instance_home,) + args
+      if self.log_directory:
+        log_directory = os.path.join(self.log_directory, args[-1].replace(':', '_'))
+        os.mkdir(log_directory)
+        args = ('--log_directory', log_directory, ) + args
+
     if self.__dict__.has_key("bt5_path"):
       args = ("--bt5_path=%s" % self.bt5_path,) + args
     instance_number = self.instance or 1

product/ERP5Type/tests/ProcessingNodeTestCase.py

@@ -149,8 +149,8 @@ class ProcessingNodeTestCase(ZopeTestCase.TestCase):
     """Start HTTP ZServer in background"""
     utils = ZopeTestCase.utils
     if utils._Z2HOST is None:
-      from Products.ERP5Type.tests.runUnitTest import tests_home
-      log = os.path.join(tests_home, "Z2.log")
+      from Products.ERP5Type.tests.runUnitTest import log_directory
+      log = os.path.join(log_directory, "Z2.log")
       message = "Running %s server at %s:%s\n"
       if int(os.environ.get('erp5_wsgi', 0)):
         from Products.ERP5.bin.zopewsgi import app_wrapper, createServer

product/ERP5Type/tests/runTestSuite.py

@@ -103,6 +103,7 @@ def main():
                       default=None)
   testnode_group.add_argument("--xvfb_bin", default=None)
   testnode_group.add_argument("--firefox_bin", default=None)
+  testnode_group.add_argument("--log_directory", default=None)
 
   args = parser.parse_args()
   if args.bt5_path is not None:
@@ -132,7 +133,8 @@ def main():
                     zserver_frontend_url_list=args.zserver_frontend_url_list,
                     bt5_path=args.bt5_path,
                     firefox_bin=args.firefox_bin,
-                    xvfb_bin=args.xvfb_bin)
+                    xvfb_bin=args.xvfb_bin,
+                    log_directory=args.log_directory)
   test_result = master.createTestResult(revision, suite.getTestList(),
     args.test_node_title, suite.allow_restart, test_suite_title,
     args.project_title)

product/ERP5Type/tests/runUnitTest.py

@@ -158,6 +158,7 @@ Options:
   --sys_path=path,path       Comma-separated list of paths which will be used to
                              extend sys.path
   --instance_home=PATH       Create/use test instance in given path
+  --log_directory=PATH       Create log files in given path
 
 When no unit test is specified, only activities are processed.
 """
@@ -492,7 +493,7 @@ def runUnitTestList(test_list, verbosity=1, debug=0, run_only=None):
     _print("conflicting options: --zeo_client and --zeo_server")
     sys.exit(1)
   instance_home =  os.environ['INSTANCE_HOME']
-  os.environ.setdefault('EVENT_LOG_FILE', os.path.join(tests_home, 'zLOG.log'))
+  os.environ.setdefault('EVENT_LOG_FILE', os.path.join(log_directory, 'zLOG.log'))
   os.environ.setdefault('EVENT_LOG_SEVERITY', '-300')
   # For numpy parallel-primitives such as numpy.dot() that is used in
   # testReceiptRecognition for example.
@@ -759,6 +760,7 @@ def main(argument_list=None):
         "products_path=",
         "sys_path=",
         "instance_home=",
+        "log_directory="
         ])
   except getopt.GetoptError, msg:
     usage(sys.stderr, msg)
@@ -769,6 +771,7 @@ def main(argument_list=None):
   debug = 0
   run_only = None
   instance_home = os.path.join(real_instance_home, 'unit_test')
+  _log_directory = instance_home
 
   bt5_path_list = []
 
@@ -877,6 +880,8 @@ def main(argument_list=None):
       sys.path.extend(arg.split(','))
     elif opt == "--instance_home":
       instance_home = os.path.abspath(arg)
+    elif opt == "--log_directory":
+      _log_directory = os.path.abspath(arg)
 
   bt5_path_list += filter(None,
     os.environ.get("erp5_tests_bt5_path", "").split(','))
@@ -895,6 +900,8 @@ def main(argument_list=None):
   initializeInstanceHome(tests_framework_home,
                          real_instance_home,
                          instance_home)
+  global log_directory
+  log_directory = _log_directory or test_home
 
   result = runUnitTestList(test_list=args,
                            verbosity=verbosity,