Turn go/neo/t/nxd/runTestSuite into -> nxdtest

Make the tool generic: load list of testcases to run from .nxdtest file
and process the correspondingly. See added top-level documentation for
details.

nxdtest

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright (C) 2018  Nexedi SA and Contributors.
+# Copyright (C) 2018-2020  Nexedi SA and Contributors.
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -17,21 +17,79 @@
 #
 # See COPYING file for full licensing terms.
 # See https://www.nexedi.com/licensing for rationale and options.
-"""runTestSuite - run neotest under Nexedi testing infrastructure.
+"""nxdtest - tox-like tool to run tests under Nexedi testing infrastructure(*).
 
-neotest must be on $PATH.
+Nxdtest runs tests defined by .nxdtest file.
+A project defines set of tests cases to verify itself in that file.
+.nxdtest file is Python program executed in special environment described below.
+
+Test cases are declared with `TestCase`. Each test case specifies:
+
+- its name,
+- a program to run,
+- (optionally) an environment for the run,
+- (optionally) a summary function to extract summary from test output.
+
+For example the following .nxdtest defines 3 test cases that run Wendelin.core
+`test.py` tests with FileStorage, ZEO and NEO as database backend::
+
+    for stor in ['fs1', 'zeo', 'neo']:
+        TestCase('test.py/%s' % stor, ['make', 'test.py'],
+                 env={'WENDELIN_CORE_TEST_DB': '<%s>' % stor,
+                 summaryf=PyTest.summary)
+
+Nxdtest only runs tests, but - unlike tox - does not prepare variants of
+software build. Nxdtest assumes that the software build is already fully
+prepared. This matches SlapOS environment, where software building is separate
+step handled by SlapOS.
+
+(*) https://www.erp5.com/NXD-Presentation.ci.testing.system.buildout
+    https://www.erp5.com/erp5-Guideline.Nexedi.Testing.Extended
+    https://stack.nexedi.com/test_status
 """
 
+from __future__ import print_function, absolute_import
+
 from erp5.util.taskdistribution import TaskDistributor
 from subprocess import Popen, PIPE
 from time import time, strftime, gmtime
 import os, sys, threading, argparse, logging, traceback, re
-
+import six
+
+# loadNXDTestFile loads .nxdtest file located @path.
+def loadNXDTestFile(path): # -> TestEnv
+    t = TestEnv()
+    g = {'TestCase': t.TestCase}    # TODO + all other public TestEnv methods
+    with open(path, "r") as f:
+        src = f.read()
+    six.exec_(src, g)
+    return t
+
+# TestCase defines one test case to run.
+class TestCase:
+    def __init__(self, name, argv, summaryf=None, **kw):
+        self.name = name            # testcase name
+        self.argv = argv            # program to run
+        self.kw   = kw              # **kw is passed to Popen
+        self.summaryf = summaryf    # function to extract summary from test output
+
+# TestEnv represents a testing environment with set of TestCases to run.
+class TestEnv:
+    def __init__(self):
+        self.byname = {} # name -> TestCase
+        self.testv  = [] # of TestCase
+
+    # TestCase adds new test case to the environment.
+    def TestCase(self, name, argv, **kw):
+        assert name not in self.byname
+        t = TestCase(name, argv, **kw)
+        self.testv.append(t)
+        self.byname[name] = t
 
 def main():
     # testnode executes us giving URL to master results collecting instance and other details
     # https://lab.nexedi.com/nexedi/erp5/blob/744f3fde/erp5/util/testnode/UnitTestRunner.py#L137
-    parser = argparse.ArgumentParser(description=__doc__)
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
     parser.add_argument('--master_url', help='The URL of Master controling many suites')
     parser.add_argument('--revision', help='The revision to test', default='dummy_revision')
     parser.add_argument('--test_suite', help='The test suite name')
@@ -49,11 +107,14 @@ def main():
         logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
         logger = logging.getLogger()
 
+    # load list of tests to run
+    tenv = loadNXDTestFile('.nxdtest')
+
     # connect to master and create 'test result' object with list of tests to run
     tool = TaskDistributor(portal_url = args.master_url, logger = logger)
     test_result = tool.createTestResult(
                     revision        = args.revision,
-                    test_name_list  = ['test-go', 'test-py', 'bench-local'],
+                    test_name_list  = [t.name for t in tenv.testv],
                     node_title      = args.test_node_title,
                     test_title      = args.test_suite_title or args.test_suite,
                     project_title   = args.project_title)
@@ -75,15 +136,15 @@ def main():
         if test_result_line is None:
             break
 
-        # run `neotest <test-name>`
-        testname = test_result_line.name
-        argv = ['neotest', testname]
+        # run tenv[name]
+        t = tenv.byname[test_result_line.name]
         tstart = time()
 
         try:
-            # NOTE runs with unchanged cwd. Instance wrapper cares to set cwd before running us.
+            # Run t.argv in t.kw['env'] environment.
+            # Test command is spawned with unchanged cwd. Instance wrapper cares to set cwd before running us.
             # bufsize=1 means 'line buffered'
-            p = Popen(argv, stdin=devnull, stdout=PIPE, stderr=PIPE, bufsize=1)
+            p = Popen(t.argv, stdin=devnull, stdout=PIPE, stderr=PIPE, bufsize=1, **t.kw)
         except:
             stdout, stderr = '', traceback.format_exc()
             sys.stderr.write(stderr)
@@ -113,10 +174,9 @@ def main():
             }
 
             # postprocess output, if we can
-            summaryf = globals().get(testname.replace('-', '_') + '_summary')
-            if summaryf is not None:
+            if t.summaryf is not None:
                 try:
-                    summary = summaryf(stdout)
+                    summary = t.summaryf(stdout)
                 except:
                     bad = traceback.format_exc()
                     sys.stderr.write(bad)
@@ -131,7 +191,7 @@ def main():
 
         # report result of test run back to master
         test_result_line.stop(
-            command     = ' '.join(argv),
+            command     = ' '.join(t.argv),
             duration    = tend - tstart,
             date        = strftime("%Y/%m/%d %H:%M:%S", gmtime(tend)),
 

setup.py

@@ -13,7 +13,7 @@ setup(
     keywords    = 'Nexedi testing infrastructure tool tox',
 
     packages    = [],
-    install_requires = ['erp5.util'],
+    install_requires = ['erp5.util', 'six'],
 
     scripts     = ['nxdtest'],