ERP5 Coverage

depends on nexedi/erp5!1695

See merge request nexedi/slapos!1290

software/erp5/instance-erp5-input-schema.json

@@ -525,6 +525,46 @@
           "default": true,
           "type": "boolean"
         },
+        "coverage": {
+          "type": "object",
+          "title": "Coverage",
+          "description": "Coverage configuration",
+          "additionalProperties": false,
+          "properties": {
+            "enabled": {
+              "description": "Collect python coverage data during test run.",
+              "default": false,
+              "type": "boolean"
+            },
+            "include": {
+              "description": "File name patterns to include in coverage data, relative to software buildout's directory. Default to all repositories defined in software by ${erp5_repository_list:repository_id_list}.",
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "examples": [
+                [
+                  "parts/erp5/*",
+                  "parts/custom-repository/*",
+                  "develop-eggs/custom-egg/*"
+                ]
+              ]
+            },
+            "branch": {
+              "description": "Enable branch coverage",
+              "type": "boolean",
+              "default": false
+            },
+            "upload-url": {
+              "description": "URL to upload coverage data. This is interpreted as a RFC 6570 URI Template, with the following parameters: test_name, test_result_id and test_result_revision. The request will be a PUT request with the coverage file content as body, suitable for WebDav servers. If the URL contains user and password, they will be used to attempt authentication using Digest and Basic authentication schemes.",
+              "type": "string",
+              "format": "uri",
+              "examples": [
+                "https://user:password@example.com/{test_result_id}/{test_name}.coverage.sqlite3"
+              ]
+            }
+          }
+        },
         "node-count": {
           "description": "Number of tests this instance can execute in parallel. This must be at least equal to the number of nodes configured on testnode running the test",
           "default": 3,

software/neoppod/software-common.cfg

@@ -125,7 +125,7 @@ inline =
   exec "$basedir/bin/mysqld" --defaults-file='{{defaults_file}}' "$@"
 
 [versions]
-coverage = 4.5.1
+coverage = 5.5
 ecdsa = 0.13
 mysqlclient = 1.3.12
 pycrypto = 2.6.1

stack/erp5/buildout.cfg

@@ -72,6 +72,7 @@ parts +=
   mroonga-mariadb
   tesseract
   zabbix-agent
+  .coveragerc
 
 # Buildoutish
   eggs-all-scripts
@@ -351,14 +352,52 @@ entry-points =
   runUnitTest=runUnitTest:main
 scripts = runUnitTest
 initialization =
-  import glob, os, sys
+  import glob, os, sys, json
+  buildout_directory = '''${buildout:directory}'''
+  parts_directory = '''${buildout:parts-directory}'''
+  repository_id_list = \
+    '''${erp5_repository_list:repository_id_list}'''.split()[::-1]
+
+  # read testrunner configuration from slapos instance parameters to
+  # configure coverage if enabled.
+  with open(os.environ['ERP5_TEST_RUNNER_CONFIGURATION']) as f:
+    test_runner_configuration = json.load(f)
+  test_runner_configuration.setdefault('coverage', {})
+  test_runner_configuration['coverage'].setdefault('enabled', False)
+
+  coverage_process = None
+  if test_runner_configuration['coverage']['enabled']:
+    test_runner_configuration['coverage'].setdefault(
+      'include', [os.path.join('parts', repo, '*') for repo in repository_id_list])
+
+    assets_directory = ''
+    test_name = sys.argv[-1].replace(':', '_')
+    if os.environ.get('SLAPOS_TEST_LOG_DIRECTORY'):
+      assets_directory = os.path.join(os.environ['SLAPOS_TEST_LOG_DIRECTORY'], test_name)
+      if not os.path.exists(assets_directory):
+        os.makedirs(assets_directory)
+    coverage_data_file = os.path.abspath(
+      os.path.join(assets_directory, 'coverage.sqlite3'))
+
+    curdir = os.path.abspath(os.curdir)
+    # change current directory when importing coverage so that it considers paths
+    # relative to the root of the software
+    os.chdir(buildout_directory)
+    import coverage
+    coverage_process = coverage.Coverage(
+      include=test_runner_configuration['coverage']['include'],
+      data_file=coverage_data_file,
+      branch=test_runner_configuration['coverage'].get('branch'),
+    )
+    coverage_process.set_option('run:relative_files', 'true')
+    coverage_process.set_option('run:plugins', ['erp5_coverage_plugin'])
+    coverage_process.start()
+    os.chdir(curdir)
+
   import Products
   Products.__path__[:0] = filter(None,
     os.getenv('INSERT_PRODUCTS_PATH', '').split(os.pathsep))
   os.environ['ZOPE_SCRIPTS'] = ''
-  parts_directory = '''${buildout:parts-directory}'''
-  repository_id_list = \
-    '''${erp5_repository_list:repository_id_list}'''.split()[::-1]
   os.environ['erp5_tests_bt5_path'] = ','.join(
     os.path.join(parts_directory, x, 'bt5') for x in repository_id_list)
   extra_path_list = '''${:extra-paths}'''.split()
@@ -371,6 +410,59 @@ initialization =
   sys.path[:0] = sum((
     glob.glob(os.path.join(x, 'Products', '*', 'tests'))
     for x in os.getenv('INSERT_PRODUCTS_PATH', '').split(os.pathsep)), [])
+  import runUnitTest
+  try:
+    sys.exit(runUnitTest.main())
+  finally:
+    if coverage_process:
+      coverage_process.stop()
+      coverage_process.save()
+      # upload the coverage so that they can be combined from another machine
+      upload_url = test_runner_configuration['coverage'].get('upload-url')
+      if upload_url:
+        import requests
+        import time
+        import uritemplate
+        from six.moves.urllib.parse import urlparse
+
+        auth_list = (None, )
+        parsed_url = urlparse(upload_url)
+        if parsed_url.username:
+          # try Digest and Basic authentication and retry 5 times to tolerate transiant errors
+          auth_list = (
+            requests.auth.HTTPDigestAuth(parsed_url.username, parsed_url.password),
+            requests.auth.HTTPBasicAuth(parsed_url.username, parsed_url.password),
+          ) *  5
+
+        url = uritemplate.URITemplate(upload_url).expand(
+          test_name=test_name,
+          # Environment variables are set in parts/erp5/product/ERP5Type/tests/runTestSuite.py
+          test_result_id=os.environ.get('ERP5_TEST_RESULT_ID', 'unknown_test_result_id'),
+          test_result_revision=os.environ.get('ERP5_TEST_RESULT_REVISION', 'unknown_test_result_revision'),
+        )
+
+        for auth in auth_list:
+          with open(coverage_data_file, 'rb') as f:
+            resp = requests.put(url, data=f, auth=auth)
+            if resp.ok:
+              # print just the hostname, not to include the auth part
+              print('Uploaded coverage data to {parsed_url.hostname}'.format(parsed_url=parsed_url))
+              break
+            print('Error {resp.status_code} uploading coverage data to {parsed_url.hostname} with {auth.__class__.__name__}'.format(
+              resp=resp, parsed_url=parsed_url, auth=auth))
+            time.sleep(1)
+        else:
+          sys.stderr.write('Error uploading coverage data to {parsed_url.hostname}\n'.format(parsed_url=parsed_url))
+
+[.coveragerc]
+recipe = slapos.recipe.template
+output = ${buildout:directory}/${:_buildout_section_name_}
+inline =
+  # coverage configuration file, useful when making html report
+  [run]
+  plugins =
+    erp5_coverage_plugin
+  relative_files = true
 
 [test-suite-runner]
 # XXX: Workaround for fact ERP5Type is not an distribution and does not
@@ -466,7 +558,6 @@ eggs = ${neoppod:eggs}
   SOAPpy
   chardet
   collective.recipe.template
-  coverage
   erp5diff
   interval
   ipdb
@@ -606,6 +697,12 @@ Acquisition-patch-options = -p1
 python-magic-patches = ${:_profile_base_location_}/../../component/egg-patch/python_magic/magic.patch#de0839bffac17801e39b60873a6c2068
 python-magic-patch-options = -p1
 
+
+# neoppod installs bin/coverage so we inject erp5 plugin here so that coverage script can use it in report
+[neoppod]
+eggs +=
+  erp5_coverage_plugin
+
 [eggs-all-scripts]
 recipe = zc.recipe.egg
 eggs =
@@ -683,6 +780,7 @@ PyStemmer = 1.3.0
 Pympler = 0.4.3
 StructuredText = 2.11.1
 WSGIUtils = 0.7
+erp5-coverage-plugin = 0.0.1
 erp5diff = 0.8.1.8
 five.formlib = 1.0.4
 five.localsitemanager = 2.0.5