Commit 3d8deba0 authored by Jérome Perrin's avatar Jérome Perrin

stack/erp5: support coverage when running tests

This replaces the broken --coverage argument from runUnitTest, coverage
needs to be started earlier and also introduces a coverage plugin to
collect coverage data for code in ZODB; python scripts and components
are supported.

To use on test nodes, set up a web dav server somewhere, configure the
instance parameters of the test suite on ERP5 to enable coverage and
upload individual results to this webdav server and then combine the
coverage data and produce a report, using the bin/coverage script from
the software release.

For the steps below, it is necessary to change working directory to the
root of the software folder.

Step 1: combine the coverage data:

    ./bin/coverage combine \
      --keep \
      /path/to/all/coverage/files/*coverage.sqlite3

( using --keep is optional, but it helps in case of mistakes )

Step 2: build an html report:

   ./bin/coverage html \
     --skip-covered \
     --omit parts/erp5/product/ERP5/Document/UnitTest.py \
     --directory /path/for/html/report/

Note that we want to omit UnitTest.py that is created during test
(see testBusinessTemplate.TestDocumentTemplateItem) and get coverage
result because it is executed in the test, but it does not exist as a
source file in the repository, so we skip it.

Of course, to produce a correct html report from a test that have been
running on test nodes, it's necessary that the software release used to
produce the html report has exactly the same version ans the one that
has been running on test nodes.

Another simpler, but slower approach is to run all the tests on the same
machine, then only running step 2 is necessary.
parent 3ea53a6c
No related merge requests found
......@@ -525,6 +525,46 @@
"default": true,
"type": "boolean"
},
"coverage": {
"type": "object",
"title": "Coverage",
"description": "Coverage configuration",
"additionalProperties": false,
"properties": {
"enabled": {
"description": "Collect python coverage data during test run.",
"default": false,
"type": "boolean"
},
"include": {
"description": "File name patterns to include in coverage data, relative to software buildout's directory. Default to all repositories defined in software by ${erp5_repository_list:repository_id_list}.",
"type": "array",
"items": {
"type": "string"
},
"examples": [
[
"parts/erp5/*",
"parts/custom-repository/*",
"develop-eggs/custom-egg/*"
]
]
},
"branch": {
"description": "Enable branch coverage",
"type": "boolean",
"default": false
},
"upload-url": {
"description": "URL to upload coverage data. This is interpreted as a RFC 6570 URI Template, with the following parameters: test_name, test_result_id and test_result_revision. The request will be a PUT request with the coverage file content as body, suitable for WebDav servers. If the URL contains user and password, they will be used to attempt authentication using Digest and Basic authentication schemes.",
"type": "string",
"format": "uri",
"examples": [
"https://user:password@example.com/{test_result_id}/{test_name}.coverage.sqlite3"
]
}
}
},
"node-count": {
"description": "Number of tests this instance can execute in parallel. This must be at least equal to the number of nodes configured on testnode running the test",
"default": 3,
......
......@@ -72,6 +72,7 @@ parts +=
mroonga-mariadb
tesseract
zabbix-agent
.coveragerc
# Buildoutish
eggs-all-scripts
......@@ -351,14 +352,52 @@ entry-points =
runUnitTest=runUnitTest:main
scripts = runUnitTest
initialization =
import glob, os, sys
import glob, os, sys, json
buildout_directory = '''${buildout:directory}'''
parts_directory = '''${buildout:parts-directory}'''
repository_id_list = \
'''${erp5_repository_list:repository_id_list}'''.split()[::-1]
# read testrunner configuration from slapos instance parameters to
# configure coverage if enabled.
with open(os.environ['ERP5_TEST_RUNNER_CONFIGURATION']) as f:
test_runner_configuration = json.load(f)
test_runner_configuration.setdefault('coverage', {})
test_runner_configuration['coverage'].setdefault('enabled', False)
coverage_process = None
if test_runner_configuration['coverage']['enabled']:
test_runner_configuration['coverage'].setdefault(
'include', [os.path.join('parts', repo, '*') for repo in repository_id_list])
assets_directory = ''
test_name = sys.argv[-1].replace(':', '_')
if os.environ.get('SLAPOS_TEST_LOG_DIRECTORY'):
assets_directory = os.path.join(os.environ['SLAPOS_TEST_LOG_DIRECTORY'], test_name)
if not os.path.exists(assets_directory):
os.makedirs(assets_directory)
coverage_data_file = os.path.abspath(
os.path.join(assets_directory, 'coverage.sqlite3'))
curdir = os.path.abspath(os.curdir)
# change current directory when importing coverage so that it considers paths
# relative to the root of the software
os.chdir(buildout_directory)
import coverage
coverage_process = coverage.Coverage(
include=test_runner_configuration['coverage']['include'],
data_file=coverage_data_file,
branch=test_runner_configuration['coverage'].get('branch'),
)
coverage_process.set_option('run:relative_files', 'true')
coverage_process.set_option('run:plugins', ['erp5_coverage_plugin'])
coverage_process.start()
os.chdir(curdir)
import Products
Products.__path__[:0] = filter(None,
os.getenv('INSERT_PRODUCTS_PATH', '').split(os.pathsep))
os.environ['ZOPE_SCRIPTS'] = ''
parts_directory = '''${buildout:parts-directory}'''
repository_id_list = \
'''${erp5_repository_list:repository_id_list}'''.split()[::-1]
os.environ['erp5_tests_bt5_path'] = ','.join(
os.path.join(parts_directory, x, 'bt5') for x in repository_id_list)
extra_path_list = '''${:extra-paths}'''.split()
......@@ -371,6 +410,64 @@ initialization =
sys.path[:0] = sum((
glob.glob(os.path.join(x, 'Products', '*', 'tests'))
for x in os.getenv('INSERT_PRODUCTS_PATH', '').split(os.pathsep)), [])
import runUnitTest
try:
sys.exit(runUnitTest.main())
finally:
if coverage_process:
coverage_process.stop()
coverage_process.save()
# Make html report for this test
os.chdir(buildout_directory)
coverage_process.html_report(
directory=os.path.join(assets_directory, 'coverage_report'),
)
# upload the coverage so that they can be combined from another machine
upload_url = test_runner_configuration['coverage'].get('upload-url')
if upload_url:
import requests
import time
import uritemplate
from six.moves.urllib.parse import urlparse
auth_list = (None, )
parsed_url = urlparse(upload_url)
if parsed_url.username:
# try Digest and Basic authentication and retry 5 times to tolerate transiant errors
auth_list = (
requests.auth.HTTPDigestAuth(parsed_url.username, parsed_url.password),
requests.auth.HTTPBasicAuth(parsed_url.username, parsed_url.password),
) * 5
url = uritemplate.URITemplate(upload_url).expand(
test_name=test_name,
# Environment variables are set in parts/erp5/product/ERP5Type/tests/runTestSuite.py
test_result_id=os.environ.get('ERP5_TEST_RESULT_ID', 'unknown_test_result_id'),
test_result_revision=os.environ.get('ERP5_TEST_RESULT_REVISION', 'unknown_test_result_revision'),
)
for auth in auth_list:
with open(coverage_data_file, 'rb') as f:
resp = requests.put(url, data=f, auth=auth)
if resp.ok:
# print just the hostname, not to include the auth part
print('Uploaded coverage data to {parsed_url.hostname}'.format(parsed_url=parsed_url))
break
print('Error {resp.status_code} uploading coverage data to {parsed_url.hostname} with {auth.__class__.__name__}'.format(
resp=resp, parsed_url=parsed_url, auth=auth))
time.sleep(1)
else:
sys.stderr.write('Error uploading coverage data to {parsed_url.hostname}\n'.format(parsed_url=parsed_url))
[.coveragerc]
recipe = slapos.recipe.template
output = ${buildout:directory}/${:_buildout_section_name_}
inline =
# coverage configuration file, useful when making html report
[run]
plugins =
erp5_coverage_plugin
relative_files = true
[test-suite-runner]
# XXX: Workaround for fact ERP5Type is not an distribution and does not
......@@ -466,7 +563,6 @@ eggs = ${neoppod:eggs}
SOAPpy
chardet
collective.recipe.template
coverage
erp5diff
interval
ipdb
......@@ -606,6 +702,12 @@ Acquisition-patch-options = -p1
python-magic-patches = ${:_profile_base_location_}/../../component/egg-patch/python_magic/magic.patch#de0839bffac17801e39b60873a6c2068
python-magic-patch-options = -p1
# neoppod installs bin/coverage so we inject erp5 plugin here so that coverage script can use it in report
[neoppod]
eggs +=
erp5_coverage_plugin
[eggs-all-scripts]
recipe = zc.recipe.egg
eggs =
......@@ -683,6 +785,7 @@ PyStemmer = 1.3.0
Pympler = 0.4.3
StructuredText = 2.11.1
WSGIUtils = 0.7
erp5-coverage-plugin = 0.0.1
erp5diff = 0.8.1.8
five.formlib = 1.0.4
five.localsitemanager = 2.0.5
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment