Initial commit

README.md

+[`coverage.py`](https://coverage.readthedocs.io/) plugin to collect coverage from business templates.
+
+# How it works ?
+
+## Python Scripts
+
+### Collecting
+
+This depend on business template installation setting the `_erp5_coverage_filename`
+property on the script instance in ZODB. This property is a string, the
+full path of the python script.
+
+### Reporting
+
+During reporting, coverage needs to know the set of lines number containing code,
+to compare it with the line number that were actually executed.
+Because python scripts are compiled as a function, they can not be parsed
+by the default Python reporter, we use a simple reporter which wraps the
+code in a function definition to compile and collect the line numbers and then
+subtract 1 to the line numbers.
+
+
+## ZODB Components
+
+This also depends on business template installation setting the
+`_erp5_coverage_filename` property on the script instance in ZODB and the dynamic
+module to have it set to its `__file__`, then coverage can load it like a
+traditional python module.
+
+
+## Page Templates, TALES Expressions
+
+Not supported, no coverage is collected.

erp5_coverage_plugin/__init__.py

+#coding: utf-8
+from __future__ import print_function, unicode_literals
+
+import os
+
+import coverage
+
+
+class PythonScriptParser(coverage.parser.PythonParser):
+  """Python parser understanding Zope's python script implicit function.
+
+  parses the code with an extra function definition on first line and
+  then substract 1 to the lines numbers.
+  """
+  def __init__(self, text=None, filename=None, exclude=None):
+    super(PythonScriptParser, self).__init__(text=text, filename=filename, exclude=exclude)
+    if filename:
+      try:
+        self.text = coverage.python.get_python_source(self.filename)
+      except OSError as err:
+        raise coverage.misc.NoSource(
+          "No source for code: '{self.filename}': {err}".format(self=self, err=err))
+    
+    self.text = 'def _():\n  %s' % (
+      '\n  '.join(line for line in self.text.splitlines())   
+    )
+    self.lines = self.text.split('\n')[1:]
+
+  def _raw_parse(self):
+    super(PythonScriptParser, self)._raw_parse()
+    self._multiline = {
+      k-1: v-1 for (k, v) in self._multiline.items()
+    }
+    # remove the "def ():\n" we added to compile as a function
+    self.raw_excluded.discard(0)
+    self.raw_docstrings.discard(0)
+    self.raw_statements.discard(0)
+    self._analyze_ast()
+
+    self.raw_excluded = {v-1 for v in self.raw_excluded}
+    self.raw_docstrings = {v-1 for v in self.raw_docstrings}
+    self.raw_statements = {v-1 for v in self.raw_statements}
+
+  def parse_source(self):
+    super(PythonScriptParser, self).parse_source()
+    self.statements.discard(0)
+
+  def arcs(self):
+    last_line = max(self.raw_statements)
+    arcs = set()
+    for l1, l2 in super(PythonScriptParser, self).arcs():
+      if l1 == 1 and l2 == -1: # remove the arc from function def
+        continue
+      if l1 not in (-1, 1, last_line):
+        l1 = l1 - 1
+      if l2 not in (-1, 1, last_line):
+        l2 = l2 - 1
+      arcs.add((l1, l2))
+    return arcs
+
+
+class PythonScriptFileReporter(coverage.python.PythonFileReporter):
+
+  @property
+  def parser(self):
+    """Overloaded to create a PythonScriptParser instead of PythonParser."""
+    if self._parser is None:
+      self._parser = PythonScriptParser(
+        filename=self.filename,
+        # TODO: this plugin does not support excludes
+        #exclude=self.coverage._exclude_regex('exclude'),
+      )
+      self._parser.parse_source()
+    return self._parser
+
+  def no_branch_lines(self):
+    return set()
+
+
+class AbstractFileTracerPlugin(
+    coverage.plugin.CoveragePlugin,
+    coverage.plugin.FileTracer):
+
+  _base_names = NotImplemented
+
+  def __init__(self, options):
+    self._options = options
+    
+  def file_tracer(self, filename):
+    if os.path.basename(filename) in self._base_names:
+      return self
+    return None
+
+
+class TALESExpressionFileTracerPlugin(AbstractFileTracerPlugin):
+  """This plugin is not really implemented, but prevent errors trying
+  to cover TALES Expressions.
+  """
+  _base_names = ('PythonExpr', )
+
+  def file_reporter(self, filename):
+    class NoFileReporter(coverage.plugin.FileReporter):
+      def source(self):
+        raise coverage.misc.NoSource("no source for TALES Expressions")
+      def lines(self):
+        return set()
+    return NoFileReporter(filename)
+
+  def source_filename(self):
+    return ''
+
+
+class PythonScriptFileTracerPlugin(AbstractFileTracerPlugin):
+  _base_names = {
+    'ERP5 Python Script',
+    'ERP5 Workflow Script',
+    'Script (Python)',
+  }
+
+  def file_reporter(self, filename):
+    return PythonScriptFileReporter(filename)
+
+  def has_dynamic_source_filename(self):
+    return True
+
+  def dynamic_source_filename(self, filename, frame):
+    for f in frame, frame.f_back, frame.f_back.f_back:
+      if '__traceback_supplement__' in f.f_globals:
+        filename = getattr(f.f_globals['__traceback_supplement__'][1], '_erp5_coverage_filename', None)
+        if filename:
+          return filename
+    return None
+
+
+def coverage_init(reg, options):
+  reg.add_file_tracer(PythonScriptFileTracerPlugin(options))
+  reg.add_file_tracer(TALESExpressionFileTracerPlugin(options))

setup.py

+from setuptools import find_packages, setup
+
+setup(
+  name='erp5_coverage_plugin',
+  version='0.0.1',
+  packages=find_packages(),
+  install_requires=['coverage'],
+  extras_require={'test': ['pytest', 'Products.PythonScripts']},
+)

tests/test_data/python_script.py

+"""docstring
+"""
+if 1 == 1 and 0 == 1:
+  1 / 0
+_ = 1 + 1
+
+multiline_statement = """
+multi
+""" + """
+line
+"""
+
+return 1

tests/test_integration.py

+import contextlib
+import json
+import os
+
+from Products.PythonScripts.PythonScript import PythonScript
+from Products.PythonScripts.tests.testPythonScript import DummyFolder
+from Testing.makerequest import makerequest
+import coverage
+import pytest
+
+
+@pytest.fixture()
+def file_system_script(tmp_path):
+  p = tmp_path / 'script.py'
+  p.write_text(
+    '''\
+if "a" == "a" and "b" == "c":
+  _ = 1 / 0 # will not be covered
+
+_ = 1 + 1
+# comment
+
+return 'returned value'
+''')
+  yield p
+
+
+@contextlib.contextmanager
+def _coverage_process(file_system_script, branch):
+  cwd = os.getcwd()
+  os.chdir(file_system_script.parent)
+  cp = coverage.Coverage(include=['./*'], branch=branch)
+  cp.set_option('run:plugins', ['erp5_coverage_plugin'])
+  cp.start()
+  yield cp
+  cp.stop()
+  os.chdir(cwd)
+
+
+@pytest.fixture()
+def coverage_process(file_system_script):
+  with _coverage_process(file_system_script, branch=False) as cp:
+    yield cp
+
+
+@pytest.fixture()
+def coverage_process_with_branch_coverage(file_system_script, request):
+  with _coverage_process(file_system_script, branch=True) as cp:
+    yield cp
+
+
+@pytest.fixture()
+def python_script(file_system_script):
+  ps = PythonScript('test_script')
+  # Important note: for this plugin to work, something must set the property on python
+  # script
+  ps._erp5_coverage_filename = str(file_system_script.absolute())
+
+  ps.ZPythonScript_edit('', file_system_script.read_text())
+  yield ps.__of__(makerequest(DummyFolder('folder')))
+
+
+@pytest.fixture()
+def python_script_with_callback(file_system_script):
+  file_system_script.write_text(
+    '''\
+result_storage = []
+def callback_function(result):
+  result_storage.append(result)
+callback_script(callback_function)
+return result_storage
+''')
+  source_code_folder = file_system_script.parent
+  callback_file_system_script = source_code_folder / 'callback.py'
+  callback_file_system_script.write_text(
+    '''\
+callback_function("returned value")
+  ''')
+
+  test_script = PythonScript('test_script')
+  callback_script = PythonScript('callback_script')
+
+  # Important note: for this plugin to work, something must set the property on python
+  # script
+  test_script._erp5_coverage_filename = str(file_system_script.absolute())
+  callback_script._erp5_coverage_filename = str(callback_file_system_script.absolute())
+
+  test_script.ZPythonScript_edit('', file_system_script.read_text())
+  callback_script.ZPythonScript_edit(
+    'callback_function', callback_file_system_script.read_text())
+  folder = makerequest(DummyFolder('folder'))
+  yield test_script.__of__(folder), callback_script.__of__(folder)
+
+
+def test_python_script(coverage_process, python_script, capsys):
+  assert python_script._exec({}, [], {}) == 'returned value'
+  coverage_process.stop()
+  assert coverage_process.report() > 0
+  assert capsys.readouterr().out == '''\
+Name        Stmts   Miss  Cover
+-------------------------------
+script.py       4      1    75%
+-------------------------------
+TOTAL           4      1    75%
+'''
+
+
+def test_python_script_with_branch_coverage(
+    coverage_process_with_branch_coverage, python_script, capsys, tmp_path):
+  assert python_script._exec({}, [], {}) == 'returned value'
+  coverage_process_with_branch_coverage.stop()
+  assert coverage_process_with_branch_coverage.report() > 0
+  assert capsys.readouterr().out == '''\
+Name        Stmts   Miss Branch BrPart  Cover
+---------------------------------------------
+script.py       4      1      2      1    67%
+---------------------------------------------
+TOTAL           4      1      2      1    67%
+'''
+  outfile = tmp_path / 'out.json'
+  assert coverage_process_with_branch_coverage.json_report(outfile=outfile) > 0
+  script_py_report = json.loads(outfile.read_text())['files']['script.py']
+  script_py_report.pop('summary')
+  assert script_py_report == {
+    'excluded_lines': [],
+    'executed_branches': [[1, 4]],
+    'executed_lines': [1, 4, 7],
+    'missing_branches': [[1, 2]],
+    'missing_lines': [2],
+  }
+
+
+def test_python_script_callback(
+    coverage_process, python_script_with_callback, capsys):
+  test_python_script, callback_script = python_script_with_callback
+  assert test_python_script._exec(
+    {'callback_script': callback_script}, [], {}) == ['returned value']
+  coverage_process.stop()
+  assert coverage_process.report() > 0
+  assert capsys.readouterr().out == '''\
+Name          Stmts   Miss  Cover
+---------------------------------
+callback.py       1      0   100%
+script.py         5      0   100%
+---------------------------------
+TOTAL             6      0   100%
+'''
+
+
+def test_python_script_callback_with_branch_coverage(
+    coverage_process_with_branch_coverage, python_script_with_callback,
+    capsys):
+  test_python_script, callback_script = python_script_with_callback
+  assert test_python_script._exec(
+    {'callback_script': callback_script}, [], {}) == ['returned value']
+  coverage_process_with_branch_coverage.stop()
+  assert coverage_process_with_branch_coverage.report() > 0
+  assert capsys.readouterr().out == '''\
+Name          Stmts   Miss Branch BrPart  Cover
+-----------------------------------------------
+callback.py       1      0      0      0   100%
+script.py         5      0      2      1    86%
+-----------------------------------------------
+TOTAL             6      0      2      1    88%
+'''

tests/test_python_script_parser.py

+import pytest
+import os.path
+from erp5_coverage_plugin import PythonScriptParser
+
+
+@pytest.fixture()
+def parser():
+  parser = PythonScriptParser(
+    filename=os.path.join(
+      os.path.dirname(__file__), 'test_data', 'python_script.py'))
+
+  parser.parse_source()
+  return parser
+
+
+def test_statements(parser):
+  assert parser.statements == {3, 4, 5, 7, 13}