Support coverage for subprocess. (#399)

Fixes #397.

tox will produce the coverage reports by default.

.coveragerc

+[run]
+source = zc.buildout
+parallel = true
+
+[report]
+exclude_lines =
+    pragma: no cover
+    if __name__ == '__main__':
+    raise NotImplementedError

.gitignore

@@ -11,3 +11,5 @@ python*/
 *.egg-info
 dist
 .tox
+.coverage
+.coverage.*

.travis.yml

@@ -18,6 +18,7 @@ notifications:
 
 install:
     - ls -al pythons
+    - export COVERAGE_PROCESS_START=
     - deactivate
     - make build
 

CHANGES.rst

@@ -13,6 +13,8 @@ Change History
   considered outdated and lead to parts being reinstalled spuriously
   under Python 2.
 
+- Add support code for doctests to be able to easily measure code
+  coverage. See `issue 397 <https://github.com/buildout/buildout/issues/397>`_.
 
 2.9.3 (2017-03-30)
 ==================

buildout.cfg

@@ -8,12 +8,14 @@ zope.interface = 4.1.3
 zope.exceptions = 4.0.8
 zc.recipe.testrunner = 2.0.0
 zope.testing = 4.5.0
+coverage = 4.4.1
 
 [py]
 recipe = zc.recipe.egg
 eggs = zc.buildout[test]
        zc.recipe.egg
        zope.testing
+       coverage
 interpreter = py
 
 [test]
@@ -21,6 +23,10 @@ recipe = zc.recipe.testrunner
 eggs =
   zc.buildout[test]
   zc.recipe.egg
+initialization =
+  import os
+  if 'COVERAGE_PROCESS_START' not in os.environ: os.environ['COVERAGE_PROCESS_START'] = os.path.abspath('../../.coveragerc')
+  if os.getenv('COVERAGE_PROCESS_START'): import coverage; coverage.process_startup()
 
 # Tests that can be run wo a network
 [oltest]

src/zc/buildout/testing.py

@@ -77,6 +77,9 @@ def ls(dir, *subs):
         dir = os.path.join(dir, *subs)
     names = sorted(os.listdir(dir))
     for name in names:
+        # If we're running under coverage, elide coverage files
+        if os.getenv("COVERAGE_PROCESS_START") and name.startswith('.coverage.'):
+            continue
         if os.path.isdir(os.path.join(dir, name)):
             print_('d ', end=' ')
         elif os.path.islink(os.path.join(dir, name)):
@@ -292,6 +295,38 @@ def buildoutSetUp(test):
     # way due to the trick above:
     os.mkdir('develop-eggs')
 
+    if os.getenv("COVERAGE_PROCESS_START"):
+        # The user has requested subprocess code coverage. Since we will be changing
+        # directories, we need to make sure this path is absolute, which means
+        # we need to temporarily return to our starting directory.
+        os.chdir(here)
+        path_to_coveragerc = os.path.abspath(os.environ['COVERAGE_PROCESS_START'])
+        os.chdir(sample)
+        assert os.path.isfile(path_to_coveragerc), path_to_coveragerc
+        os.environ['COVERAGE_PROCESS_START'] = path_to_coveragerc
+
+        # Before we return to the current directory and destroy the
+        # temporary working directory, we need to copy all the coverage files
+        # back so that they can be `coverage combine`d.
+        def copy_coverage_files():
+            coveragedir = os.path.dirname(path_to_coveragerc)
+            import glob
+            for f in glob.glob('.coverage*'):
+                shutil.copy(f, coveragedir)
+        __tear_downs.insert(0, copy_coverage_files)
+
+
+        # Now we must modify the newly created bin/buildout to
+        # actually begin coverage.
+        with open('bin/buildout') as f:
+            import textwrap
+            lines = f.read().splitlines()
+            assert lines[1] == '', lines
+            lines[1] = 'import coverage; coverage.process_startup()'
+
+        with open('bin/buildout', 'w') as f:
+            f.write('\n'.join(lines))
+
     def start_server(path):
         port, thread = _start_server(path, name=path)
         url = 'http://localhost:%s/' % port

src/zc/buildout/testing.txt

@@ -96,7 +96,7 @@ number of names to the test namespace:
     down at the end of the test.  The server URL is returned.
 
     You can cause the server to start and stop logging it's output
-    using: 
+    using:
 
        >>> get(server_url+'enable_server_logging')
 
@@ -120,6 +120,37 @@ number of names to the test namespace:
     directory.  If the setup argument is a directory, then the
     setup.py file in that directory is used.
 
+Code Coverage Measurement
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``buildoutSetUp`` prepares a new ``bin/buildout`` executable for
+doctests to run. Because this is will run in a subprocess, there are
+some extra steps needed to collect code coverage data for buildout and
+buildout recipes. This function does most of the heavy lifting for
+you, you only need to do a few things.
+
+First, you need to define a coverage configuration ('.coveragerc')
+which specifies that multiple different runs should be collected::
+
+    [run]
+    source = my.egg.name
+    parallel = true
+
+Second, when you invoke the test runner, you need to define the
+``COVERAGE_PROCESS_START`` environment variable to point to this
+configuration and invoke it with coverage::
+
+    COVERAGE_PROCESS_START=.coveragerc coverage run -m zope.testrunner --test-path=src
+
+Finally, after the tests have finished, you need to combine the
+resulting coverage data files and then you can produce reports::
+
+    coverage combine
+    coverage report
+
+.. caution:: You must be working in an environment (e.g., a virtual
+			 environment) that has coverage already installed.
+
 ``zc.buildout.testing.buildoutTearDown(test)``
 ----------------------------------------------
 

tox.ini

@@ -4,10 +4,13 @@ envlist =
 
 [testenv]
 commands =
+    pip install coverage
     # buildout's dev.py wants python to not have setuptools
     pip uninstall -y zc.buildout setuptools pip
     python dev.py
     {toxinidir}/bin/test -1 -v -c
+    coverage combine
+    coverage report
 # since we're removing setuptools we can't possibly reuse the virtualenv
 recreate = true
 # if the user has ccache installed, PATH may contain /usr/lib/ccache that has a