Merge pull request #402 from rafaelbco/master

Fix #153: buildout should cache working set environments

zc.recipe.egg_/CHANGES.rst

@@ -4,7 +4,8 @@ Change History
 2.0.4 (unreleased)
 ==================
 
-- Nothing changed yet.
+- Fix #153: buildout should cache working set environments
+  [rafaelbco]
 
 
 2.0.3 (2015-10-02)

zc.recipe.egg_/setup.py

@@ -43,6 +43,8 @@ setup(
         + '\n' +
         read('src', 'zc', 'recipe', 'egg', 'api.rst')
         + '\n' +
+        read('src', 'zc', 'recipe', 'egg', 'working_set_caching.rst')
+        + '\n' +
         'Download\n'
         '*********\n'
         ),

zc.recipe.egg_/src/zc/recipe/egg/egg.py

@@ -14,15 +14,18 @@
 """Install packages as eggs
 """
 
+import copy
 import logging
 import os
 import re
 import sys
 import zc.buildout.easy_install
-import zipfile
+
 
 class Eggs(object):
 
+    _WORKING_SET_CACHE_ATTR_NAME = '_zc_recipe_egg_working_set_cache'
+
     def __init__(self, buildout, name, options):
         self.buildout = buildout
         self.name = name
@@ -43,13 +46,13 @@ class Eggs(object):
 
         allow_hosts = b_options['allow-hosts']
         allow_hosts = tuple([host.strip() for host in allow_hosts.split('\n')
-                               if host.strip()!=''])
+                            if host.strip() != ''])
         self.allow_hosts = allow_hosts
 
         options['eggs-directory'] = b_options['eggs-directory']
-        options['_e'] = options['eggs-directory'] # backward compat.
+        options['_e'] = options['eggs-directory']  # backward compat.
         options['develop-eggs-directory'] = b_options['develop-eggs-directory']
-        options['_d'] = options['develop-eggs-directory'] # backward compat.
+        options['_d'] = options['develop-eggs-directory']  # backward compat.
 
     def working_set(self, extra=()):
         """Separate method to just get the working set
@@ -57,31 +60,27 @@ class Eggs(object):
         This is intended for reuse by similar recipes.
         """
         options = self.options
-        b_options = self.buildout['buildout']
+        buildout_section = self.buildout['buildout']
 
         # Backward compat. :(
         options['executable'] = sys.executable
 
-        distributions = [
+        orig_distributions = [
             r.strip()
             for r in options.get('eggs', self.name).split('\n')
-            if r.strip()]
-        orig_distributions = distributions[:]
-        distributions.extend(extra)
-
-        if self.buildout['buildout'].get('offline') == 'true':
-            ws = zc.buildout.easy_install.working_set(
-                distributions,
-                [options['develop-eggs-directory'], options['eggs-directory']]
-                )
-        else:
-            ws = zc.buildout.easy_install.install(
-                distributions, options['eggs-directory'],
-                links=self.links,
-                index=self.index,
-                path=[options['develop-eggs-directory']],
-                newest=self.buildout['buildout'].get('newest') == 'true',
-                allow_hosts=self.allow_hosts)
+            if r.strip()
+            ]
+
+        ws = self._working_set(
+            distributions=orig_distributions + list(extra),
+            develop_eggs_dir=options['develop-eggs-directory'],
+            eggs_dir=options['eggs-directory'],
+            offline=(buildout_section.get('offline') == 'true'),
+            newest=(buildout_section.get('newest') == 'true'),
+            links=self.links,
+            index=self.index,
+            allow_hosts=self.allow_hosts,
+            )
 
         return orig_distributions, ws
 
@@ -91,13 +90,81 @@ class Eggs(object):
 
     update = install
 
+    def _working_set(
+        self,
+        distributions,
+        eggs_dir,
+        develop_eggs_dir,
+        offline=False,
+        newest=True,
+        links=(),
+        index=None,
+        allow_hosts=('*',),
+    ):
+        """Helper function to build a working set.
+
+        Return an instance of `pkg_resources.WorkingSet`.
+
+        Results are cached. The cache key is composed by all the arguments
+        passed to the function. See also `self._get_cache_storage()`.
+        """
+        cache_storage = self._get_cache_storage()
+        cache_key = (
+            tuple(distributions),
+            eggs_dir,
+            develop_eggs_dir,
+            offline,
+            newest,
+            tuple(links),
+            index,
+            tuple(allow_hosts),
+        )
+        if cache_key not in cache_storage:
+            if offline:
+                ws = zc.buildout.easy_install.working_set(
+                    distributions,
+                    [develop_eggs_dir, eggs_dir]
+                    )
+            else:
+                ws = zc.buildout.easy_install.install(
+                    distributions, eggs_dir,
+                    links=links,
+                    index=index,
+                    path=[develop_eggs_dir],
+                    newest=newest,
+                    allow_hosts=allow_hosts)
+            cache_storage[cache_key] = ws
+
+        # `pkg_resources.WorkingSet` instances are mutable, so we need to return
+        # a copy.
+        return copy.deepcopy(cache_storage[cache_key])
+
+    def _get_cache_storage(self):
+        """Return a mapping where to store generated working sets.
+
+        The cache storage is stored in an attribute of `self.buildout` with
+        name given by `self._WORKING_SET_CACHE_ATTR_NAME`.
+        """
+        cache_storage = getattr(
+            self.buildout,
+            self._WORKING_SET_CACHE_ATTR_NAME,
+            None)
+        if cache_storage is None:
+            cache_storage = {}
+            setattr(
+                self.buildout,
+                self._WORKING_SET_CACHE_ATTR_NAME,
+                cache_storage)
+        return cache_storage
+
+
 class Scripts(Eggs):
 
     def __init__(self, buildout, name, options):
         super(Scripts, self).__init__(buildout, name, options)
 
         options['bin-directory'] = buildout['buildout']['bin-directory']
-        options['_b'] = options['bin-directory'] # backward compat.
+        options['_b'] = options['bin-directory']  # backward compat.
 
         self.extra_paths = [
             os.path.join(buildout['buildout']['directory'], p.strip())
@@ -107,7 +174,6 @@ class Scripts(Eggs):
         if self.extra_paths:
             options['extra-paths'] = '\n'.join(self.extra_paths)
 
-
         relative_paths = options.get(
             'relative-paths',
             buildout['buildout'].get('relative-paths', 'false')
@@ -122,6 +188,7 @@ class Scripts(Eggs):
     parse_entry_point = re.compile(
         '([^=]+)=(\w+(?:[.]\w+)*):(\w+(?:[.]\w+)*)$'
         ).match
+
     def install(self):
         reqs, ws = self.working_set()
         options = self.options
@@ -166,6 +233,7 @@ class Scripts(Eggs):
 
     update = install
 
+
 def get_bool(options, name, default=False):
     value = options.get(name)
     if not value:

zc.recipe.egg_/src/zc/recipe/egg/tests.py

@@ -109,6 +109,16 @@ def test_suite():
                  ''),
                 ]),
             ),
+        doctest.DocFileSuite(
+            'working_set_caching.rst',
+            setUp=setUp, tearDown=zc.buildout.testing.buildoutTearDown,
+            optionflags=doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS,
+            checker=renormalizing.RENormalizing([
+               zc.buildout.testing.normalize_path,
+               zc.buildout.testing.normalize_endings,
+               zc.buildout.testing.not_found,
+               ])
+            ),
         ))
     return suite
 

zc.recipe.egg_/src/zc/recipe/egg/working_set_caching.rst

+Working set caching
+===================
+
+Working sets are cached, to improve speed on buildouts with multiple similar
+parts based on ``zc.recipe.egg``.
+
+The egg-recipe instance's ``_working_set`` helper method is used to make
+the caching easier. It does the same job as ``working_set()`` but with some
+differences:
+
+- The signature is different: all information needed to build the working set
+  is passed as parameters.
+- The return value is simpler: only an instance of ``pkg_resources.WorkingSet``
+  is returned.
+
+Here's an example:
+
+    >>> from zc.buildout import testing
+    >>> from zc.recipe.egg.egg import Eggs
+    >>> import os
+    >>> import pkg_resources
+    >>> recipe = Eggs(buildout=testing.Buildout(), name='fake-part', options={})
+    >>> eggs_dir = os.path.join(sample_buildout, 'eggs')
+    >>> develop_eggs_dir = os.path.join(sample_buildout, 'develop-eggs')
+    >>> testing.install_develop('zc.recipe.egg', develop_eggs_dir)
+    >>> ws = recipe._working_set(
+    ...     distributions=['zc.recipe.egg', 'demo<0.3'],
+    ...     eggs_dir=eggs_dir,
+    ...     develop_eggs_dir=develop_eggs_dir,
+    ...     index=link_server,
+    ... )
+    Getting...
+    >>> isinstance(ws, pkg_resources.WorkingSet)
+    True
+    >>> sorted(dist.project_name for dist in ws)
+    ['demo', 'demoneeded', 'setuptools', 'zc.buildout', 'zc.recipe.egg']
+
+We'll monkey patch a method in the ``easy_install`` module in order to verify if
+the cache is working:
+
+    >>> import zc.buildout.easy_install
+    >>> old_install = zc.buildout.easy_install.Installer.install
+    >>> def new_install(*args, **kwargs):
+    ...     print('Building working set.')
+    ...     return old_install(*args, **kwargs)
+    >>> zc.buildout.easy_install.Installer.install = new_install
+
+Now we check if the caching is working by verifying if the same working set is
+built only once.
+
+    >>> ws_args_1 = dict(
+    ...     distributions=['demo>=0.1'],
+    ...     eggs_dir=eggs_dir,
+    ...     develop_eggs_dir=develop_eggs_dir,
+    ...     offline=True,
+    ... )
+    >>> ws_args_2 = dict(ws_args_1)
+    >>> ws_args_2['distributions'] = ['demoneeded']
+    >>> recipe._working_set(**ws_args_1)
+    Building working set.
+    <pkg_resources.WorkingSet object at ...>
+    >>> recipe._working_set(**ws_args_1)
+    <pkg_resources.WorkingSet object at ...>
+    >>> recipe._working_set(**ws_args_2)
+    Building working set.
+    <pkg_resources.WorkingSet object at ...>
+    >>> recipe._working_set(**ws_args_1)
+    <pkg_resources.WorkingSet object at ...>
+    >>> recipe._working_set(**ws_args_2)
+    <pkg_resources.WorkingSet object at ...>
+
+Undo monkey patch:
+
+    >>> zc.buildout.easy_install.Installer.install = old_install
+
+Since ``pkg_resources.WorkingSet`` instances are mutable, we must ensure that
+``working_set()`` always returns a pristine copy. Otherwise callers would be
+able to modify instances inside the cache.
+
+Let's create a working set:
+
+    >>> ws = recipe._working_set(**ws_args_1)
+    >>> sorted(dist.project_name for dist in ws)
+    ['demo', 'demoneeded']
+
+Now we add a distribution to it:
+
+    >>> dist = pkg_resources.get_distribution('zc.recipe.egg')
+    >>> ws.add(dist)
+    >>> sorted(dist.project_name for dist in ws)
+    ['demo', 'demoneeded', 'zc.recipe.egg']
+
+Let's call the working_set function again and see if the result remains valid:
+
+    >>> ws = recipe._working_set(**ws_args_1)
+    >>> sorted(dist.project_name for dist in ws)
+    ['demo', 'demoneeded']