Commit 88219037 authored by Leonardo Rochael Almeida's avatar Leonardo Rochael Almeida Committed by GitHub

Merge pull request #402 from rafaelbco/master

Fix #153: buildout should cache working set environments
parents 2517f01c a4781e45
...@@ -4,7 +4,8 @@ Change History ...@@ -4,7 +4,8 @@ Change History
2.0.4 (unreleased) 2.0.4 (unreleased)
================== ==================
- Nothing changed yet. - Fix #153: buildout should cache working set environments
[rafaelbco]
2.0.3 (2015-10-02) 2.0.3 (2015-10-02)
......
...@@ -43,6 +43,8 @@ setup( ...@@ -43,6 +43,8 @@ setup(
+ '\n' + + '\n' +
read('src', 'zc', 'recipe', 'egg', 'api.rst') read('src', 'zc', 'recipe', 'egg', 'api.rst')
+ '\n' + + '\n' +
read('src', 'zc', 'recipe', 'egg', 'working_set_caching.rst')
+ '\n' +
'Download\n' 'Download\n'
'*********\n' '*********\n'
), ),
......
...@@ -14,15 +14,18 @@ ...@@ -14,15 +14,18 @@
"""Install packages as eggs """Install packages as eggs
""" """
import copy
import logging import logging
import os import os
import re import re
import sys import sys
import zc.buildout.easy_install import zc.buildout.easy_install
import zipfile
class Eggs(object): class Eggs(object):
_WORKING_SET_CACHE_ATTR_NAME = '_zc_recipe_egg_working_set_cache'
def __init__(self, buildout, name, options): def __init__(self, buildout, name, options):
self.buildout = buildout self.buildout = buildout
self.name = name self.name = name
...@@ -43,7 +46,7 @@ class Eggs(object): ...@@ -43,7 +46,7 @@ class Eggs(object):
allow_hosts = b_options['allow-hosts'] allow_hosts = b_options['allow-hosts']
allow_hosts = tuple([host.strip() for host in allow_hosts.split('\n') allow_hosts = tuple([host.strip() for host in allow_hosts.split('\n')
if host.strip()!='']) if host.strip() != ''])
self.allow_hosts = allow_hosts self.allow_hosts = allow_hosts
options['eggs-directory'] = b_options['eggs-directory'] options['eggs-directory'] = b_options['eggs-directory']
...@@ -57,31 +60,27 @@ class Eggs(object): ...@@ -57,31 +60,27 @@ class Eggs(object):
This is intended for reuse by similar recipes. This is intended for reuse by similar recipes.
""" """
options = self.options options = self.options
b_options = self.buildout['buildout'] buildout_section = self.buildout['buildout']
# Backward compat. :( # Backward compat. :(
options['executable'] = sys.executable options['executable'] = sys.executable
distributions = [ orig_distributions = [
r.strip() r.strip()
for r in options.get('eggs', self.name).split('\n') for r in options.get('eggs', self.name).split('\n')
if r.strip()] if r.strip()
orig_distributions = distributions[:] ]
distributions.extend(extra)
if self.buildout['buildout'].get('offline') == 'true': ws = self._working_set(
ws = zc.buildout.easy_install.working_set( distributions=orig_distributions + list(extra),
distributions, develop_eggs_dir=options['develop-eggs-directory'],
[options['develop-eggs-directory'], options['eggs-directory']] eggs_dir=options['eggs-directory'],
) offline=(buildout_section.get('offline') == 'true'),
else: newest=(buildout_section.get('newest') == 'true'),
ws = zc.buildout.easy_install.install(
distributions, options['eggs-directory'],
links=self.links, links=self.links,
index=self.index, index=self.index,
path=[options['develop-eggs-directory']], allow_hosts=self.allow_hosts,
newest=self.buildout['buildout'].get('newest') == 'true', )
allow_hosts=self.allow_hosts)
return orig_distributions, ws return orig_distributions, ws
...@@ -91,6 +90,74 @@ class Eggs(object): ...@@ -91,6 +90,74 @@ class Eggs(object):
update = install update = install
def _working_set(
self,
distributions,
eggs_dir,
develop_eggs_dir,
offline=False,
newest=True,
links=(),
index=None,
allow_hosts=('*',),
):
"""Helper function to build a working set.
Return an instance of `pkg_resources.WorkingSet`.
Results are cached. The cache key is composed by all the arguments
passed to the function. See also `self._get_cache_storage()`.
"""
cache_storage = self._get_cache_storage()
cache_key = (
tuple(distributions),
eggs_dir,
develop_eggs_dir,
offline,
newest,
tuple(links),
index,
tuple(allow_hosts),
)
if cache_key not in cache_storage:
if offline:
ws = zc.buildout.easy_install.working_set(
distributions,
[develop_eggs_dir, eggs_dir]
)
else:
ws = zc.buildout.easy_install.install(
distributions, eggs_dir,
links=links,
index=index,
path=[develop_eggs_dir],
newest=newest,
allow_hosts=allow_hosts)
cache_storage[cache_key] = ws
# `pkg_resources.WorkingSet` instances are mutable, so we need to return
# a copy.
return copy.deepcopy(cache_storage[cache_key])
def _get_cache_storage(self):
"""Return a mapping where to store generated working sets.
The cache storage is stored in an attribute of `self.buildout` with
name given by `self._WORKING_SET_CACHE_ATTR_NAME`.
"""
cache_storage = getattr(
self.buildout,
self._WORKING_SET_CACHE_ATTR_NAME,
None)
if cache_storage is None:
cache_storage = {}
setattr(
self.buildout,
self._WORKING_SET_CACHE_ATTR_NAME,
cache_storage)
return cache_storage
class Scripts(Eggs): class Scripts(Eggs):
def __init__(self, buildout, name, options): def __init__(self, buildout, name, options):
...@@ -107,7 +174,6 @@ class Scripts(Eggs): ...@@ -107,7 +174,6 @@ class Scripts(Eggs):
if self.extra_paths: if self.extra_paths:
options['extra-paths'] = '\n'.join(self.extra_paths) options['extra-paths'] = '\n'.join(self.extra_paths)
relative_paths = options.get( relative_paths = options.get(
'relative-paths', 'relative-paths',
buildout['buildout'].get('relative-paths', 'false') buildout['buildout'].get('relative-paths', 'false')
...@@ -122,6 +188,7 @@ class Scripts(Eggs): ...@@ -122,6 +188,7 @@ class Scripts(Eggs):
parse_entry_point = re.compile( parse_entry_point = re.compile(
'([^=]+)=(\w+(?:[.]\w+)*):(\w+(?:[.]\w+)*)$' '([^=]+)=(\w+(?:[.]\w+)*):(\w+(?:[.]\w+)*)$'
).match ).match
def install(self): def install(self):
reqs, ws = self.working_set() reqs, ws = self.working_set()
options = self.options options = self.options
...@@ -166,6 +233,7 @@ class Scripts(Eggs): ...@@ -166,6 +233,7 @@ class Scripts(Eggs):
update = install update = install
def get_bool(options, name, default=False): def get_bool(options, name, default=False):
value = options.get(name) value = options.get(name)
if not value: if not value:
......
...@@ -109,6 +109,16 @@ def test_suite(): ...@@ -109,6 +109,16 @@ def test_suite():
''), ''),
]), ]),
), ),
doctest.DocFileSuite(
'working_set_caching.rst',
setUp=setUp, tearDown=zc.buildout.testing.buildoutTearDown,
optionflags=doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS,
checker=renormalizing.RENormalizing([
zc.buildout.testing.normalize_path,
zc.buildout.testing.normalize_endings,
zc.buildout.testing.not_found,
])
),
)) ))
return suite return suite
......
Working set caching
===================
Working sets are cached, to improve speed on buildouts with multiple similar
parts based on ``zc.recipe.egg``.
The egg-recipe instance's ``_working_set`` helper method is used to make
the caching easier. It does the same job as ``working_set()`` but with some
differences:
- The signature is different: all information needed to build the working set
is passed as parameters.
- The return value is simpler: only an instance of ``pkg_resources.WorkingSet``
is returned.
Here's an example:
>>> from zc.buildout import testing
>>> from zc.recipe.egg.egg import Eggs
>>> import os
>>> import pkg_resources
>>> recipe = Eggs(buildout=testing.Buildout(), name='fake-part', options={})
>>> eggs_dir = os.path.join(sample_buildout, 'eggs')
>>> develop_eggs_dir = os.path.join(sample_buildout, 'develop-eggs')
>>> testing.install_develop('zc.recipe.egg', develop_eggs_dir)
>>> ws = recipe._working_set(
... distributions=['zc.recipe.egg', 'demo<0.3'],
... eggs_dir=eggs_dir,
... develop_eggs_dir=develop_eggs_dir,
... index=link_server,
... )
Getting...
>>> isinstance(ws, pkg_resources.WorkingSet)
True
>>> sorted(dist.project_name for dist in ws)
['demo', 'demoneeded', 'setuptools', 'zc.buildout', 'zc.recipe.egg']
We'll monkey patch a method in the ``easy_install`` module in order to verify if
the cache is working:
>>> import zc.buildout.easy_install
>>> old_install = zc.buildout.easy_install.Installer.install
>>> def new_install(*args, **kwargs):
... print('Building working set.')
... return old_install(*args, **kwargs)
>>> zc.buildout.easy_install.Installer.install = new_install
Now we check if the caching is working by verifying if the same working set is
built only once.
>>> ws_args_1 = dict(
... distributions=['demo>=0.1'],
... eggs_dir=eggs_dir,
... develop_eggs_dir=develop_eggs_dir,
... offline=True,
... )
>>> ws_args_2 = dict(ws_args_1)
>>> ws_args_2['distributions'] = ['demoneeded']
>>> recipe._working_set(**ws_args_1)
Building working set.
<pkg_resources.WorkingSet object at ...>
>>> recipe._working_set(**ws_args_1)
<pkg_resources.WorkingSet object at ...>
>>> recipe._working_set(**ws_args_2)
Building working set.
<pkg_resources.WorkingSet object at ...>
>>> recipe._working_set(**ws_args_1)
<pkg_resources.WorkingSet object at ...>
>>> recipe._working_set(**ws_args_2)
<pkg_resources.WorkingSet object at ...>
Undo monkey patch:
>>> zc.buildout.easy_install.Installer.install = old_install
Since ``pkg_resources.WorkingSet`` instances are mutable, we must ensure that
``working_set()`` always returns a pristine copy. Otherwise callers would be
able to modify instances inside the cache.
Let's create a working set:
>>> ws = recipe._working_set(**ws_args_1)
>>> sorted(dist.project_name for dist in ws)
['demo', 'demoneeded']
Now we add a distribution to it:
>>> dist = pkg_resources.get_distribution('zc.recipe.egg')
>>> ws.add(dist)
>>> sorted(dist.project_name for dist in ws)
['demo', 'demoneeded', 'zc.recipe.egg']
Let's call the working_set function again and see if the result remains valid:
>>> ws = recipe._working_set(**ws_args_1)
>>> sorted(dist.project_name for dist in ws)
['demo', 'demoneeded']
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment