Skip to content

S
slapos.buildout
Commit eeb0c30e authored by Jim Fulton's avatar Jim Fulton
Browse files
Options

Merge pull request #46 from reinout/reinout-versions 

Merging buildout-versions' functionality into buildout
parents c623ce01 fc1bc5d0
Show whitespace changes
Inline Side-by-side
Showing with 314 additions and 14 deletions
.travis.yml
View file @ eeb0c30e
......@@ -5,10 +5,6 @@ env:
- PYTHON_VER=3.2
- PYTHON_VER=3.3
branches:
only:
- master
notifications:
email:
- buildout-development@googlegroups.com
......
CHANGES.rst
View file @ eeb0c30e
Change History
**************
2.0.0b2 (unreleased)
====================
- Integrated the `buildout-versions
<http://packages.python.org/buildout-versions/>`_ extension into buildout
itself. For this, a few options were added to buildout:
- If ``show-picked-versions`` is set to true, all picked versions are
printed at the end of the buildout run. This saves you from running
buildout in verbose mode and extracting the picked versions from the
output.
- If ``update-versions-file`` is set to a filename (relative to the buildout
directory), the ``show-picked-versions`` output is appended to that file.
2.0.0b1 (2013-01-21)
====================
- buildout options can be given on the command line using the form::
- Buildout options can be given on the command line using the form::
option_name=value
......
src/zc/buildout/buildout.py
View file @ eeb0c30e
......@@ -29,6 +29,7 @@ except ImportError:
import zc.buildout.configparser
import copy
import datetime
import distutils.errors
import glob
import itertools
......@@ -136,7 +137,9 @@ _buildout_default_options = _annotate_section({
'parts-directory': 'parts',
'prefer-final': 'true',
'python': 'buildout',
'show-picked-versions': 'false',
'socket-timeout': '',
'update-versions-file': '',
'use-dependency-links': 'true',
}, 'DEFAULT_VALUE')
......@@ -206,7 +209,7 @@ class Buildout(DictMixin):
# Set up versions section, if necessary
if 'versions' not in data['buildout']:
data['buildout']['versions'] = 'versions', 'DEFAULT_VALUE'
data['buildout']['versions'] = ('versions', 'DEFAULT_VALUE')
if 'versions' not in data:
data['versions'] = {}
......@@ -304,7 +307,7 @@ class Buildout(DictMixin):
# finish w versions
if versions_section_name:
# refetching section name just to avoid a warnin
# refetching section name just to avoid a warning
versions = self[versions_section_name]
else:
# remove annotations
......@@ -319,6 +322,11 @@ class Buildout(DictMixin):
bool_option(options, 'use-dependency-links'))
zc.buildout.easy_install.allow_picked_versions(
bool_option(options, 'allow-picked-versions'))
self.show_picked_versions = bool_option(options,
'show-picked-versions')
self.update_versions_file = options['update-versions-file']
zc.buildout.easy_install.show_picked_versions(
self.show_picked_versions)
download_cache = options.get('download-cache')
if download_cache:
......@@ -631,6 +639,7 @@ class Buildout(DictMixin):
elif (not installed_parts) and installed_exists:
os.remove(self['buildout']['installed'])
self._print_picked_versions()
self._unload_extensions()
def _update_installed(self, **buildout_options):
......@@ -944,6 +953,14 @@ class Buildout(DictMixin):
def _load_extensions(self):
__doing__ = 'Loading extensions.'
specs = self['buildout'].get('extensions', '').split()
for superceded_extension in ['buildout-versions',
'buildout.dumppickedversions']:
if superceded_extension in specs:
msg = ("The extension %s is now included in buildout itself.\n"
"Remove the extension from your configuration and "
"look at the `show-picked-versions`\n"
"option in buildout's documentation.")
raise zc.buildout.UserError(msg % superceded_extension)
if specs:
path = [self['buildout']['develop-eggs-directory']]
if self.offline:
......@@ -977,6 +994,45 @@ class Buildout(DictMixin):
'zc.buildout.unloadextension'):
ep.load()(self)
def _print_picked_versions(self):
Installer = zc.buildout.easy_install.Installer
if not self.show_picked_versions:
return
if not Installer._picked_versions:
# Don't print empty output.
return
output = ['[versions]']
required_output = []
for dist_, version in sorted(Installer._picked_versions.items()):
if dist_ in Installer._required_by:
required_output.append('')
required_output.append('# Required by:')
for req_ in sorted(Installer._required_by[dist_]):
required_output.append('# '+req_)
target = required_output
else:
target = output
target.append("%s = %s" % (dist_, version))
output.extend(required_output)
print_("Versions had to be automatically picked.")
print_("The following part definition lists the versions picked:")
print_('\n'.join(output))
if self.update_versions_file:
# Also write to the versions file.
if os.path.exists(self.update_versions_file):
output[:1] = [
'',
'# Added by buildout at %s' % datetime.datetime.now(),
]
output.append('')
f = open(self.update_versions_file, 'a')
f.write('\n'.join(output))
f.close()
print_("This information has been written to " +
self.update_versions_file)
def setup(self, args):
if not args:
raise zc.buildout.UserError(
......
src/zc/buildout/buildout.txt
View file @ eeb0c30e
......@@ -833,8 +833,12 @@ COMMAND_LINE_VALUE).
DEFAULT_VALUE
python= buildout
DEFAULT_VALUE
show-picked-versions= false
DEFAULT_VALUE
socket-timeout=
DEFAULT_VALUE
update-versions-file=
DEFAULT_VALUE
use-dependency-links= true
DEFAULT_VALUE
versions= versions
......@@ -2373,7 +2377,9 @@ database is shown.
parts-directory = /sample-buildout/parts
prefer-final = true
python = buildout
show-picked-versions = false
socket-timeout =
update-versions-file =
use-dependency-links = true
verbosity = 20
versions = versions
......
src/zc/buildout/easy_install.py
View file @ eeb0c30e
......@@ -69,6 +69,7 @@ buildout_and_distribute_path = [
FILE_SCHEME = re.compile('file://', re.I).match
class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
"""Will allow urls that are local to the system.
......@@ -120,11 +121,14 @@ _easy_install_cmd = 'from setuptools.command.easy_install import main; main()'
class Installer:
_versions = {}
_required_by = {}
_picked_versions = {}
_download_cache = None
_install_from_cache = False
_prefer_final = True
_use_dependency_links = True
_allow_picked_versions = True
_show_picked_versions = False
def __init__(self,
dest=None,
......@@ -167,7 +171,7 @@ class Installer:
self._index = _get_index(index, links, self._allow_hosts)
if versions is not None:
self._versions = versions
self._versions = normalize_versions(versions)
def _satisfied(self, req, source=None):
dists = [dist for dist in self._env[req.project_name] if dist in req]
......@@ -516,6 +520,8 @@ class Installer:
):
logger.debug('Picked: %s = %s',
dist.project_name, dist.version)
self._picked_versions[dist.project_name] = dist.version
if not self._allow_picked_versions:
raise zc.buildout.UserError(
'Picked: %s = %s' % (dist.project_name, dist.version)
......@@ -545,10 +551,9 @@ class Installer:
def _constrain(self, requirement):
constraint = self._versions.get(requirement.project_name)
constraint = self._versions.get(requirement.project_name.lower())
if constraint:
requirement = _constrained_requirement(constraint, requirement)
return requirement
def install(self, specs, working_set=None):
......@@ -680,10 +685,20 @@ class Installer:
if tmp != self._download_cache:
shutil.rmtree(tmp)
def normalize_versions(versions):
"""Return version dict with keys normalized to lowercase.
PyPI is case-insensitive and not all distributions are consistent in
their own naming.
"""
return dict([(k.lower(), v) for (k, v) in versions.items()])
def default_versions(versions=None):
old = Installer._versions
if versions is not None:
Installer._versions = versions
Installer._versions = normalize_versions(versions)
return old
def download_cache(path=-1):
......@@ -718,6 +733,13 @@ def allow_picked_versions(setting=None):
Installer._allow_picked_versions = bool(setting)
return old
def show_picked_versions(setting=None):
old = Installer._show_picked_versions
if setting is not None:
Installer._show_picked_versions = bool(setting)
return old
def install(specs, dest,
links=(), index=None,
executable=sys.executable,
......@@ -1226,9 +1248,10 @@ class MissingDistribution(zc.buildout.UserError):
return "Couldn't find a distribution for %r." % str(req)
def _log_requirement(ws, req):
if not logger.isEnabledFor(logging.DEBUG):
if (not logger.isEnabledFor(logging.DEBUG) and
not Installer._show_picked_versions):
# Sorting the working set and iterating over it's requirements
# is expensive, so short cirtuit the work if it won't even be
# is expensive, so short circuit the work if it won't even be
# logged. When profiling a simple buildout with 10 parts with
# identical and large working sets, this resulted in a
# decrease of run time from 93.411 to 15.068 seconds, about a
......@@ -1240,6 +1263,10 @@ def _log_requirement(ws, req):
for dist in ws:
if req in dist.requires():
logger.debug(" required by %s." % dist)
req_ = str(req)
if req_ not in Installer._required_by:
Installer._required_by[req_] = set()
Installer._required_by[req_].add(str(dist.as_requirement()))
def _fix_file_links(links):
for link in links:
......
src/zc/buildout/repeatable.txt
View file @ eeb0c30e
......@@ -227,3 +227,166 @@ We can also disable checking versions:
Uninstalling foo.
Installing foo.
recipe v2
Easier reporting and managing of versions (new in buildout 2.0)
---------------------------------------------------------------
Since buildout 2.0, the functionality of the `buildout-versions
<http://packages.python.org/buildout-versions/>`_ extension is part of
buildout itself. This makes reporting and managing versions easier.
If you set the ``show-picked-versions`` option, buildout will print
versions it picked at the end of its run:
>>> write('buildout.cfg',
... '''
... [buildout]
... parts = foo
... find-links = %s
... show-picked-versions = true
...
... [versions]
...
... [foo]
... recipe = spam
... ''' % join('recipe', 'dist'))
>>> print_(system(buildout), end='') # doctest: +ELLIPSIS
Updating foo.
recipe v2
Versions had to be automatically picked.
The following part definition lists the versions picked:
[versions]
distribute = 0.6.99
spam = 2
When everything is pinned, no output is generated:
>>> write('buildout.cfg',
... '''
... [buildout]
... parts = foo
... find-links = %s
... show-picked-versions = true
...
... [versions]
... distribute = 0.6.34
... spam = 2
...
... [foo]
... recipe = spam
... ''' % join('recipe', 'dist'))
>>> print_(system(buildout), end='') # doctest: +ELLIPSIS
Updating foo.
recipe v2
The Python package index is case-insensitive. Both
http://pypi.python.org/simple/Django/ and
http://pypi.python.org/simple/dJaNgO/ work. And distributions aren't always
naming themselves consistently case-wise. So all version names are normalized
and case differences won't impact the pinning:
>>> write('buildout.cfg',
... '''
... [buildout]
... parts = foo
... find-links = %s
... show-picked-versions = true
...
... [versions]
... distriBUTE = 0.6.34
... Spam = 2
...
... [foo]
... recipe = spam
... ''' % join('recipe', 'dist'))
>>> print_(system(buildout), end='') # doctest: +ELLIPSIS
Updating foo.
recipe v2
Sometimes it is handy to have a separate file with versions. This is a regular
buildout file with a single ``[versions]`` section. You include it by
extending from that versions file:
>>> write('my_versions.cfg',
... '''
... [versions]
... distribute = 0.6.34
... spam = 2
... ''')
>>> write('buildout.cfg',
... '''
... [buildout]
... parts = foo
... extends = my_versions.cfg
... find-links = %s
... show-picked-versions = true
...
... [foo]
... recipe = spam
... ''' % join('recipe', 'dist'))
>>> print_(system(buildout), end='') # doctest: +ELLIPSIS
Updating foo.
recipe v2
If not everything is pinned and buildout has to pick versions, you can tell
buildout to append the versions to your versions file. It simply appends them
at the end.
>>> write('my_versions.cfg',
... '''
... [versions]
... distribute = 0.6.34
... ''')
>>> write('buildout.cfg',
... '''
... [buildout]
... parts = foo
... extends = my_versions.cfg
... update-versions-file = my_versions.cfg
... find-links = %s
... show-picked-versions = true
...
... [foo]
... recipe = spam
... ''' % join('recipe', 'dist'))
>>> print_(system(buildout), end='') # doctest: +ELLIPSIS
Updating foo.
recipe v2
Versions had to be automatically picked.
The following part definition lists the versions picked:
[versions]
spam = 2
This information has been written to my_versions.cfg
The versions file now contains the extra pin:
>>> 'spam = 2' in open('my_versions.cfg').read()
True
And re-running buildout doesn't report any picked versions anymore:
>>> print_(system(buildout), end='') # doctest: +ELLIPSIS
Updating foo.
recipe v2
Because buildout now includes buildout-versions' (and the older
buildout.dumppickedversions') functionality, it warns if these extensions are
configured.
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... parts = foo
... extensions = buildout-versions
...
... [foo]
... recipe = spam
... """)
>>> print_(system(buildout), end='') # doctest: +ELLIPSIS
While:
Installing.
Loading extensions.
Error: The extension buildout-versions is now included in buildout itself.
Remove the extension from your configuration and look at the `show-picked-versions`
option in buildout's documentation.
src/zc/buildout/tests.py
View file @ eeb0c30e
......@@ -404,6 +404,41 @@ that we can't find. when run in verbose mode
Error: Couldn't find a distribution for 'demoneeded'.
"""
def show_who_requires_picked_versions():
"""
The show-picked-versions prints the versions, but it also prints who
required the picked distributions.
We do not need to run in verbose mode for that to work:
>>> make_dist_that_requires(sample_buildout, 'sampley', ['distribute'])
>>> make_dist_that_requires(sample_buildout, 'samplea', ['sampleb'])
>>> make_dist_that_requires(sample_buildout, 'sampleb',
... ['sampley', 'samplea'])
>>> write('buildout.cfg',
... '''
... [buildout]
... parts = eggs
... show-picked-versions = true
... develop = sampley samplea sampleb
...
... [eggs]
... recipe = zc.recipe.egg
... eggs = samplea
... ''')
>>> print_(system(buildout), end='') # doctest: +ELLIPSIS
Develop: ...
Installing eggs.
Versions had to be automatically picked.
The following part definition lists the versions picked:
[versions]
<BLANKLINE>
# Required by:
# sampley==1
distribute = 0.6.99
"""
def test_comparing_saved_options_with_funny_characters():
"""
If an option has newlines, extra/odd spaces or a %, we need to make sure
......@@ -3205,6 +3240,7 @@ def test_suite():
'We have a develop egg: zc.buildout X.X.'),
(re.compile(r'\\[\\]?'), '/'),
(re.compile('WindowsError'), 'OSError'),
(re.compile('distribute = \S+'), 'distribute = 0.6.99'),
(re.compile(r'\[Error 17\] Cannot create a file '
r'when that file already exists: '),
'[Errno 17] File exists: '
......@@ -3320,6 +3356,7 @@ def test_suite():
'distribute.egg'),
(re.compile('zc.buildout-\S+-'),
'zc.buildout.egg'),
(re.compile('distribute = \S+'), 'distribute = 0.6.99'),
(re.compile('File "\S+one.py"'),
'File "one.py"'),
(re.compile(r'We have a develop egg: (\S+) (\S+)'),
......@@ -3391,4 +3428,3 @@ def test_suite():
))
return unittest.TestSuite(test_suite)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7