Skip to content

S
slapos.buildout
Commit c1598ef6 authored by Jim Fulton's avatar Jim Fulton
Browse files
Options

Improved intro text and description of setup. 

Added documentation of buildout:directory option.

Added support for extending configurations through multiple
configuration files.

Added command-line options to:

- Specify a configuration file

- Override configuration options

Renamed several options to use -s rather than _s.
parent 299c235c
Hide whitespace changes
Inline Side-by-side
Showing with 658 additions and 93 deletions
src/zc/buildout/buildout.py
View file @ c1598ef6
......@@ -51,46 +51,40 @@ class Options(dict):
class Buildout(dict):
def __init__(self):
self._buildout_dir = os.path.abspath(os.getcwd())
self._config_file = self.buildout_path('buildout.cfg')
super(Buildout, self).__init__(self._open(
directory = self._buildout_dir,
eggs_directory = 'eggs',
bin_directory = 'bin',
parts_directory = 'parts',
installed = '.installed.cfg',
))
options = self['buildout']
links = options.get('find_links', '')
self._links = links and links.split() or ()
for name in ('bin', 'parts', 'eggs'):
d = self.buildout_path(options[name+'_directory'])
setattr(self, name, d)
if not os.path.exists(d):
os.mkdir(d)
def __init__(self, config_file, cloptions):
config_file = os.path.abspath(config_file)
self._config_file = config_file
super(Buildout, self).__init__()
# default options
data = dict(buildout={'directory': os.path.dirname(config_file),
'eggs-directory': 'eggs',
'bin-directory': 'bin',
'parts-directory': 'parts',
'installed': '.installed.cfg',
},
)
# load user defaults, which override defaults
if 'HOME' in os.environ:
user_config = os.path.join(os.environ['HOME'],
'.buildout', 'default.cfg')
if os.path.exists(user_config):
_update(data, _open(os.path.dirname(user_config), user_config,
[]))
# load configuration files
_update(data, _open(os.path.dirname(config_file), config_file, []))
# apply command-line options
for (section, option, value) in cloptions:
options = data.get(section)
if options is None:
options = self[section] = {}
options[option] = value
_template_split = re.compile('([$]{\w+:\w+})').split
def _open(self, **predefined):
# Open configuration files
parser = ConfigParser.SafeConfigParser()
parser.add_section('buildout')
for k, v in predefined.iteritems():
parser.set('buildout', k, v)
parser.read(self._config_file)
data = dict([
(section,
Options(self, section,
[(k, v.strip()) for (k, v) in parser.items(section)])
)
for section in parser.sections()
])
# do substitutions
converted = {}
for section, options in data.iteritems():
for option, value in options.iteritems():
......@@ -100,7 +94,22 @@ class Buildout(dict):
options[option] = value
converted[(section, option)] = value
return data
# copy data into self:
for section, options in data.iteritems():
self[section] = Options(self, section, options)
# initialize some attrs and buildout directories.
options = self['buildout']
links = options.get('find-links', '')
self._links = links and links.split() or ()
self._buildout_dir = options['directory']
for name in ('bin', 'parts', 'eggs'):
d = self.buildout_path(options[name+'-directory'])
setattr(self, name, d)
if not os.path.exists(d):
os.mkdir(d)
def _dosubs(self, section, option, value, data, converted, seen):
key = section, option
......@@ -116,6 +125,7 @@ class Buildout(dict):
seen.pop()
return value
_template_split = re.compile('([$]{\w+:\w+})').split
def _dosubs_esc(self, value, data, converted, seen):
value = self._template_split(value)
subs = []
......@@ -141,7 +151,7 @@ class Buildout(dict):
def buildout_path(self, *names):
return os.path.join(self._buildout_dir, *names)
def install(self):
def install(self, install_parts):
self._develop()
new_part_options = self._gather_part_info()
installed_part_options = self._read_installed_part_options()
......@@ -150,6 +160,12 @@ class Buildout(dict):
new_old_parts = []
for part in old_parts:
if install_parts and (part not in install_parts):
# We were asked to install specific parts and this
# wasn't one of them. Leave it alone.
new_old_parts.append(part)
continue
installed_options = installed_part_options[part].copy()
installed = installed_options.pop('__buildout_installed__')
if installed_options != new_part_options.get(part):
......@@ -162,10 +178,11 @@ class Buildout(dict):
new_parts = []
try:
for part in new_part_options['buildout']['parts'].split():
installed = self._install(part)
new_part_options[part]['__buildout_installed__'] = installed
if (not install_parts) or (part in install_parts):
installed = self._install(part)
new_part_options[part]['__buildout_installed__'] = installed
installed_part_options[part] = new_part_options[part]
new_parts.append(part)
installed_part_options[part] = new_part_options[part]
new_old_parts = [p for p in new_old_parts if p != part]
finally:
new_parts.extend(new_old_parts)
......@@ -274,14 +291,62 @@ class Buildout(dict):
for d in installed]
return ' '.join(installed)
def _save_installed_options(self, installed_options):
parser = ConfigParser.SafeConfigParser()
for section in installed_options:
parser.add_section(section)
for option, value in installed_options[section].iteritems():
parser.set(section, option, value)
parser.write(open(self._installed_path(), 'w'))
f = open(self._installed_path(), 'w')
_save_options('buildout', installed_options['buildout'], f)
for part in installed_options['buildout']['parts'].split():
print >>f
_save_options(part, installed_options[part], f)
f.close()
def _save_options(section, options, f):
print >>f, '[%s]' % section
items = options.items()
items.sort()
for option, value in items:
print >>f, option, '=', str(value).replace('\n', '\n\t')
def _open(base, filename, seen):
"""Open a configuration file and return the result as a dictionary,
Recursively open other files based on buildout options found.
"""
filename = os.path.join(base, filename)
if filename in seen:
raise ValueError("Recursive file include", seen, filename)
base = os.path.dirname(filename)
seen.append(filename)
result = {}
parser = ConfigParser.SafeConfigParser()
parser.readfp(open(filename))
extends = extended_by = None
for section in parser.sections():
options = dict(parser.items(section))
if section == 'buildout':
extends = options.pop('extends', extends)
extended_by = options.pop('extended-by', extended_by)
result[section] = options
if extends:
extends = extends.split()
extends.reverse()
for fname in extends:
result = _update(_open(base, fname, seen), result)
if extended_by:
for fname in extended_by.split():
result = _update(result, _open(base, fname, seen))
seen.pop()
return result
def _dir_hash(dir):
hash = md5.new()
for (dirpath, dirnames, filenames) in os.walk(dir):
......@@ -306,5 +371,44 @@ def _dists_sig(dists, base):
result.append(location)
return result
def main():
Buildout().install()
def _update(d1, d2):
for section in d2:
if section in d1:
d1[section].update(d2[section])
else:
d1[section] = d2[section]
return d1
def _error(*message):
sys.syderr.write(' '.join(message) +'\n')
sys.exit(1)
def main(args=None):
if args is None:
args = sys.argv[1:]
if args and args[0] == '-c':
args.pop(0)
if not args:
_error("No configuration file specified,")
config_file = args.pop(0)
else:
config_file = 'buildout.cfg'
options = []
while args and '=' in args[0]:
option, value = args.pop(0).split('=', 1)
if len(option.split(':')) != 2:
_error('Invalid option:', option)
section, option = option.split(':')
options.append((section.strip(), option.strip(), value.strip()))
buildout = Buildout(config_file, options)
if args:
command = args.pop(0)
if command != 'install':
_error('invalid command:', command)
else:
command = 'install'
getattr(buildout, command)(args)
src/zc/buildout/buildout.txt
View file @ c1598ef6
......@@ -2,22 +2,22 @@ Defining Buildouts
==================
This document describes how to define buildouts using buildout
configuation files and recipes. It doesn't describe how to bootstrap
a buildout. To find out how to do that, see bootstrap.txt. For the
examples we show here, we've created a sample buildout that already
contains the mimimal software needed for a buildout.
configuation files and recipes. There are two ways to set up the
buildout software and create a buildout:
Buildouts are defined using configuration files. These are in the
format defined by the Python ConfigParser module, with an extension
that we'll describe later. When a buildout is run, it looks for
the file buildout.cfg in the directory where the buidout is run. It
will optionally look for buildout-instance.cfg. Typically, buidout.cfg
contains information common to all instances of a buildout and is
checked in, and buildout-instance.cfg has instance-specific information.
1. Install the zc.buildout egg with easy_install and use the buildout
script installed in a Python scripts area.
2. Use the buildout bootstrap script to install both the setuptools
and zc.buildout eggs into your buildout. This allows you to use
the buildout software without modifying a Python install.
The buildout script is installed into your buildout local scripts
area.
We have a sample buildout that has already been created for us. It
has the absolute minimum information. We have bin, eggs and parts
directories, and a configuration file:
directories, a configuration file, and an .installed,cfg that contains
informatiion about previously-installed parts:
>>> ls(sample_buildout)
- .installed.cfg
......@@ -26,24 +26,20 @@ directories, and a configuration file:
d eggs
d parts
The bin directory contains scripts. A minimal buildout has a build
script and a py_zc.buildout script:
The bin directory contains scripts. In the examples shown here, we've
used a hybrid approach for creating the to ease automated setup. We
have a buildout script in our buildout script directory, but the eggs
actually live elsewhere.
>>> ls(sample_buildout, 'bin')
- buildout
The build script is what we run to build things out. The
py_zc.buildout script gives us a Python prompt with the Python path
set to that needed by the zc.buildout package.
The eggs directory is initially empty. This is typically the case
when the zc.buildout and setuptools are installed externally to the
buildout:
>>> ls(sample_buildout, 'eggs')
They can also be installed locally in a buildout, in which case they's
show up as eggs in the eggs directory.
Buildouts are defined using configuration files. These are in the
format defined by the Python ConfigParser module, with an extension
that we'll describe later. When a buildout is run, it looks for the
file buildout.cfg in the directory where the buidout is run.
The parts directory is initially empty:
......@@ -62,7 +58,6 @@ interesting:
[buildout]
parts =
The minimal configuration file has a buildout section that defines no
parts:
......@@ -76,7 +71,7 @@ a confguration file.
A part is created by a recipe. Recipes are always installed as Python
eggs. They can be downloaded from an package server, such as the
Python package index, or they can be developed as part of a project.
Python Package Index, or they can be developed as part of a project.
Let's create a recipe as part of the sample project. We'll create a
recipe for creating directories.
......@@ -224,11 +219,10 @@ installed:
parts = data_dir
<BLANKLINE>
[data_dir]
__buildout_installed__ = mystuff
__buildout_signature__ = recipes-O3ypTgKOkHMqMwKvMfvHnA==
path = mystuff
recipe = recipes:mkdir
__buildout_signature__ = recipes-O3ypTgKOkHMqMwKvMfvHnA==
__buildout_installed__ = mystuff
<BLANKLINE>
Note that the directory we installed is included in .installed.cfg.
......@@ -361,26 +355,454 @@ the buildout:
We can see that mydata was not recreated.
Multiple configuration files
----------------------------
You can use multiple configuration files. From your main
configuration file, you can include other configuration files in 2
ways:
- Your configuration file can "extend" another configuration file.
Option are read from the other configuration file if they aren't
already defined by your configuration file.
- Your configuration file can be "extended-by" another configuration
file, In this case, the options in the other configuration file
override options in your configuration file.
The configuration files your file extends or is extended by can extend
or be extended by other configuration files. The same file may be
used more than once although, of course, cycles aren't allowed.
To see how this works, we use an example:
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... extends = base.cfg
...
... [debug]
... op = buldout
... """)
>>> write(sample_buildout, 'base.cfg',
... """
... [buildout]
... develop = recipes
... parts = debug
...
... [debug]
... recipe = recipes:debug
... op = base
... """)
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
op buldout
recipe recipes:debug
The example is pretty trivial, but the pattern it illustrates is
pretty common. In a more practical example, the base buildout might
represent a product and the extending buildout might be a
customization.
Here is a more eleborate example.
>>> import tempfile
>>> extensions = tempfile.mkdtemp()
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... extends = b1.cfg b2.cfg
... extended-by = e1.cfg %(e2)s
...
... [debug]
... op = %%(name)s
...
... [DEFAULT]
... name = buildout
... """ % dict(e2=os.path.join(extensions, 'e2.cfg')))
>>> write(sample_buildout, 'b1.cfg',
... """
... [buildout]
... extends = base.cfg
...
... [debug]
... op1 = %(name)s 1
... op2 = %(name)s 2
... op3 = %(name)s 3
...
... [DEFAULT]
... name = b1
... """)
>>> write(sample_buildout, 'b2.cfg',
... """
... [buildout]
... extends = base.cfg
...
... [debug]
... op3 = %(name)s 3
... op4 = %(name)s 4
... op5 = %(name)s 5
...
... [DEFAULT]
... name = b2
... """)
>>> write(sample_buildout, 'base.cfg',
... """
... [buildout]
... develop = recipes
... parts = debug
...
... [debug]
... recipe = recipes:debug
... name = base
... """)
>>> write(sample_buildout, 'e1.cfg',
... """
... [debug]
... op1 = %(name)s 1
...
... [DEFAULT]
... name = e1
... """)
>>> write(extensions, 'e2.cfg',
... """
... [buildout]
... extends = eb.cfg
... extended-by = ee.cfg
... """)
>>> write(extensions, 'eb.cfg',
... """
... [debug]
... op5 = %(name)s 5
...
... [DEFAULT]
... name = eb
... """)
>>> write(extensions, 'ee.cfg',
... """
... [debug]
... op6 = %(name)s 6
...
... [DEFAULT]
... name = ee
... """)
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
name ee
op buildout
op1 e1 1
op2 b1 2
op3 b2 3
op4 b2 4
op5 eb 5
op6 ee 6
recipe recipes:debug
There are several things to note about this example:
- We can name multiple files in an extends or extended-by option.
- We can reference files recursively.
- DEFAULT sections only directly affect the configuration file they're
used in, but they can have secondary effects. For example, the name
option showed up in the debug section because it was defined in the
debug sections in several of the input files by virtue of being in
their DEFAULT sections.
- Relative file names are determined relative to the directory
containing the referencing configuration file. The files eb.cfg and
ee.cfg were found in the extensions directory because they were
referenced from a file in that directory.
User defaults
-------------
If the file $HOME/.buildout/defaults.cfg, exists, it is read before
reading the configuration file. ($HOME is the value of the HOME
enviornment variable. The '/' is replaced by the operating system file
delimiter.)
>>> home = tempfile.mkdtemp()
>>> mkdir(home, '.buildout')
>>> write(home, '.buildout', 'default.cfg',
... """
... [debug]
... op1 = 1
... op7 = 7
... """)
>>> os.environ['HOME'] = home
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
name ee
op buildout
op1 e1 1
op2 b1 2
op3 b2 3
op4 b2 4
op5 eb 5
op6 ee 6
op7 7
recipe recipes:debug
Command-line usage
------------------
A number of arguments can be given on the buildout command line. The
command usage is::
buildout [-c file] [options] [command [command arguments]]
The -c option can be used to specify a configuration file, rather than
buildout.cfg in the current durectory. Options are of the form::
section_name:option_name=value
for example, as in:
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')
... + ' debug:op1=foo'),
name ee
op buildout
op1 foo
op2 b1 2
op3 b2 3
op4 b2 4
op5 eb 5
op6 ee 6
op7 7
recipe recipes:debug
Currently, the default and only command is 'install' and it takes a
list of parts to install. if any parts are specified, then only those
parts are installed. To illustrate this, we'll update our
configuration and run the buildout in the usual way:
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... develop = recipes
... parts = debug d1 d2 d3
...
... [d1]
... recipe = recipes:mkdir
... path = d1
...
... [d2]
... recipe = recipes:mkdir
... path = d2
...
... [d3]
... recipe = recipes:mkdir
... path = d3
...
... [debug]
... recipe = recipes:debug
... """)
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
op1 1
op7 7
recipe recipes:debug
Creating directory d1
Creating directory d2
Creating directory d3
>>> ls(sample_buildout)
- .installed.cfg
- b1.cfg
- b2.cfg
- base.cfg
d bin
- buildout.cfg
d d1
d d2
d d3
- e1.cfg
d eggs
d parts
d recipes
>>> cat(sample_buildout, '.installed.cfg')
[buildout]
parts = debug d1 d2 d3
<BLANKLINE>
[debug]
__buildout_installed__ =
__buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
op1 = 1
op7 = 7
recipe = recipes:debug
<BLANKLINE>
[d1]
__buildout_installed__ = d1
__buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
path = d1
recipe = recipes:mkdir
<BLANKLINE>
[d2]
__buildout_installed__ = d2
__buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
path = d2
recipe = recipes:mkdir
<BLANKLINE>
[d3]
__buildout_installed__ = d3
__buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
path = d3
recipe = recipes:mkdir
Now we'll update our configuration file:
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... develop = recipes
... parts = debug d2 d3 d4
...
... [d2]
... recipe = recipes:mkdir
... path = data2
...
... [d3]
... recipe = recipes:mkdir
... path = data3
...
... [d4]
... recipe = recipes:mkdir
... path = data4
...
... [debug]
... recipe = recipes:debug
... x = 1
... """)
and run the buildout specifying just d2 and d3:
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')
... + ' install d3 d4'),
Creating directory data3
Creating directory data4
>>> ls(sample_buildout)
- .installed.cfg
- b1.cfg
- b2.cfg
- base.cfg
d bin
- buildout.cfg
d d1
d d2
d data3
d data4
- e1.cfg
d eggs
d parts
d recipes
Only the d2 and d3 recipes ran. d2 was removed and data2 and data3
were created.
The .installed.cfg is only updated for the recipes that ran:
>>> cat(sample_buildout, '.installed.cfg')
[buildout]
parts = debug d2 d3 d4 d1
<BLANKLINE>
[debug]
__buildout_installed__ =
__buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
op1 = 1
op7 = 7
recipe = recipes:debug
<BLANKLINE>
[d2]
__buildout_installed__ = d2
__buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
path = d2
recipe = recipes:mkdir
<BLANKLINE>
[d3]
__buildout_installed__ = data3
__buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
path = data3
recipe = recipes:mkdir
<BLANKLINE>
[d4]
__buildout_installed__ = data4
__buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
path = data4
recipe = recipes:mkdir
<BLANKLINE>
[d1]
__buildout_installed__ = d1
__buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
path = d1
recipe = recipes:mkdir
Note that the installed data for debug, d1, and d2 haven't changed,
because we didn't install those parts and that the d1 and d2
directories are still there.
Now, if we run the buildout without arguments:
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
op1 1
op7 7
recipe recipes:debug
x 1
Creating directory data2
We see the output of the debug recipe and that data2 was created. We
also see that d1 and d2 have gone away:
>>> ls(sample_buildout)
- .installed.cfg
- b1.cfg
- b2.cfg
- base.cfg
d bin
- buildout.cfg
d data2
d data3
d data4
- e1.cfg
d eggs
d parts
d recipes
Alternate directory locations
-----------------------------
The buildout normally puts the bin, eggs, and parts directories in the
directory it is run from. You can provide alternate locations, and
even names for these directories.
directory in the directory containing the configuration file. You can
provide alternate locations, and even names for these directories.
>>> import tempfile
>>> alt = tempfile.mkdtemp()
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... develop = recipes
... parts =
... eggs_directory = %(alt)s/basket
... bin_directory = %(alt)s/scripts
... parts_directory = %(alt)s/work
... """ % dict(alt=alt))
... eggs-directory = %(basket)s
... bin-directory = %(scripts)s
... parts-directory = %(work)s
... """ % dict(
... basket = os.path.join(alt, 'basket'),
... scripts = os.path.join(alt, 'scripts'),
... work = os.path.join(alt, 'work'),
... ))
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
......@@ -394,3 +816,32 @@ even names for these directories.
>>> import shutil
>>> shutil.rmtree(alt)
You can also specify an alternate buildout directory:
>>> alt = tempfile.mkdtemp()
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... directory = %(alt)s
... develop = %(recipes)s
... parts =
... """ % dict(
... alt=alt,
... recipes=os.path.join(sample_buildout, 'recipes'),
... ))
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
>>> ls(alt)
- .installed.cfg
d bin
d eggs
d parts
>>> ls(alt, 'eggs')
- recipes.egg-link
>>> import shutil
>>> shutil.rmtree(alt)
src/zc/buildout/tests.py
View file @ c1598ef6
......@@ -29,7 +29,7 @@ Asking for a section that doesn't exist, yields a key error:
>>> import os
>>> os.chdir(sample_buildout)
>>> import zc.buildout.buildout
>>> buildout = zc.buildout.buildout.Buildout()
>>> buildout = zc.buildout.buildout.Buildout('buildout.cfg', [])
>>> buildout['eek']
Traceback (most recent call last):
...
......@@ -62,9 +62,7 @@ It is an error to create a variable-reference cycle:
[('buildout', 'y'), ('buildout', 'z'), ('buildout', 'x')],
('buildout', 'y'))
'''
'''
def runsetup(d):
here = os.getcwd()
......@@ -119,6 +117,18 @@ def linkerSetUp(test):
def linkerTearDown(test):
shutil.rmtree(test.globs['_sample_eggs_container'])
zc.buildout.testing.buildoutTearDown(test)
def buildoutSetUp(test):
zc.buildout.testing.buildoutSetUp(test)
test.globs['_oldhome'] = os.environ.get('HOME')
def buildoutTearDoen(test):
if test.globs['_oldhome'] is not None:
os.environ['HOME'] = test.globs['_oldhome']
shutil.rmtree(test.globs['extensions'])
shutil.rmtree(test.globs['home'])
zc.buildout.testing.buildoutTearDown(test)
def test_suite():
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7