Merge pull request #87 from buildout/jim-meta-recipes

Jim meta recipes

CHANGES.rst

 Change History
 **************
 
+2.1.0 (2013-??-??)
+==================
+
+- Added `meta-recipe support`_.
+
 2.0.1 (2013-02-16)
 ==================
 

setup.py

@@ -12,7 +12,7 @@
 #
 ##############################################################################
 name = "zc.buildout"
-version = "2.0.0b2"
+version = "2.0.1"
 
 import os
 from setuptools import setup
@@ -53,6 +53,8 @@ long_description=(
         + '\n' +
         read('src', 'zc', 'buildout', 'debugging.txt')
         + '\n' +
+        read('src', 'zc', 'buildout', 'meta-recipes.txt')
+        + '\n' +
         read('src', 'zc', 'buildout', 'testing.txt')
         + '\n' +
         read('src', 'zc', 'buildout', 'easy_install.txt')

src/zc/buildout/buildout.py

@@ -157,7 +157,7 @@ class Buildout(DictMixin):
         data = dict(buildout=_buildout_default_options.copy())
         self._buildout_dir = os.getcwd()
 
-        if not _isurl(config_file):
+        if config_file and not _isurl(config_file):
             config_file = os.path.abspath(config_file)
             base = os.path.dirname(config_file)
             if not os.path.exists(config_file):
@@ -762,11 +762,11 @@ class Buildout(DictMixin):
                         for k, v in _spacey_defaults:
                             value = value.replace(k, v)
                         options[option] = value
-                result[section] = Options(self, section, options)
+                result[section] = self.Options(self, section, options)
 
             return result, True
         else:
-            return ({'buildout': Options(self, 'buildout', {'parts': ''})},
+            return ({'buildout': self.Options(self, 'buildout', {'parts': ''})},
                     False,
                     )
 
@@ -829,7 +829,8 @@ class Buildout(DictMixin):
             try:
                 timeout = int(timeout)
                 import socket
-                self._logger.info('Setting socket time out to %d seconds.', timeout)
+                self._logger.info(
+                    'Setting socket time out to %d seconds.', timeout)
                 socket.setdefaulttimeout(timeout)
             except ValueError:
                 self._logger.warning("Default socket timeout is used !\n"
@@ -1063,9 +1064,21 @@ class Buildout(DictMixin):
 
     runsetup = setup # backward compat.
 
-    def annotate(self, args):
+    def annotate(self, args=None):
         _print_annotate(self._annotated)
 
+    def print_options(self):
+        for section in sorted(self._data):
+            if section == 'buildout' or section == self['buildout']['versions']:
+                continue
+            print_('['+section+']')
+            for k, v in sorted(self._data[section].items()):
+                if '\n' in v:
+                    v = '\n  ' + v.replace('\n', '\n  ')
+                else:
+                    v = ' '+v
+                print_("%s =%s" % (k, v))
+
     def __getitem__(self, section):
         __doing__ = 'Getting section %s.', section
         try:
@@ -1078,13 +1091,34 @@ class Buildout(DictMixin):
         except KeyError:
             raise MissingSection(section)
 
-        options = Options(self, section, data)
+        options = self.Options(self, section, data)
         self._data[section] = options
         options._initialize()
         return options
 
-    def __setitem__(self, key, value):
-        raise NotImplementedError('__setitem__')
+    def __setitem__(self, name, data):
+        if name in self._raw:
+            raise KeyError("Section already exists", name)
+        self._raw[name] = dict((k, str(v)) for (k, v) in data.items())
+        self[name] # Add to parts
+
+    def parse(self, data):
+        try:
+            from cStringIO import StringIO
+        except ImportError:
+            from io import StringIO
+        import textwrap
+
+        sections = zc.buildout.configparser.parse(
+            StringIO(textwrap.dedent(data)), '')
+        for name in sections:
+            if name in self._raw:
+                raise KeyError("Section already exists", name)
+            self._raw[name] = dict((k, str(v))
+                                   for (k, v) in sections[name].items())
+
+        for name in sections:
+            self[name] # Add to parts
 
     def __delitem__(self, key):
         raise NotImplementedError('__delitem__')
@@ -1159,20 +1193,20 @@ class Options(DictMixin):
             if '${' in v:
                 self._dosub(k, v)
 
-        if self.name == 'buildout':
+        if name == 'buildout':
             return # buildout section can never be a part
 
-        recipe = self.get('recipe')
-        if not recipe:
-            return
+        if self.get('recipe'):
+            self.initialize()
+            self.buildout._parts.append(name)
 
+    def initialize(self):
         reqs, entry = _recipe(self._data)
         buildout = self.buildout
         recipe_class = _install_and_load(reqs, 'zc.buildout', entry, buildout)
 
-        __doing__ = 'Initializing part %s.', name
+        name = self.name
         self.recipe = recipe_class(buildout, name, self)
-        buildout._parts.append(name)
 
     def _do_extend_raw(self, name, data, doing):
         if name == 'buildout':
@@ -1357,6 +1391,8 @@ class Options(DictMixin):
     def __repr__(self):
         return repr(dict(self))
 
+Buildout.Options = Options
+
 _spacey_nl = re.compile('[ \t\r\f\v]*\n[ \t\r\f\v\n]*'
                         '|'
                         '^[ \t\r\f\v]+'

src/zc/buildout/buildout.txt

@@ -390,7 +390,7 @@ We'll get a user error, not a traceback.
     While:
       Installing.
       Getting section data-dir.
-      Initializing part data-dir.
+      Initializing section data-dir.
     Error: Invalid Path
 
 

src/zc/buildout/meta-recipes.txt

+Meta-recipe support
+===================
+
+
+Buildout recipes provide reusable Python modules for common
+configuration tasks. The most widely used recipes tend to provide
+low-level functions, like installing eggs or software distributions,
+creating configuration files, and so on.  The normal recipe framework
+is fairly well suited to building these general components.
+
+Full-blown applications may require many, often tens, of parts.
+Defining the many parts that make up an application can be tedious and
+often entails a lot of repetition.  Buildout provides a number of
+mechanisms to avoid repetition, including merging of configuration
+files and macros, but these, while useful to an extent, don't scale
+very well.  Buildout isn't and shouldn't be a programming language.
+
+Meta-recipes allow us to bring Python to bear to provide higher-level
+abstractions for buildouts.
+
+A meta-recipe is a regular Python recipe that primarily operates by
+creating parts.  A meta recipe isn't merely a high level recipe.  It's
+a recipe that defers most or all of it's work to lower-level recipes by
+manipulating the buildout database.
+
+A `presentation at PyCon 2011
+<http://blip.tv/pycon-us-videos-2009-2010-2011/pycon-2011-deploying-applications-with-zc-buildout-4897770>`_
+described early work with meta recipes.
+
+A simple meta-recipe example
+============================
+
+Let's look at a fairly simple meta-recipe example.  First, consider a
+buildout configuration that builds a database deployment::
+
+  [buildout]
+  parts = ctl pack
+
+  [deployment]
+  recipe = zc.recipe.deployment
+  name = ample
+  user = zope
+
+  [ctl]
+  recipe = zc.recipe.rhrc
+  deployment = deployment
+  chkconfig = 345 99 10
+  parts = main
+
+  [main]
+  recipe = zc.zodbrecipes:server
+  deployment = deployment
+  address = 8100
+  path = /var/databases/ample/main.fs
+  zeo.conf =
+     <zeo>
+        address ${:address}
+     </zeo>
+     %import zc.zlibstorage
+     <zlibstorage>
+       <filestorage>
+          path ${:path}
+       </filestorage>
+     </zlibstorage>
+
+  [pack]
+  recipe = zc.recipe.deployment:crontab
+  deployment = deployment
+  times = 1 2 * * 6
+  command = ${buildout:bin-directory}/zeopack -d3 -t00 ${main:address}
+
+.. -> low_level
+
+This buildout doesn't build software. Rather it builds configuration
+for deploying a database configuration using already-deployed
+software.  For the purpose of this document, however, the details are
+totally unimportant.
+
+Rather than crafting the configuration above every time, we can write
+a meta-recipe that crafts it for us.  We'll use our meta-recipe as
+follows::
+
+  [buildout]
+  parts = ample
+
+  [ample]
+  recipe = com.example.ample:db
+  path = /var/databases/ample/main.fs
+
+The idea here is that the meta recipe allows us to specify the minimal
+information necessary.  A meta-recipe often automates policies and
+assumptions that are application and organization dependent.  The
+example above assumes, for example, that we want to pack to 3
+days in the past on Saturdays.
+
+So now, let's see the meta recipe that automates this::
+
+  class Recipe:
+
+      def __init__(self, buildout, name, options):
+
+          buildout.parse('''
+              [deployment]
+              recipe = zc.recipe.deployment
+              name = %s
+              user = zope
+              ''' % name)
+
+          buildout['main'] = dict(
+              recipe = 'zc.zodbrecipes:server',
+              deployment = 'deployment',
+              address = 8100,
+              path = options['path'],
+              **{
+                'zeo.conf': '''
+                  <zeo>
+                    address ${:address}
+                  </zeo>
+
+                  %import zc.zlibstorage
+
+                  <zlibstorage>
+                    <filestorage>
+                      path ${:path}
+                    </filestorage>
+                  </zlibstorage>
+                  '''}
+              )
+
+          buildout.parse('''
+              [pack]
+              recipe = zc.recipe.deployment:crontab
+              deployment = deployment
+              times = 1 2 * * 6
+              command =
+                ${buildout:bin-directory}/zeopack -d3 -t00 ${main:address}
+
+              [ctl]
+              recipe = zc.recipe.rhrc
+              deployment = deployment
+              chkconfig = 345 99 10
+              parts = main
+              ''')
+
+      def install(self):
+          pass
+
+      update = install
+
+.. -> source
+
+    >>> exec(source)
+
+The meta recipe just adds parts to the buildout. It does this by
+setting items and and calling the ``parse`` method.  The ``parse``
+method just takes a string in buildout configuration syntax.  It's
+useful when we want to add static, or nearly static part data.  The
+setting items syntax is useful when we have non-trivial computation
+for part data.
+
+The order that we add parts is important.  When adding a part, any
+string substitutions and other dependencies are evaluated, so the
+referenced parts must be defined first.  This is why, for example, the
+``pack`` part is added after the ``main`` part.
+
+Note that the meta recipe supplied an integer for one of the
+options. In addition to strings, it's legal to supply integer values.
+
+There are a few things to note about this example:
+
+- The install and update methods are empty.
+
+  While not required, this is a very common pattern for meta
+  recipes. Most meta recipes, simply invoke other recipes.
+
+- Setting a buildout item or calling parse, adds any sections with
+  recipes as parts.
+
+- An exception will be raised if a section already exists.
+
+Testing
+-------
+
+Now, let's test our meta recipe.  We'll test it without actually
+running buildout. Rather, we'll use a specialized buildout provided by
+the zc.buildout.testing module.
+
+    >>> import zc.buildout.testing
+    >>> buildout = zc.buildout.testing.Buildout()
+
+The testing buildout is intended to be passed to recipes being
+tested:
+
+    >>> _ = Recipe(buildout, 'ample', dict(path='/var/databases/ample/main.fs'))
+
+After running the recipe, we should see the buildout database
+populated by the recipe:
+
+    >>> buildout.print_options()
+    [ctl]
+    chkconfig = 345 99 10
+    deployment = deployment
+    parts = main
+    recipe = zc.recipe.rhrc
+    [deployment]
+    name = ample
+    recipe = zc.recipe.deployment
+    user = zope
+    [main]
+    address = 8100
+    deployment = deployment
+    path = /var/databases/ample/main.fs
+    recipe = zc.zodbrecipes:server
+    zeo.conf =
+    <BLANKLINE>
+                      <zeo>
+                        address 8100
+                      </zeo>
+    <BLANKLINE>
+                      %import zc.zlibstorage
+    <BLANKLINE>
+                      <zlibstorage>
+                        <filestorage>
+                          path /var/databases/ample/main.fs
+                        </filestorage>
+                      </zlibstorage>
+    <BLANKLINE>
+    [pack]
+    command = /sample-buildout/bin/zeopack -d3 -t00 8100
+    deployment = deployment
+    recipe = zc.recipe.deployment:crontab
+    times = 1 2 * * 6

src/zc/buildout/testing.py

@@ -171,6 +171,19 @@ def wait_until(label, func, *args, **kw):
         time.sleep(0.01)
     raise ValueError('Timed out waiting for: '+label)
 
+class TestOptions(zc.buildout.buildout.Options):
+
+    def initialize(self):
+        pass
+
+class Buildout(zc.buildout.buildout.Buildout):
+
+    def __init__(self):
+        zc.buildout.buildout.Buildout.__init__(
+            self, '', [('buildout', 'directory', os.getcwd())])
+
+    Options = TestOptions
+
 def buildoutSetUp(test):
 
     test.globs['__tear_downs'] = __tear_downs = []

src/zc/buildout/tests.py

@@ -1390,7 +1390,7 @@ def internal_errors():
     While:
       Installing.
       Getting section data-dir.
-      Initializing part data-dir.
+      Initializing section data-dir.
     <BLANKLINE>
     An internal error occured due to a bug in either zc.buildout or in a
     recipe being used:
@@ -3236,7 +3236,7 @@ def test_suite():
                      'zc.\1 = >=1.99'),
                     ])
                 ) + manuel.capture.Manuel(),
-            'buildout.txt',
+            'buildout.txt', 'meta-recipes.txt',
             setUp=buildout_txt_setup,
             tearDown=zc.buildout.testing.buildoutTearDown,
             ),