Added the ability to load configuration from URLs.

CHANGES.txt

@@ -20,6 +20,14 @@ priorities include:
 Change History
 **************
 
+1.0.0b17 (2006-12-07)
+=====================
+
+Feature Changes
+---------------
+
+- Configuration files can now be loaded from URLs.
+
 1.0.0b16 (2006-12-07)
 =====================
 

setup.py

@@ -7,7 +7,7 @@ def read(*rnames):
 name = "zc.buildout"
 setup(
     name = name,
-    version = "1.0.0b16",
+    version = "1.0.0b17",
     author = "Jim Fulton",
     author_email = "jim@zope.com",
     description = "System for managing development buildouts",

src/zc/buildout/buildout.py

@@ -25,6 +25,7 @@ import shutil
 import cStringIO
 import sys
 import tempfile
+import urllib2
 import ConfigParser
 import UserDict
 
@@ -41,6 +42,7 @@ except AttributeError:
 pkg_resources_loc = pkg_resources.working_set.find(
     pkg_resources.Requirement.parse('setuptools')).location
 
+_isurl = re.compile('([a-zA-Z0-9+.-]+)://').match
 
 class MissingOption(zc.buildout.UserError, KeyError):
     """A required option was missing
@@ -58,16 +60,11 @@ class Buildout(UserDict.DictMixin):
 
     def __init__(self, config_file, cloptions,
                  user_defaults=True, windows_restart=False):
-        config_file = os.path.abspath(config_file)
-        self._config_file = config_file
+
         self.__windows_restart = windows_restart
-        if not os.path.exists(config_file):
-            print 'Warning: creating', config_file
-            open(config_file, 'w').write('[buildout]\nparts = \n')
 
         # default options
         data = dict(buildout={
-            'directory': os.path.dirname(config_file),
             'eggs-directory': 'eggs',
             'develop-eggs-directory': 'develop-eggs',
             'bin-directory': 'bin',
@@ -79,6 +76,16 @@ class Buildout(UserDict.DictMixin):
             'log-format': '%(name)s: %(message)s',
             })
 
+        if not _isurl(config_file):
+            config_file = os.path.abspath(config_file)
+            base = os.path.dirname(config_file)
+            if not os.path.exists(config_file):
+                print 'Warning: creating', config_file
+                open(config_file, 'w').write('[buildout]\nparts = \n')
+            data['buildout']['directory'] = os.path.dirname(config_file)
+        else:
+            base = None
+
         # load user defaults, which override defaults
         if user_defaults and 'HOME' in os.environ:
             user_config = os.path.join(os.environ['HOME'],
@@ -98,6 +105,7 @@ class Buildout(UserDict.DictMixin):
             options[option] = value
                 # The egg dire
 
+
         self._raw = data
         self._data = {}
         self._parts = []
@@ -822,8 +830,6 @@ def _save_options(section, options, f):
         if value.endswith('\n\t'):
             value = value[:-2] + '%(__buildout_space_n__)s'
         print >>f, option, '=', value
-            
-    
 
 def _open(base, filename, seen):
     """Open a configuration file and return the result as a dictionary,
@@ -831,18 +837,32 @@ def _open(base, filename, seen):
     Recursively open other files based on buildout options found.
     """
 
-    filename = os.path.join(base, filename)
+    if _isurl(filename):
+        fp = urllib2.urlopen(filename)
+        base = filename[:filename.rfind('/')]
+    elif _isurl(base):
+        if os.path.isabs(filename):
+            fp = open(filename)
+            base = os.path.dirname(filename)
+        else:
+            filename = base + '/' + filename
+            fp = urllib2.urlopen(filename)
+            base = filename[:filename.rfind('/')]
+    else:
+        filename = os.path.join(base, filename)
+        fp = open(filename)
+        base = os.path.dirname(filename)
+
     if filename in seen:
         raise zc.buildout.UserError("Recursive file include", seen, filename)
 
-    base = os.path.dirname(filename)
     seen.append(filename)
 
     result = {}
 
     parser = ConfigParser.RawConfigParser()
     parser.optionxform = lambda s: s
-    parser.readfp(open(filename))
+    parser.readfp(fp)
     extends = extended_by = None
     for section in parser.sections():
         options = dict(parser.items(section))

src/zc/buildout/buildout.txt

@@ -741,6 +741,98 @@ There are several things to note about this example:
 - Relative file names in extended options are interpreted relative to
   the directory containing the referencing configuration file.
 
+Loading Configuration from URLs
+-------------------------------
+
+Configuration files can be loaded from URLs.  To see how this works,
+we'll set up a web server with some configuration files.
+
+    >>> server_data = tmpdir('server_data')
+
+    >>> write(server_data, "r1.cfg",
+    ... """
+    ... [debug]
+    ... op1 = r1 1
+    ... op2 = r1 2
+    ... """)
+
+    >>> write(server_data, "r2.cfg",
+    ... """
+    ... [buildout]
+    ... extends = r1.cfg
+    ... 
+    ... [debug]
+    ... op2 = r2 2
+    ... op3 = r2 3
+    ... """)
+
+    >>> server_url = start_server(server_data)
+
+    >>> write('client.cfg', 
+    ... """
+    ... [buildout]
+    ... develop = recipes
+    ... parts = debug
+    ... extends = %(url)s/r2.cfg
+    ...
+    ... [debug]
+    ... recipe = recipes:debug
+    ... name = base
+    ... """ % dict(url=server_url))
+
+
+    >>> print system(buildout+ ' -c client.cfg'),
+    buildout: Develop: /sample-buildout/recipes
+    buildout: Uninstalling debug
+    buildout: Installing debug
+    name base
+    op1 r1 1
+    op2 r2 2
+    op3 r2 3
+    recipe recipes:debug
+
+Here we specified a URL for the file we extended.  The file we
+downloaded, itself refered to a file on the server using a relative
+URL reference.  Relative references are interpreted relative to the
+base URL when they appear in configuration files loaded via URL.
+
+We can also specify a URL as the configuration file to be used by a
+buildout.  
+
+    >>> os.remove('client.cfg')
+    >>> write(server_data, 'remote.cfg', 
+    ... """
+    ... [buildout]
+    ... develop = recipes
+    ... parts = debug
+    ... extends = r2.cfg
+    ...
+    ... [debug]
+    ... recipe = recipes:debug
+    ... name = remote
+    ... """)
+
+    >>> print system(buildout + ' -c ' + server_url + '/remote.cfg'),
+    Error: Missing option: buildout:directory
+
+Normally, the buildout directory defaults to directory
+containing a configuration file.  This won't work for configuration
+files loaded from URLs.  In this case, the buildout directory would
+normally be defined on the command line:
+
+    >>> print system(buildout
+    ...              + ' -c ' + server_url + '/remote.cfg'
+    ...              + ' buildout:directory=' + sample_buildout
+    ...              ),
+    buildout: Develop: /sample-buildout/recipes
+    buildout: Uninstalling debug
+    buildout: Installing debug
+    name remote
+    op1 r1 1
+    op2 r2 2
+    op3 r2 3
+    recipe recipes:debug
+
 User defaults
 -------------