merged the allow-hosts option

CHANGES.txt

@@ -7,6 +7,8 @@ Change History
 1.0.4 (unreleased)
 ==================
 
+- Added the `allow-hosts` option (tarek)
+
 - Quote the 'executable' argument when trying to detect the python 
   version using popen4. (sidnei)
 

src/zc/buildout/allowhosts.txt

+Allow hosts
+-----------
+
+On some environments the links visited by `zc.buildout` can be forbidden
+by paranoiac firewalls. These URL might be on the chain of links 
+visited by `zc.buildout` wheter they are defined in the `find-links` option,
+wheter they are defined by various eggs in their `url`, `download_url`, 
+`dependency_links` metadata.
+
+It is even harder to track that package_index works like a spider and 
+might visit links and go to other location.
+
+The `allow-hosts` option provides a way to prevent this, and 
+works exactly like the one provided in `easy_install`
+(see `easy_install allow-hosts option`_).
+
+You can provide a list of allowed host, together with wildcards::
+
+    [buildout]
+    ...
+    
+    allow-hosts =
+        *.python.org
+        example.com
+
+Let's create a develop egg in our buildout that specifies
+`dependency_links` which points to a server in the outside world::
+
+    >>> mkdir(sample_buildout, 'allowdemo')
+    >>> write(sample_buildout, 'allowdemo', 'dependencydemo.py',
+    ...       'import eggrecipekss.core')
+    >>> write(sample_buildout, 'allowdemo', 'setup.py',
+    ... '''from setuptools import setup; setup(
+    ...     name='allowdemo', py_modules=['dependencydemo'],
+    ...     install_requires = 'kss.core',
+    ...     dependency_links = ['http://dist.plone.org'],
+    ...     zip_safe=True, version='1')
+    ... ''')
+
+Now let's configure the buildout to use the develop egg, 
+together with some rules that disallow any website but PyPI and
+local files::
+
+    >>> write(sample_buildout, 'buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... develop = allowdemo
+    ... parts = eggs
+    ... allow-hosts =
+    ...     pypi.python.org
+    ...
+    ... [eggs]
+    ... recipe = zc.recipe.egg:eggs
+    ... eggs = allowdemo
+    ... ''')
+
+Now we can run the buildout and make sure all attempts to dist.plone.org fails::
+
+    >>> print system(buildout)
+    Develop: '/sample-buildout/allowdemo'
+    Installing eggs.
+    <BLANKLINE>
+    Link to http://dist.plone.org ***BLOCKED*** by --allow-hosts
+    <BLANKLINE>
+    Couldn't find index page for 'kss.core' (maybe misspelled?)
+    Getting distribution for 'kss.core'.
+    While:
+      Installing eggs.
+      Getting distribution for 'kss.core'.
+    Error: Couldn't find a distribution for 'kss.core'.
+    <BLANKLINE>
+
+That's what we wanted : this will prevent any attempt to access
+unwanted domains. For instance, some packages are listing in their
+links `svn://` links. These can lead to error in some cases, and
+can therefore be protected like this::
+
+XXX (showcase with a svn:// file)
+
+    >>> write(sample_buildout, 'buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... develop = allowdemo
+    ... parts = eggs
+    ... allow-hosts =
+    ...     ^(!svn://).*
+    ...
+    ... [eggs]
+    ... recipe = zc.recipe.egg:eggs
+    ... eggs = allowdemo
+    ... ''')
+
+Now we can run the buildout and make sure all attempts to dist.plone.org fails::
+
+    >>> print system(buildout)
+    Develop: '/sample-buildout/allowdemo'
+    Installing eggs.
+    <BLANKLINE>
+    Link to http://dist.plone.org ***BLOCKED*** by --allow-hosts
+    <BLANKLINE>
+    Couldn't find index page for 'kss.core' (maybe misspelled?)
+    Getting distribution for 'kss.core'.
+    While:
+      Installing eggs.
+      Getting distribution for 'kss.core'.
+    Error: Couldn't find a distribution for 'kss.core'.
+    <BLANKLINE>
+
+

src/zc/buildout/buildout.py

@@ -136,6 +136,10 @@ class Buildout(UserDict.DictMixin):
 
         links = options.get('find-links', '')
         self._links = links and links.split() or ()
+        
+        allow_hosts = options.get('allow-hosts', '*').split('\n')
+        self._allow_hosts = tuple([host.strip() for host in allow_hosts 
+                                   if host.strip() != ''])
 
         self._buildout_dir = options['directory']
         for name in ('bin', 'parts', 'eggs', 'develop-eggs'):
@@ -665,7 +669,8 @@ class Buildout(UserDict.DictMixin):
             self['buildout']['eggs-directory'],
             links = self['buildout'].get('find-links', '').split(),
             index = self['buildout'].get('index'),
-            path = [self['buildout']['develop-eggs-directory']]
+            path = [self['buildout']['develop-eggs-directory']],
+            allow_hosts = self._allow_hosts
             )
 
         upgraded = []
@@ -743,7 +748,7 @@ class Buildout(UserDict.DictMixin):
                 working_set=pkg_resources.working_set,
                 links = self['buildout'].get('find-links', '').split(),
                 index = self['buildout'].get('index'),
-                newest=self.newest)
+                newest=self.newest, allow_hosts=self._allow_hosts)
 
             # Clear cache because extensions might now let us read pages we
             # couldn't read before.
@@ -836,6 +841,7 @@ def _install_and_load(spec, group, entry, buildout):
                 path=path,
                 working_set=pkg_resources.working_set,
                 newest=buildout.newest,
+                allow_hosts=buildout._allow_hosts
                 )
 
         __doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry

src/zc/buildout/buildout.txt

@@ -2385,7 +2385,31 @@ We see that our extension is loaded and executed:
     ext ['buildout']
     Develop: '/sample-bootstrapped/demo'
 
+Allow hosts
+-----------
 
+On some environments the links visited by `zc.buildout` can be forbidden
+by paranoiac firewalls. These URL might be on the chain of links 
+visited by `zc.buildout` wheter they are defined in the `find-links` option,
+wheter they are defined by various eggs in their `url`, `download_url`, 
+`dependency_links` metadata.
+
+It is even harder to track that package_index works like a spider and 
+might visit links and go to other location.
+
+The `allow-hosts` option provides a way to prevent this, and 
+works exactly like the one provided in `easy_install`.
+
+You can provide a list of allowed host, together with wildcards::
+
+    [buildout]
+    ...
+    
+    allow-hosts =
+        *.python.org
+        example.com
+
+All urls that does not match these hosts will not be visited.
 
 .. [#future_recipe_methods] In the future, additional methods may be
        added. Older recipes with fewer methods will still be

src/zc/buildout/easy_install.py

@@ -71,8 +71,21 @@ def _get_version(executable):
         _versions[executable] = version
         return version
 
+FILE_SCHEME = re.compile('file://', re.I).match
+
+class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
+    """Will allow urls that are local to the system.
+
+    No matter what is allow_hosts.
+    """
+    def url_ok(self, url, fatal=False):
+        if FILE_SCHEME(url):
+            return True
+        return setuptools.package_index.PackageIndex.url_ok(self, url, False)
+        
+
 _indexes = {}
-def _get_index(executable, index_url, find_links):
+def _get_index(executable, index_url, find_links, allow_hosts=('*',)):
     key = executable, index_url, tuple(find_links)
     index = _indexes.get(key)
     if index is not None:
@@ -80,9 +93,8 @@ def _get_index(executable, index_url, find_links):
 
     if index_url is None:
         index_url = default_index_url
-
-    index = setuptools.package_index.PackageIndex(
-        index_url, python=_get_version(executable)
+    index = AllowHostsPackageIndex(
+        index_url, hosts=allow_hosts, python=_get_version(executable)
         )
         
     if find_links:
@@ -124,8 +136,10 @@ class Installer:
                  newest=True,
                  versions=None,
                  use_dependency_links=None,
+                 allow_hosts=('*',)
                  ):
         self._dest = dest
+        self._allow_hosts = allow_hosts
 
         if self._install_from_cache:
             if not self._download_cache:
@@ -152,7 +166,7 @@ class Installer:
         self._newest = newest
         self._env = pkg_resources.Environment(path,
                                               python=_get_version(executable))
-        self._index = _get_index(executable, index, links)
+        self._index = _get_index(executable, index, links, self._allow_hosts)
 
         if versions is not None:
             self._versions = versions
@@ -162,9 +176,9 @@ class Installer:
         if not dists:
             logger.debug('We have no distributions for %s that satisfies %r.',
                          req.project_name, str(req))
+            
             return None, self._obtain(req, source)
 
-
         # Note that dists are sorted from best to worst, as promised by
         # env.__getitem__
 
@@ -345,9 +359,9 @@ class Installer:
             shutil.rmtree(tmp)
             
     def _obtain(self, requirement, source=None):
-
         # initialize out index for this project:
         index = self._index
+        
         if index.obtain(requirement) is None:
             # Nothing is available.
             return None
@@ -425,7 +439,7 @@ class Installer:
         # Maybe an existing dist is already the best dist that satisfies the
         # requirement
         dist, avail = self._satisfied(requirement)
-
+        
         if dist is None:
             if self._dest is not None:
                 logger.info(*__doing__)
@@ -514,7 +528,8 @@ class Installer:
                         logger.debug('Adding find link %r from %s', link, dist)
                         self._links.append(link)
                         self._index = _get_index(self._executable,
-                                                 self._index_url, self._links)
+                                                 self._index_url, self._links, 
+                                                 self._allow_hosts)
 
         for dist in dists:
             # Check whether we picked a version and, if we did, report it:
@@ -729,18 +744,19 @@ def install(specs, dest,
             links=(), index=None,
             executable=sys.executable, always_unzip=False,
             path=None, working_set=None, newest=True, versions=None,
-            use_dependency_links=None):
+            use_dependency_links=None, allow_hosts=('*',)):
     installer = Installer(dest, links, index, executable, always_unzip, path,
-                          newest, versions, use_dependency_links)
+                          newest, versions, use_dependency_links, 
+                          allow_hosts=allow_hosts)
     return installer.install(specs, working_set)
 
 
 def build(spec, dest, build_ext,
           links=(), index=None,
           executable=sys.executable,
-          path=None, newest=True, versions=None):
+          path=None, newest=True, versions=None, allow_hosts=('*',)):
     installer = Installer(dest, links, index, executable, True, path, newest,
-                          versions)
+                          versions, allow_hosts=allow_hosts)
     return installer.build(spec, build_ext)
 
         

src/zc/buildout/tests.py

@@ -2657,6 +2657,7 @@ def test_suite():
         
         doctest.DocFileSuite(
             'easy_install.txt', 'downloadcache.txt', 'dependencylinks.txt',
+            'allowhosts.txt',
             setUp=easy_install_SetUp,
             tearDown=zc.buildout.testing.buildoutTearDown,
 

zc.recipe.egg_/src/zc/recipe/egg/egg.py

@@ -39,6 +39,11 @@ class Eggs(object):
             options['index'] = index
         self.index = index
 
+        allow_hosts = buildout['buildout'].get('allow-hosts', '*')
+        allow_hosts = tuple([host.strip() for host in allow_hosts.split('\n')
+                               if host.strip()!=''])
+        self.allow_hosts = allow_hosts 
+
         options['eggs-directory'] = buildout['buildout']['eggs-directory']
         options['_e'] = options['eggs-directory'] # backward compat.
         options['develop-eggs-directory'
@@ -78,6 +83,7 @@ class Eggs(object):
                 always_unzip=options.get('unzip') == 'true',
                 path=[options['develop-eggs-directory']],
                 newest=self.buildout['buildout'].get('newest') == 'true',
+                allow_hosts=self.allow_hosts
                 )
 
         return orig_distributions, ws