Merge pull request #458 from NextThought/issue457

Add 'allow-unknown-extras'

CHANGES.rst

@@ -4,20 +4,23 @@ Change History
 2.11.6 (unreleased)
 ===================
 
-- Nothing changed yet.
+- Add a new buildout option ``allow-unknown-extras`` to enable
+  installing requirements that specify extras that do not exist. This
+  needs a corresponding update to zc.recipe.egg. See `issue 457
+  <https://github.com/buildout/buildout/issues/457>`_.
 
 
 2.11.5 (2018-06-19)
 ===================
 
-- Fix for `issue 295 <https://github.com/buildout/buildout/issues/295>`. On
+- Fix for `issue 295 <https://github.com/buildout/buildout/issues/295>`_. On
   windows, deletion of temporary egg files is more robust now.
 
 
 2.11.4 (2018-05-14)
 ===================
 
-- Fix for `issue 451 <https://github.com/buildout/buildout/issues/451>`:
+- Fix for `issue 451 <https://github.com/buildout/buildout/issues/451>`_:
   distributions with a version number that normalizes to a shorter version
   number (3.3.0 to 3.3, for instance) can be installed now.
 

doc/reference.rst

@@ -231,6 +231,11 @@ allow-picked-versions, default: 'true'
   Indicate whether it should be possible to install requirements whose
   `versions aren't pinned <pinned-versions>`.
 
+allow-unknown-extras, default: 'false'
+  Specify whether requirements that specify an extra not provided by
+  the target distribution should be allowed. When this is false, such
+  a requirement is an error.
+
 bin-directory, default: bin
   The directory where generated scripts should be installed. If this
   is a relative path, it's evaluated relative to the buildout

doc/topics/variables-extending-and-substitutions.rst

@@ -107,6 +107,7 @@ with the later (right/bottom) configurations overriding earlier
     a = 11
     allow-hosts = *
     allow-picked-versions = true
+    allow-unknown-extras = false
     b = 21
     bin-directory = ...
     c = 31

src/zc/buildout/allow-unknown-extras.txt

+======================
+ Allow Unknown Extras
+======================
+
+Sometimes we need to allow unknown extras.
+
+The ``allow-unknown-extras`` option lets us do that in a buildout
+configuration, just as we can directly calling ``easy_install``
+works exactly like the one provided in ``easy_install``.
+
+Let's create a develop egg that requires a bogus extra.
+
+    >>> mkdir(sample_buildout, 'allowdemo')
+    >>> write(sample_buildout, 'allowdemo', 'dependencydemo.py',
+    ...       'import eggrecipekss.core')
+    >>> write(sample_buildout, 'allowdemo', 'setup.py',
+    ... '''from setuptools import setup; setup(
+    ...     name='allowdemo', py_modules=['dependencydemo'],
+    ...     zip_safe=True, version='1')
+    ... ''')
+
+Now let's configure the buildout to use the develop egg with a bogus extra.
+
+    >>> write(sample_buildout, 'buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... develop = allowdemo
+    ... parts = eggs
+    ...
+    ... [eggs]
+    ... recipe = zc.recipe.egg:eggs
+    ... eggs = allowdemo[bad_extra]
+    ... ''')
+
+Now we can run the buildout and see that it fails:
+
+    >>> print_(system(buildout), end='') # doctest: +ELLIPSIS
+    Develop: '/sample-buildout/allowdemo'
+    Installing eggs...
+    ...
+    While:
+      Installing eggs.
+    Error: Couldn't find the required extra...
+
+If we flip the option on, the buildout succeeds
+
+    >>> write(sample_buildout, 'buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... develop = allowdemo
+    ... parts = eggs
+    ... allow-unknown-extras = true
+    ...
+    ... [eggs]
+    ... recipe = zc.recipe.egg:eggs
+    ... eggs = allowdemo[bad_extra]
+    ... ''')
+
+
+Now we can run the buildout and only get a warning::
+
+    >>> print_(system(buildout), end='') # doctest: +ELLIPSIS
+    Develop: '/sample-buildout/allowdemo'
+    Installing eggs...
+    allowdemo 1 does not provide the extra 'bad_extra'

src/zc/buildout/buildout.py

@@ -285,6 +285,7 @@ _buildout_default_options = _annotate_section({
     'socket-timeout': '',
     'update-versions-file': '',
     'use-dependency-links': 'true',
+    'allow-unknown-extras': 'false',
     }, 'DEFAULT_VALUE')
 
 

src/zc/buildout/buildout.txt

@@ -804,6 +804,8 @@ the origin of the value (file name or ``COMPUTED_VALUE``, ``DEFAULT_VALUE``,
         DEFAULT_VALUE
     allow-picked-versions= true
         DEFAULT_VALUE
+    allow-unknown-extras= false
+        DEFAULT_VALUE
     bin-directory= bin
         DEFAULT_VALUE
     develop= recipes
@@ -882,6 +884,11 @@ You get more information about the way values are computed::
        AS DEFAULT_VALUE
        SET VALUE = true
     <BLANKLINE>
+    allow-unknown-extras= false
+    <BLANKLINE>
+       AS DEFAULT_VALUE
+       SET VALUE = false
+    <BLANKLINE>
     bin-directory= bin
     <BLANKLINE>
        AS DEFAULT_VALUE
@@ -2775,6 +2782,7 @@ database is shown::
     [buildout]
     allow-hosts = *
     allow-picked-versions = true
+    allow-unknown-extras = false
     bin-directory = /sample-buildout/bin
     develop-eggs-directory = /sample-buildout/develop-eggs
     directory = /sample-buildout

src/zc/buildout/easy_install.py

@@ -63,7 +63,7 @@ default_index_url = os.environ.get(
 logger = logging.getLogger('zc.buildout.easy_install')
 
 url_match = re.compile('[a-z0-9+.-]+://').match
-is_source_encoding_line = re.compile('coding[:=]\s*([-\w.]+)').search
+is_source_encoding_line = re.compile(r'coding[:=]\s*([-\w.]+)').search
 # Source encoding regex from http://www.python.org/dev/peps/pep-0263/
 
 is_win32 = sys.platform == 'win32'
@@ -222,7 +222,7 @@ def dist_needs_pkg_resources(dist):
     )
 
 
-class Installer:
+class Installer(object):
 
     _versions = {}
     _required_by = {}
@@ -233,6 +233,7 @@ class Installer:
     _use_dependency_links = True
     _allow_picked_versions = True
     _store_required_by = False
+    _allow_unknown_extras = False
 
     def __init__(self,
                  dest=None,
@@ -246,10 +247,12 @@ class Installer:
                  use_dependency_links=None,
                  allow_hosts=('*',),
                  check_picked=True,
+                 allow_unknown_extras=False,
                  ):
         assert executable == sys.executable, (executable, sys.executable)
         self._dest = dest if dest is None else pkg_resources.normalize_path(dest)
         self._allow_hosts = allow_hosts
+        self._allow_unknown_extras = allow_unknown_extras
 
         if self._install_from_cache:
             if not self._download_cache:
@@ -733,7 +736,34 @@ class Installer:
                     pkg_resources.VersionConflict(dist, req), ws)
 
             best[req.key] = dist
-            extra_requirements = dist.requires(req.extras)[::-1]
+
+            missing_requested = sorted(
+                set(req.extras) - set(dist.extras)
+            )
+            for missing in missing_requested:
+                logger.warning(
+                    '%s does not provide the extra \'%s\'',
+                    dist, missing
+                )
+
+            if missing_requested:
+                if not self._allow_unknown_extras:
+                    raise zc.buildout.UserError(
+                        "Couldn't find the required extra. "
+                        "This means the requirement is incorrect. "
+                        "If the requirement is itself from software you "
+                        "requested, then there might be a bug in "
+                        "requested software. You can ignore this by "
+                        "using 'allow-unknown-extras=true', however "
+                        "that may simply cause needed software to be omitted."
+                    )
+
+                extra_requirements = sorted(
+                    set(dist.extras) & set(req.extras)
+                )
+            else:
+                extra_requirements = dist.requires(req.extras)[::-1]
+
             for extra_requirement in extra_requirements:
                 self._requirements_and_constraints.append(
                     "Requirement of %s: %s" % (
@@ -912,6 +942,7 @@ def install(specs, dest,
             include_site_packages=None,
             allowed_eggs_from_site_packages=None,
             check_picked=True,
+            allow_unknown_extras=False,
             ):
     assert executable == sys.executable, (executable, sys.executable)
     assert include_site_packages is None
@@ -921,7 +952,8 @@ def install(specs, dest,
                           always_unzip, path,
                           newest, versions, use_dependency_links,
                           allow_hosts=allow_hosts,
-                          check_picked=check_picked)
+                          check_picked=check_picked,
+                          allow_unknown_extras=allow_unknown_extras)
     return installer.install(specs, working_set)
 
 buildout_and_setuptools_dists = list(install(['zc.buildout'], None,

src/zc/buildout/easy_install.txt

@@ -86,6 +86,10 @@ relative_paths
    allows scripts to work when scripts and eggs are moved, as long as
    they are both moved in the same way.
 
+allow_unknown_extras
+    Install the requirements, even if one of them specifies an
+    extra not provided by the distribution.
+
 The install method returns a working set containing the distributions
 needed to meet the given requirements.
 
@@ -210,6 +214,32 @@ dependencies.  We might do this to specify a specific version.
 
     >>> rmdir(dest)
 
+Unknown extras
+--------------
+
+Attempting to install a requirement with an extra it doesn't provide
+is an error.
+
+    >>> ws = zc.buildout.easy_install.install(
+    ...     ['demo[unknown_extra]'], dest, links=[link_server],
+    ...     index=link_server+'index/')
+    Traceback (most recent call last):
+    ...
+    UserError: Couldn't find the required extra...
+
+We can pass the ``allow_unknown_extras`` argument to force the
+installation to proceed.
+
+    >>> ws = zc.buildout.easy_install.install(
+    ...     ['demo[unknown_extra]'], dest, links=[link_server],
+    ...     index=link_server+'index/',
+    ...     allow_unknown_extras=True)
+
+    >>> ls(dest)
+    d  demo-0.3-py2.4.egg
+
+    >>> rmdir(dest)
+
 Case issues
 -----------
 

src/zc/buildout/tests.py

@@ -3712,7 +3712,7 @@ def test_suite():
 
         doctest.DocFileSuite(
             'easy_install.txt', 'downloadcache.txt', 'dependencylinks.txt',
-            'allowhosts.txt',
+            'allowhosts.txt', 'allow-unknown-extras.txt',
             setUp=easy_install_SetUp,
             tearDown=zc.buildout.testing.buildoutTearDown,
             optionflags=doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS,
@@ -3730,7 +3730,7 @@ def test_suite():
                 (re.compile(r'\\[\\]?'), '/'),
                 (re.compile('(\n?)-  ([a-zA-Z_.-]+)\n-  \\2.exe\n'),
                  '\\1-  \\2\n'),
-                ]+(sys.version_info < (2, 5) and [
+               ]+(sys.version_info < (2, 5) and [
                   (re.compile('.*No module named runpy.*', re.S), ''),
                   (re.compile('.*usage: pdb.py scriptfile .*', re.S), ''),
                   (re.compile('.*Error: what does not exist.*', re.S), ''),

zc.recipe.egg_/src/zc/recipe/egg/egg.py

@@ -23,6 +23,8 @@ import re
 import sys
 import zc.buildout.easy_install
 
+from zc.buildout.buildout import bool_option
+
 
 class Eggs(object):
 
@@ -82,6 +84,7 @@ class Eggs(object):
             links=self.links,
             index=self.index,
             allow_hosts=self.allow_hosts,
+            allow_unknown_extras=bool_option(buildout_section, 'allow-unknown-extras')
             )
 
         return orig_distributions, ws
@@ -127,6 +130,7 @@ class Eggs(object):
         links=(),
         index=None,
         allow_hosts=('*',),
+        allow_unknown_extras=False,
     ):
         """Helper function to build a working set.
 
@@ -145,6 +149,7 @@ class Eggs(object):
             tuple(links),
             index,
             tuple(allow_hosts),
+            allow_unknown_extras,
         )
         if cache_key not in cache_storage:
             if offline:
@@ -159,7 +164,8 @@ class Eggs(object):
                     index=index,
                     path=[develop_eggs_dir],
                     newest=newest,
-                    allow_hosts=allow_hosts)
+                    allow_hosts=allow_hosts,
+                    allow_unknown_extras=allow_unknown_extras)
             ws = self._sort_working_set(ws)
             cache_storage[cache_key] = ws