Add 'allow-unknown-extras'

Fixes #457

CHANGES.rst

@@ -4,20 +4,23 @@ Change History
 2.11.6 (unreleased)
 ===================
 
-- Nothing changed yet.
+- Add a new buildout option ``allow-unknown-extras`` to enable
+  installing requirements that specify extras that do not exist. This
+  needs a corresponding update to zc.recipe.egg. See `issue 457
+  <https://github.com/buildout/buildout/issues/457>`_.
 
 
 2.11.5 (2018-06-19)
 ===================
 
-- Fix for `issue 295 <https://github.com/buildout/buildout/issues/295>`. On
+- Fix for `issue 295 <https://github.com/buildout/buildout/issues/295>`_. On
   windows, deletion of temporary egg files is more robust now.
 
 
 2.11.4 (2018-05-14)
 ===================
 
-- Fix for `issue 451 <https://github.com/buildout/buildout/issues/451>`:
+- Fix for `issue 451 <https://github.com/buildout/buildout/issues/451>`_:
   distributions with a version number that normalizes to a shorter version
   number (3.3.0 to 3.3, for instance) can be installed now.
 

doc/reference.rst

@@ -231,6 +231,11 @@ allow-picked-versions, default: 'true'
   Indicate whether it should be possible to install requirements whose
   `versions aren't pinned <pinned-versions>`.
 
+allow-unknown-extras, default: 'false'
+  Specify whether requirements that specify an extra not provided by
+  the target distribution should be allowed. When this is false, such
+  a requirement is an error.
+
 bin-directory, default: bin
   The directory where generated scripts should be installed. If this
   is a relative path, it's evaluated relative to the buildout

doc/topics/variables-extending-and-substitutions.rst

@@ -107,6 +107,7 @@ with the later (right/bottom) configurations overriding earlier
     a = 11
     allow-hosts = *
     allow-picked-versions = true
+    allow-unknown-extras = false
     b = 21
     bin-directory = ...
     c = 31

src/zc/buildout/buildout.py

@@ -285,6 +285,7 @@ _buildout_default_options = _annotate_section({
     'socket-timeout': '',
     'update-versions-file': '',
     'use-dependency-links': 'true',
+    'allow-unknown-extras': 'false',
     }, 'DEFAULT_VALUE')
 
 

src/zc/buildout/buildout.txt

@@ -804,6 +804,8 @@ the origin of the value (file name or ``COMPUTED_VALUE``, ``DEFAULT_VALUE``,
         DEFAULT_VALUE
     allow-picked-versions= true
         DEFAULT_VALUE
+    allow-unknown-extras= false
+        DEFAULT_VALUE
     bin-directory= bin
         DEFAULT_VALUE
     develop= recipes
@@ -882,6 +884,11 @@ You get more information about the way values are computed::
        AS DEFAULT_VALUE
        SET VALUE = true
     <BLANKLINE>
+    allow-unknown-extras= false
+    <BLANKLINE>
+       AS DEFAULT_VALUE
+       SET VALUE = false
+    <BLANKLINE>
     bin-directory= bin
     <BLANKLINE>
        AS DEFAULT_VALUE
@@ -2775,6 +2782,7 @@ database is shown::
     [buildout]
     allow-hosts = *
     allow-picked-versions = true
+    allow-unknown-extras = false
     bin-directory = /sample-buildout/bin
     develop-eggs-directory = /sample-buildout/develop-eggs
     directory = /sample-buildout

src/zc/buildout/easy_install.py

@@ -63,7 +63,7 @@ default_index_url = os.environ.get(
 logger = logging.getLogger('zc.buildout.easy_install')
 
 url_match = re.compile('[a-z0-9+.-]+://').match
-is_source_encoding_line = re.compile('coding[:=]\s*([-\w.]+)').search
+is_source_encoding_line = re.compile(r'coding[:=]\s*([-\w.]+)').search
 # Source encoding regex from http://www.python.org/dev/peps/pep-0263/
 
 is_win32 = sys.platform == 'win32'
@@ -222,7 +222,7 @@ def dist_needs_pkg_resources(dist):
     )
 
 
-class Installer:
+class Installer(object):
 
     _versions = {}
     _required_by = {}
@@ -233,6 +233,7 @@ class Installer:
     _use_dependency_links = True
     _allow_picked_versions = True
     _store_required_by = False
+    _allow_unknown_extras = False
 
     def __init__(self,
                  dest=None,
@@ -246,10 +247,12 @@ class Installer:
                  use_dependency_links=None,
                  allow_hosts=('*',),
                  check_picked=True,
+                 allow_unknown_extras=False,
                  ):
         assert executable == sys.executable, (executable, sys.executable)
         self._dest = dest if dest is None else pkg_resources.normalize_path(dest)
         self._allow_hosts = allow_hosts
+        self._allow_unknown_extras = allow_unknown_extras
 
         if self._install_from_cache:
             if not self._download_cache:
@@ -733,7 +736,20 @@ class Installer:
                     pkg_resources.VersionConflict(dist, req), ws)
 
             best[req.key] = dist
-            extra_requirements = dist.requires(req.extras)[::-1]
+            if self._allow_unknown_extras:
+                missing_requested = sorted(
+                    set(req.extras) - set(dist.extras)
+                )
+                for missing in missing_requested:
+                    logger.warning(
+                        '%s does not provide the extra \'%s\'',
+                        dist, missing
+                    )
+                extra_requirements = sorted(
+                    set(dist.extras) & set(req.extras)
+                )
+            else:
+                extra_requirements = dist.requires(req.extras)[::-1]
             for extra_requirement in extra_requirements:
                 self._requirements_and_constraints.append(
                     "Requirement of %s: %s" % (
@@ -912,6 +928,7 @@ def install(specs, dest,
             include_site_packages=None,
             allowed_eggs_from_site_packages=None,
             check_picked=True,
+            allow_unknown_extras=False,
             ):
     assert executable == sys.executable, (executable, sys.executable)
     assert include_site_packages is None
@@ -921,7 +938,8 @@ def install(specs, dest,
                           always_unzip, path,
                           newest, versions, use_dependency_links,
                           allow_hosts=allow_hosts,
-                          check_picked=check_picked)
+                          check_picked=check_picked,
+                          allow_unknown_extras=allow_unknown_extras)
     return installer.install(specs, working_set)
 
 buildout_and_setuptools_dists = list(install(['zc.buildout'], None,

src/zc/buildout/easy_install.txt

@@ -86,6 +86,10 @@ relative_paths
    allows scripts to work when scripts and eggs are moved, as long as
    they are both moved in the same way.
 
+allow_unknown_extras
+    Install the requirements, even if one of them specifies an
+    extra not provided by the distribution.
+
 The install method returns a working set containing the distributions
 needed to meet the given requirements.
 
@@ -210,6 +214,32 @@ dependencies.  We might do this to specify a specific version.
 
     >>> rmdir(dest)
 
+Unknown extras
+--------------
+
+Attempting to install a requirement with an extra it doesn't provide
+is an error.
+
+    >>> ws = zc.buildout.easy_install.install(
+    ...     ['demo[unknown_extra]'], dest, links=[link_server],
+    ...     index=link_server+'index/')
+    Traceback (most recent call last):
+    ...
+    UnknownExtra: demo 0.3 has no such extra feature 'unknown_extra'
+
+We can pass the ``allow_unknown_extras`` argument to force the
+installation to proceed.
+
+    >>> ws = zc.buildout.easy_install.install(
+    ...     ['demo[unknown_extra]'], dest, links=[link_server],
+    ...     index=link_server+'index/',
+    ...     allow_unknown_extras=True)
+
+    >>> ls(dest)
+    d  demo-0.3-py2.4.egg
+
+    >>> rmdir(dest)
+
 Case issues
 -----------
 

src/zc/buildout/tests.py

@@ -3730,6 +3730,8 @@ def test_suite():
                 (re.compile(r'\\[\\]?'), '/'),
                 (re.compile('(\n?)-  ([a-zA-Z_.-]+)\n-  \\2.exe\n'),
                  '\\1-  \\2\n'),
+                (re.compile(r'^(\w+\.)*(UnknownExtra: )'),
+                 '\2'),
                 ]+(sys.version_info < (2, 5) and [
                   (re.compile('.*No module named runpy.*', re.S), ''),
                   (re.compile('.*usage: pdb.py scriptfile .*', re.S), ''),