Merge pull request #325 from buildout/abi-tag-eggs

Added a buildout option, ``abi-tag-eggs``

CHANGES.rst

 Change History
 **************
 
+- Added a buildout option, ``abi-tag-eggs`` that, when true, causes
+  the `ABI tag <https://www.python.org/dev/peps/pep-0425/#abi-tag>`_
+  for the buildout environment to be added to the eggs directory name.
+
+  This is useful when switching Python implementations (e.g. CPython
+  vs PyPI or debug builds vs regular builds), especially when
+  environment differences aren't reflected in egg names.  It also has
+  the side benefit of making eggs directories smaller, because eggs
+  for different Python versions are in different directories.
+
 2.6.0 (2017-01-29)
 ==================
 

LICENSE.txt

@@ -42,3 +42,31 @@ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+===============================================================================
+
+The module, zc.buildout.pep425tags, is copied from the wheel package,
+which uses the MIT license:
+
+"wheel" copyright (c) 2012-2014 Daniel Holth <dholth@fastmail.fm> and
+contributors.
+
+The MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

src/zc/buildout/buildout.py

@@ -384,7 +384,14 @@ class Buildout(DictMixin):
 
         download_cache = options.get('download-cache')
         extends_cache = options.get('extends-cache')
+
+        if bool_option(options, 'abi-tag-eggs', 'false'):
+            from zc.buildout.pep425tags import get_abi_tag
+            options['eggs-directory'] = os.path.join(
+                options['eggs-directory'], get_abi_tag())
+
         eggs_cache = options.get('eggs-directory')
+
         for cache in [download_cache, extends_cache, eggs_cache]:
             if cache:
                 cache = os.path.join(options['directory'], cache)
@@ -729,7 +736,7 @@ class Buildout(DictMixin):
         __doing__ = 'Setting up buildout directories'
 
         # Create buildout directories
-        for name in ('bin', 'parts', 'eggs', 'develop-eggs'):
+        for name in ('bin', 'parts', 'develop-eggs'):
             d = self['buildout'][name+'-directory']
             if not os.path.exists(d):
                 self._logger.info('Creating directory %r.', d)

src/zc/buildout/buildout.txt

@@ -2695,6 +2695,15 @@ All of these options can be overridden by configuration files or by
 command-line assignments.  We've discussed most of these options
 already, but let's review them and touch on some we haven't discussed:
 
+``abi-tag-eggs``
+    Add an `ABI tag
+    <https://www.python.org/dev/peps/pep-0425/#abi-tag>`_ to the
+    directory name given by the ``eggs-directory`` option.  This is
+    useful when switching between python implementations when details
+    of the implementation aren't reflected in egg names.  It also has
+    the side benefit of making eggs directories smaller, because eggs
+    for different Python versions are in different directories.
+
 ``allow-hosts``
     On some environments the links visited by ``zc.buildout`` can be forbidden
     by paranoid firewalls. These URLs might be in the chain of links visited

src/zc/buildout/pep425tags.py

+"""Generate and work with PEP 425 Compatibility Tags."""
+
+import sys
+import warnings
+
+try:
+    import sysconfig
+except ImportError:  # pragma nocover
+    # Python < 2.7
+    import distutils.sysconfig as sysconfig
+import distutils.util
+
+
+def get_config_var(var):
+    try:
+        return sysconfig.get_config_var(var)
+    except IOError as e:  # pip Issue #1074
+        warnings.warn("{0}".format(e), RuntimeWarning)
+        return None
+
+
+def get_abbr_impl():
+    """Return abbreviated implementation name."""
+    if hasattr(sys, 'pypy_version_info'):
+        pyimpl = 'pp'
+    elif sys.platform.startswith('java'):
+        pyimpl = 'jy'
+    elif sys.platform == 'cli':
+        pyimpl = 'ip'
+    else:
+        pyimpl = 'cp'
+    return pyimpl
+
+
+def get_impl_ver():
+    """Return implementation version."""
+    impl_ver = get_config_var("py_version_nodot")
+    if not impl_ver or get_abbr_impl() == 'pp':
+        impl_ver = ''.join(map(str, get_impl_version_info()))
+    return impl_ver
+
+
+def get_impl_version_info():
+    """Return sys.version_info-like tuple for use in decrementing the minor
+    version."""
+    if get_abbr_impl() == 'pp':
+        # as per https://github.com/pypa/pip/issues/2882
+        return (sys.version_info[0], sys.pypy_version_info.major,
+                sys.pypy_version_info.minor)
+    else:
+        return sys.version_info[0], sys.version_info[1]
+
+
+def get_flag(var, fallback, expected=True, warn=True):
+    """Use a fallback method for determining SOABI flags if the needed config
+    var is unset or unavailable."""
+    val = get_config_var(var)
+    if val is None:
+        if warn:
+            warnings.warn("Config variable '{0}' is unset, Python ABI tag may "
+                          "be incorrect".format(var), RuntimeWarning, 2)
+        return fallback()
+    return val == expected
+
+
+def get_abi_tag():
+    """Return the ABI tag based on SOABI (if available) or emulate SOABI
+    (CPython 2, PyPy)."""
+    soabi = get_config_var('SOABI')
+    impl = get_abbr_impl()
+    if not soabi and impl in ('cp', 'pp') and hasattr(sys, 'maxunicode'):
+        d = ''
+        m = ''
+        u = ''
+        if get_flag('Py_DEBUG',
+                    lambda: hasattr(sys, 'gettotalrefcount'),
+                    warn=(impl == 'cp')):
+            d = 'd'
+        if get_flag('WITH_PYMALLOC',
+                    lambda: impl == 'cp',
+                    warn=(impl == 'cp')):
+            m = 'm'
+        if get_flag('Py_UNICODE_SIZE',
+                    lambda: sys.maxunicode == 0x10ffff,
+                    expected=4,
+                    warn=(impl == 'cp' and
+                          sys.version_info < (3, 3))) \
+                and sys.version_info < (3, 3):
+            u = 'u'
+        abi = '%s%s%s%s%s' % (impl, get_impl_ver(), d, m, u)
+    elif soabi and soabi.startswith('cpython-'):
+        abi = 'cp' + soabi.split('-')[1]
+    elif soabi:
+        abi = soabi.replace('.', '_').replace('-', '_')
+    else:
+        abi = None
+    return abi
+
+
+def get_platform():
+    """Return our platform name 'win32', 'linux_x86_64'"""
+    # XXX remove distutils dependency
+    result = distutils.util.get_platform().replace('.', '_').replace('-', '_')
+    if result == "linux_x86_64" and sys.maxsize == 2147483647:
+        # pip pull request #3497
+        result = "linux_i686"
+    return result
+
+
+def get_supported(versions=None, supplied_platform=None):
+    """Return a list of supported tags for each version specified in
+    `versions`.
+
+    :param versions: a list of string versions, of the form ["33", "32"], 
+        or None. The first version will be assumed to support our ABI.
+    """
+    supported = []
+    
+    # Versions must be given with respect to the preference
+    if versions is None:
+        versions = []
+        version_info = get_impl_version_info()
+        major = version_info[:-1]
+        # Support all previous minor Python versions.
+        for minor in range(version_info[-1], -1, -1):
+            versions.append(''.join(map(str, major + (minor,))))
+            
+    impl = get_abbr_impl()
+    
+    abis = []
+
+    abi = get_abi_tag()
+    if abi:
+        abis[0:0] = [abi]
+ 
+    abi3s = set()
+    import imp
+    for suffix in imp.get_suffixes():
+        if suffix[0].startswith('.abi'):
+            abi3s.add(suffix[0].split('.', 2)[1])
+
+    abis.extend(sorted(list(abi3s)))
+
+    abis.append('none')
+
+    platforms = []
+    if supplied_platform:
+        platforms.append(supplied_platform)
+    platforms.append(get_platform())
+    
+    # Current version, current API (built specifically for our Python):
+    for abi in abis:
+        for arch in platforms:
+            supported.append(('%s%s' % (impl, versions[0]), abi, arch))
+            
+    # No abi / arch, but requires our implementation:
+    for i, version in enumerate(versions):
+        supported.append(('%s%s' % (impl, version), 'none', 'any'))
+        if i == 0:
+            # Tagged specifically as being cross-version compatible 
+            # (with just the major version specified)
+            supported.append(('%s%s' % (impl, versions[0][0]), 'none', 'any')) 
+    
+    # Major Python version + platform; e.g. binaries not using the Python API
+    supported.append(('py%s' % (versions[0][0]), 'none', arch))
+
+    # No abi / arch, generic Python
+    for i, version in enumerate(versions):
+        supported.append(('py%s' % (version,), 'none', 'any'))
+        if i == 0:
+            supported.append(('py%s' % (version[0]), 'none', 'any'))
+        
+    return supported

src/zc/buildout/tests.py

@@ -3033,6 +3033,35 @@ def parse_with_section_expr():
 
     """
 
+def test_abi_tag_eggs():
+    r"""
+    >>> mkdir('..', 'bo')
+    >>> cd('..', 'bo')
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... parts = egg
+    ... abi-tag-eggs = true
+    ... [egg]
+    ... recipe = zc.recipe.egg
+    ... eggs = demo
+    ... ''')
+    >>> _ = system(join('..', 'sample-buildout', 'bin', 'buildout')
+    ...            + ' bootstrap')
+    >>> _ = system(join('bin', 'buildout'))
+    >>> ls('.')
+    d  bin
+    -  buildout.cfg
+    d  develop-eggs
+    d  eggs
+    d  parts
+    >>> from zc.buildout.pep425tags import get_abi_tag
+    >>> ls(join('eggs', get_abi_tag())) # doctest: +ELLIPSIS
+    d...
+    d  setuptools-34.0.3-py3.5.egg
+    ...
+    """
+
 if sys.platform == 'win32':
     del buildout_honors_umask # umask on dohs is academic