[feat] zc.recipe.egg: Support on the fly patches.

- Support on the fly patches in zc.recipe.egg by ``EGGNAME-patches``,
  ``EGGNAME-patch-options``, ``EGGNAME-patch-binary`` (or
  ``patch-binary``) and ``EGGNAME-patch-revision`` options.

- Support on the fly patches in zc.recipe.egg:custom by ``patches``,
  ``patch-options``, ``patch-binary`` and ``patch-revision`` options.
  (options ``EGGNAME-*`` are also supported as well).

Specified patches are automatically applied on required eggs as well.

This fixes cache of patches.

Clean-up + fix issue found at slapos!1674

src/zc/buildout/easy_install.py

@@ -81,6 +81,9 @@ is_source_encoding_line = re.compile(r'coding[:=]\s*([-\w.]+)').search
 is_win32 = sys.platform == 'win32'
 is_jython = sys.platform.startswith('java')
 
+PATCH_MARKER = 'SlapOSPatched'
+orig_versions_re = re.compile(r'[+\-]%s\d+' % PATCH_MARKER)
+
 if is_jython:
     import java.lang.System
     jython_os_name = (java.lang.System.getProperties()['os.name']).lower()
@@ -464,6 +467,11 @@ class Installer(object):
             zc.buildout.rmtree.rmtree(tmp)
 
     def _obtain(self, requirement, source=None):
+        # get the non-patched version
+        req = str(requirement)
+        if PATCH_MARKER in req:
+            requirement = pkg_resources.Requirement.parse(re.sub(orig_versions_re, '', req))
+
         # initialize out index for this project:
         index = self._index
 
@@ -676,7 +684,7 @@ class Installer(object):
 
         return requirement
 
-    def install(self, specs, working_set=None):
+    def install(self, specs, working_set=None, patch_dict=None):
 
         logger.debug('Installing %s.', repr(specs)[1:-1])
         __doing__ = _doing_list, self._requirements_and_constraints
@@ -700,6 +708,9 @@ class Installer(object):
             ws = working_set
 
         for requirement in requirements:
+            if patch_dict and requirement.project_name in patch_dict:
+                self._env.scan(
+                    self.build(str(requirement), {}, patch_dict=patch_dict))
             for dist in self._get_dist(requirement, ws):
                 self._maybe_add_setuptools(ws, dist)
 
@@ -752,6 +763,9 @@ class Installer(object):
                 else:
                     logger.debug('Adding required %r', str(req))
                 self._log_requirement(ws, req)
+                if patch_dict and req.project_name in patch_dict:
+                    self._env.scan(
+                        self.build(str(req), {}, patch_dict=patch_dict))
                 for dist in self._get_dist(req, ws):
                     self._maybe_add_setuptools(ws, dist)
             if dist not in req:
@@ -798,7 +812,7 @@ class Installer(object):
             processed[req] = True
         return ws
 
-    def build(self, spec, build_ext):
+    def build(self, spec, build_ext, patch_dict=None):
 
         requirement = self._constrain(pkg_resources.Requirement.parse(spec))
 
@@ -849,12 +863,31 @@ class Installer(object):
                             )
                     base = os.path.dirname(setups[0])
 
+                setup_cfg_dict = {'build_ext':build_ext}
+                if patch_dict:
+                    setup_cfg_dict.update(
+                      {'egg_info':{'tag_build':'+%s%03d' % (PATCH_MARKER,
+                                                            patch_dict['patch_revision'])}})
+                    download = zc.buildout.download.Download(
+                        cache=tmp, hash_name=True)
+                    patch_dict = patch_dict[dist.project_name]
+                    for patch in patch_dict['patches']:
+                        url, md5sum = (patch.strip().split('#', 1) + [''])[:2]
+                        path, is_temp = download(url, md5sum=md5sum or None)
+                        args = [patch_dict['patch_binary']]
+                        args += patch_dict['patch_options']
+                        popen = subprocess.Popen(args,
+                            cwd=base, stdin=open(path))
+                        popen.communicate()
+                        if popen.returncode:
+                            raise subprocess.CalledProcessError(
+                                popen.returncode, ' '.join(args))
                 setup_cfg = os.path.join(base, 'setup.cfg')
                 if not os.path.exists(setup_cfg):
                     f = open(setup_cfg, 'w')
                     f.close()
                 setuptools.command.setopt.edit_config(
-                    setup_cfg, dict(build_ext=build_ext))
+                    setup_cfg, setup_cfg_dict)
 
                 dists = self._call_pip_wheel(base, self._dest, dist)
 
@@ -974,6 +1007,7 @@ def install(specs, dest,
             allowed_eggs_from_site_packages=None,
             check_picked=True,
             allow_unknown_extras=False,
+            patch_dict=None,
             ):
     assert executable == sys.executable, (executable, sys.executable)
     assert include_site_packages is None
@@ -985,18 +1019,19 @@ def install(specs, dest,
                           allow_hosts=allow_hosts,
                           check_picked=check_picked,
                           allow_unknown_extras=allow_unknown_extras)
-    return installer.install(specs, working_set)
+    return installer.install(specs, working_set, patch_dict=patch_dict)
 
 
 def build(spec, dest, build_ext,
           links=(), index=None,
           executable=sys.executable,
-          path=None, newest=True, versions=None, allow_hosts=('*',)):
+          path=None, newest=True, versions=None, allow_hosts=('*',),
+          patch_dict=None):
     assert executable == sys.executable, (executable, sys.executable)
     installer = Installer(dest, links, index, executable,
                           True, path, newest,
                           versions, allow_hosts=allow_hosts)
-    return installer.build(spec, build_ext)
+    return installer.build(spec, build_ext, patch_dict=patch_dict)
 
 
 def _rm(*paths):

zc.recipe.egg_/src/zc/recipe/egg/README.rst

@@ -9,6 +9,19 @@ eggs
     requirement strings.  Each string must be given on a separate
     line.
 
+patch-binary
+   The path to the patch executable.
+
+EGGNAME-patches
+   A new-line separated list of patchs to apply when building.
+
+EGGNAME-patch-options
+   Options to give to the patch program when applying patches.
+
+EGGNAME-patch-revision
+   An integer to specify the revision (default is the number of
+   patches).
+
 find-links
    A list of URLs, files, or directories to search for distributions.
 

zc.recipe.egg_/src/zc/recipe/egg/custom.py

@@ -15,9 +15,12 @@
 """
 import logging
 import os
+import re
 import sys
 
+import pkg_resources
 import zc.buildout.easy_install
+from .egg import _get_patch_dict
 
 logger = logging.getLogger(__name__)
 
@@ -70,6 +73,10 @@ class Base:
                 except KeyError:
                     pass
 
+    def _get_patch_dict(self, options, distribution):
+        req, = pkg_resources.parse_requirements(distribution)
+        return _get_patch_dict(options, [req.project_name])
+
 
 class Custom(Base):
 
@@ -127,10 +134,11 @@ class Custom(Base):
             extra_path = os.pathsep.join(ws.entries)
             self.environment['PYTHONEXTRAPATH'] = os.environ['PYTHONEXTRAPATH'] = extra_path
 
+        patch_dict = self._get_patch_dict(options, distribution)
         return zc.buildout.easy_install.build(
             distribution, options['_d'], self.build_ext,
             self.links, self.index, sys.executable,
-            [options['_e']], newest=self.newest,
+            [options['_e']], newest=self.newest, patch_dict=patch_dict,
             )
 
 

zc.recipe.egg_/src/zc/recipe/egg/custom.rst

@@ -24,6 +24,19 @@ setup-eggs
    A new-line separated list of eggs that need to be installed
    beforehand. It is useful to meet the `setup_requires` requirement.
 
+patch-binary
+   The path to the patch executable.
+
+patches
+   A new-line separated list of patchs to apply when building.
+
+patch-options
+   Options to give to the patch program when applying patches.
+
+patch-revision
+   An integer to specify the revision (default is the number of
+   patches).
+
 define
    A comma-separated list of names of C preprocessor variables to
    define.

zc.recipe.egg_/src/zc/recipe/egg/egg.py

@@ -23,6 +23,24 @@ import zc.buildout.easy_install
 
 from zc.buildout.buildout import bool_option
 
+def _get_patch_dict(options, eggs):
+    patch_dict = {}
+    def get_option(key, default):
+        if len(eggs) == 1 or key != 'patches':
+            default = options.get(key, default)
+        return options.get('%s-%s' % (egg, key), default)
+    for egg in eggs:
+        patches = list(filter(None, map(str.strip,
+            get_option('patches', '').splitlines())))
+        if patches:
+            patch_dict[egg] = {
+                'patches': patches,
+                'patch_options': get_option('patch-options', '-p0').split(),
+                'patch_binary': get_option('patch-binary', 'patch'),
+                'patch_revision': len(patches),
+            }
+    return patch_dict
+
 
 class Eggs(object):
 
@@ -57,6 +75,10 @@ class Eggs(object):
         options['develop-eggs-directory'] = b_options['develop-eggs-directory']
         options['_d'] = options['develop-eggs-directory']  # backward compat.
 
+    def _get_patch_dict(self, options):
+        egg_list = [x[:-8] for x in options if x.endswith('-patches')]
+        return _get_patch_dict(options, egg_list)
+
     def working_set(self, extra=()):
         """Separate method to just get the working set
 
@@ -134,6 +156,7 @@ class Eggs(object):
                     [develop_eggs_dir, eggs_dir]
                     )
             else:
+                patch_dict = self._get_patch_dict(self.options)
                 ws = zc.buildout.easy_install.install(
                     distributions, eggs_dir,
                     links=links,
@@ -141,7 +164,8 @@ class Eggs(object):
                     path=[develop_eggs_dir],
                     newest=newest,
                     allow_hosts=allow_hosts,
-                    allow_unknown_extras=allow_unknown_extras)
+                    allow_unknown_extras=allow_unknown_extras,
+                    patch_dict=patch_dict)
             ws = zc.buildout.easy_install.sort_working_set(
                 ws, buildout_dir, eggs_dir, develop_eggs_dir
                 )

zc.recipe.egg_/src/zc/recipe/egg/patches.rst

+
+Patching eggs before installation
+---------------------------------
+
+The SlapOS extensions of ``zc.recipe.egg`` supports applying patches before installing eggs.
+
+The syntax is to use a version with the magic string ``SlapOSPatched`` plus the number of
+patches to apply.
+
+
+Let's use a patch for demoneeded egg:
+
+    >>> write(sample_buildout, 'demoneeded.patch',
+    ... r"""diff -ru before/demoneeded-1.1/eggrecipedemoneeded.py after/demoneeded-1.1/eggrecipedemoneeded.py
+    ... --- before/demoneeded-1.1/eggrecipedemoneeded.py 2020-09-08 09:27:36.000000000 +0200
+    ... +++ after/demoneeded-1.1/eggrecipedemoneeded.py 2020-09-08 09:46:16.482243822 +0200
+    ... @@ -1,3 +1,3 @@
+    ... -y=1
+    ... +y="patched demoneeded"
+    ...  def f():
+    ...    pass
+    ... \ No newline at end of file
+    ... """)
+
+First, we install demoneeded directly:
+
+    >>> write(sample_buildout, 'buildout.cfg',
+    ... """
+    ... [buildout]
+    ... parts = demoneeded
+    ...
+    ... [demoneeded]
+    ... recipe = zc.recipe.egg:eggs
+    ... eggs = demoneeded
+    ... find-links = %(server)s
+    ... index = %(server)s/index
+    ... demoneeded-patches =
+    ...   ./demoneeded.patch#4b8ad56711dd0d898a2b7957e9604079
+    ... demoneeded-patch-options = -p2
+    ...
+    ... [versions]
+    ... demoneeded = 1.1+SlapOSPatched001
+    ... """ % dict(server=link_server))
+
+When running buildout, we have a warning that a different version is installed, but that's not fatal.
+
+    >>> print_(system(buildout), end='')
+    Installing demoneeded.
+    patching file eggrecipedemoneeded.py
+    Installing demoneeded 1.1
+    Caused installation of a distribution:
+    demoneeded 1.1+slapospatched001
+    with a different version.
+
+The installed egg has the slapospatched001 marker
+
+    >>> ls(sample_buildout, 'eggs')
+    d  demoneeded-1.1+slapospatched001-pyN.N.egg
+    ...pip...
+    ...setuptools...
+    ...wheel...
+    zc.buildout.egg
+
+The code of the egg has been patched:
+
+    >>> import glob
+    >>> import os.path
+    >>> cat(glob.glob(os.path.join(sample_buildout, 'eggs', 'demoneeded-1.1+slapospatched001*', 'eggrecipedemoneeded.py'))[0])
+    y="patched demoneeded"
+    def f():
+      pass
+
+Reset the state and also remove the installed egg
+
+    >>> remove('.installed.cfg')
+    >>> rmdir(glob.glob(os.path.join(sample_buildout, 'eggs', 'demoneeded-1.1+slapospatched001*'))[0])
+
+
+In the previous example we applied patches to an egg installed directly, but
+the same technique can be used to apply patches on eggs installed as dependencies.
+
+In this example we install demo and apply a patch to demoneeded, which is a dependency to demo. 
+
+    >>> write(sample_buildout, 'buildout.cfg',
+    ... """
+    ... [buildout]
+    ... parts = demo
+    ...
+    ... [demo]
+    ... recipe = zc.recipe.egg
+    ... eggs = demo
+    ... find-links = %(server)s
+    ... index = %(server)s/index
+    ... demoneeded-patches =
+    ...   ./demoneeded.patch#4b8ad56711dd0d898a2b7957e9604079
+    ... demoneeded-patch-options = -p2
+    ...
+    ... [versions]
+    ... demoneeded = 1.1+SlapOSPatched001
+    ... """ % dict(server=link_server))
+
+When running buildout, we also have that warning that a different version is installed.
+
+    >>> print_(system(buildout), end='')
+    Installing demo.
+    Getting distribution for 'demo'.
+    Got demo 0.3.
+    patching file eggrecipedemoneeded.py
+    Installing demoneeded 1.1
+    Caused installation of a distribution:
+    demoneeded 1.1+slapospatched001
+    with a different version.
+    Generated script '/sample-buildout/bin/demo'.
+
+The installed egg has the slapospatched001 marker
+
+    >>> ls(sample_buildout, 'eggs')
+    d  demo-0.3-pyN.N.egg
+    d  demoneeded-1.1+slapospatched001-pyN.N.egg
+    ...pip...
+    ...setuptools...
+    ...wheel...
+    zc.buildout.egg
+
+If we run the demo script we see the patch was applied:
+
+    >>> print_(system(join(sample_buildout, 'bin', 'demo')), end='')
+    3 patched demoneeded

zc.recipe.egg_/src/zc/recipe/egg/tests.py

@@ -106,6 +106,28 @@ def test_suite():
                 ''),
                ])
             ),
+        doctest.DocFileSuite(
+            'patches.rst',
+            setUp=setUp, tearDown=zc.buildout.testing.buildoutTearDown,
+            optionflags=doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS,
+            checker=renormalizing.RENormalizing([
+               zc.buildout.testing.normalize_path,
+               zc.buildout.testing.normalize_endings,
+               zc.buildout.testing.normalize_script,
+               zc.buildout.testing.normalize_egg_py,
+               zc.buildout.tests.normalize_bang,
+               zc.buildout.tests.normalize_S,
+               zc.buildout.testing.not_found,
+               zc.buildout.testing.python27_warning,
+               zc.buildout.testing.python27_warning_2,
+               zc.buildout.testing.easyinstall_deprecated,
+               (re.compile(r'[d-]  zc.buildout(-\S+)?[.]egg(-link)?'),
+                'zc.buildout.egg'),
+               (re.compile(r'[d-]  setuptools-[^-]+-'), 'setuptools-X-'),
+               (re.compile(r'eggs\\\\demo'), 'eggs/demo'),
+               (re.compile(r'[a-zA-Z]:\\\\foo\\\\bar'), '/foo/bar'),
+               ])
+            ),
         ]
     if not WINDOWS:
         suites.append(