Introduce an InstallMonitor

To monitor which files are created by buildout when installing parts

This version is based on git because it was easy to implement, but with
git tools, it's easy to inspect what exactly installing a part had
changed and also to revert to a previous state by using git commands
with the same `--git-dir` and `--work-dir`.

src/zc/buildout/buildout.py

@@ -51,6 +51,8 @@ import pprint
 import zc.buildout
 import zc.buildout.download
 
+from zc.buildout.install_monitor import InstallMonitor
+
 PY3 = sys.version_info[0] == 3
 if PY3:
     text_type = str
@@ -257,6 +259,8 @@ class Buildout(DictMixin):
         data = dict(buildout=_buildout_default_options.copy())
         self._buildout_dir = os.getcwd()
 
+        self._install_monitor = InstallMonitor(self)
+
         if config_file and not _isurl(config_file):
             config_file = os.path.abspath(config_file)
             base = os.path.dirname(config_file)
@@ -634,6 +638,7 @@ class Buildout(DictMixin):
             self.install(())
 
     def install(self, install_args):
+        self._install_monitor.begin_install(install_args)
         try:
             self._install_parts(install_args)
         finally:
@@ -644,6 +649,7 @@ class Buildout(DictMixin):
                     del self.installed_part_options
         if self.show_picked_versions or self.update_versions_file:
             self._print_picked_versions()
+        self._install_monitor.end_install()
         self._unload_extensions()
 
     def _install_parts(self, install_args):
@@ -754,7 +760,10 @@ class Buildout(DictMixin):
             elif not uninstall_missing:
                 continue
 
+            self._install_monitor.begin_uninstall_part(part, installed_part_options)
             self._uninstall_part(part, installed_part_options)
+            self._install_monitor.end_uninstall_part(part, installed_part_options)
+
             installed_parts = [p for p in installed_parts if p != part]
             installed_part_options['buildout']['parts'] = (
                 ' '.join(installed_parts))
@@ -785,7 +794,9 @@ class Buildout(DictMixin):
                         part)
 
                 try:
+                    self._install_monitor.begin_install_part(part, update=True)
                     updated_files = self[part]._call(update)
+                    self._install_monitor.end_install_part(part, updated_files, update=True)
                 except Exception:
                     installed_parts.remove(part)
                     self._uninstall(installed_files)
@@ -804,7 +815,9 @@ class Buildout(DictMixin):
                 self._logger.info(*__doing__)
                 if self.dry_run:
                     continue
+                self._install_monitor.begin_install_part(part)
                 installed_files = self[part]._call(recipe.install)
+                self._install_monitor.end_install_part(part, installed_files)
                 if installed_files is None:
                     self._logger.warning(
                         "The %s install returned None.  A path or "

src/zc/buildout/install_monitor.py

+import os
+import logging
+import subprocess
+import time
+import textwrap
+import pprint
+
+
+"""
+Monitor what files are installed by buildout
+
+# Notes
+
+## Packaging
+
+Instead of direct patches to buildout, this could be a buildout extension
+
+ * parts installation: ? monkey patchs ?
+ * `end_install` -> `zc.buildout.unloadextension`
+
+## random notes about monitor
+
+I had in mind we could have a ptrace monitor that knows were part is
+supposed to install and abort the process before it writes in a place
+where it is not supposed to write.
+
+Hard part is to defined what's an "allowed directory".
+
+For "custom" recipes, a section can install in:
+
+   * `${buildout:parts-directory}/${:_buildout_section_name}`
+   * `${buildout:bin-directory}` ( but is this really needed ? if it is, it must
+      be with restrictions, to prevent a part from overriding scripts from another part)
+   * the corresponding cache subdirectory for recipes using cache
+
+A `LD_PRELOAD` or [PRoot](https://github.com/proot-me/PRoot) based solution seems also
+possible, but the dynamic aspects of the cache seems to make this hard.
+
+
+For eggs, installing egg `X` should only be able to write to `eggs/X-{version_spec}`.
+
+Setuptools have some [sandboxing](https://github.com/pypa/setuptools/blob/c2262d9fe4eaac507ff128ae60b6682e8d132e4d/setuptools/sandbox.py) support (that I have not studied at this point), but it only monitor the
+python part of the installation process.
+
+Develop eggs seems to be similar - only write to `develop-eggs/X-{version_spec}` but this also
+probably need to include some hash in version spec.
+
+( because `eggs` and `develop-eggs` can be shared too )
+
+Related bugs that could be solved by such a "don't write here" approach:
+  https://nexedi.erp5.net/bug_module/20150413-15EC498
+  https://nexedi.erp5.net/bug_module/20110718-8E43A9
+
+"""
+
+class GitInstallMonitor(object):
+    """Stupid monitor making a (big) git commit to hold everything installed by each part.
+    
+    Assumes git command is in path
+    """
+    def __init__(self, buildout):
+        self._buildout = buildout
+        self._logger = logging.getLogger('zc.buildout.installmonitor')
+        self._start_install_time = {}
+
+    def _record_changes(self, message):
+        self._logger.debug("Adding %s to git, this may take some time", self._work_tree)
+        if self._git("status", "--porcelain"):
+            self._git("add", self._work_tree)
+            self._git("commit", "-m", message)
+            self._logger.debug("Added %s to git", self._work_tree)
+        else:
+            self._logger.info("No changes to record")
+
+    def _git(self, *args):
+        return subprocess.check_output(
+            (
+                'git',
+                '--git-dir', self._git_dir,
+                '--work-tree', self._work_tree,
+            ) + args,
+            env={
+                'GIT_COMMITTER_NAME': 'SlapOS Install Monitor',
+                'GIT_COMMITTER_EMAIL': 'noemail@example.com',
+                'GIT_AUTHOR_NAME': 'SlapOS Install Monitor',
+                'GIT_AUTHOR_EMAIL': 'noemail@example.com',
+            })
+
+    def begin_install(self, install_args):
+        """Called at the beginning of buildout installation.
+        """
+        self._start_install_time[self] = time.time()
+        self._work_tree = os.path.dirname(
+            os.path.commonprefix((
+                self._buildout['buildout']['parts-directory'],
+                self._buildout['buildout']['directory'],
+                self._buildout['buildout']['slapos-recipe-cmmi-shared-path'],
+            )))
+        self._git_dir = os.path.join(
+            self._buildout['buildout']['directory'],
+            'slapos.gitinstallmonitor.git'
+        )
+
+        self._git("init")
+        self._logger.info(
+            "Initialised git dir in %s with work tree %s",
+            self._git_dir,
+            self._work_tree
+        )
+
+        # ignore git dir ( as a path relative to work tree )
+        with open(os.path.join(self._git_dir, 'info', 'exclude'), 'w') as gitignore:
+            gitignore.write(self._git_dir[len(self._work_tree):])
+
+        self._record_changes("Begin install with args {install_args}".format(**locals()))
+
+    def end_install(self):
+        took = time.time() - self._start_install_time[self]
+        self._record_changes("Finished installation in {took:.2f} seconds".format(**locals()))
+
+    def begin_install_part(self, name, update=False):
+        self._start_install_time[name] = time.time()
+
+    def end_install_part(self, name, installed_files, update=False):
+        took = time.time() - self._start_install_time[name]
+        recipe = self._buildout[name]['recipe']
+        installed_files = '\n  '.join(installed_files or [])
+        options = '\n  '.join(pprint.pformat(dict(self._buildout[name])).splitlines())
+        did = "Updated" if update else "Installed"
+
+        self._record_changes(
+            textwrap.dedent("""\
+            {did} part {name} in {took:.2f} seconds
+
+            Recipe: {recipe}
+            Options:
+              {options}
+            Installed files:
+              {installed_files}
+            """).format(**locals()))
+
+    def begin_uninstall_part(self, name, installed_part_options):
+        self._start_install_time[name] = time.time()
+
+    def end_uninstall_part(self, name, installed_part_options):
+        took = time.time() - self._start_install_time[name]
+        self._record_changes(
+            textwrap.dedent("""\
+            Uninstalled part {name} in {took:.2f} seconds
+
+            Installed part options:
+              {installed_part_options}
+            """).format(**locals()))
+
+
+InstallMonitor = GitInstallMonitor