WIP theia resilient: add transfer.py and disable import autorun

software/theia/buildout.hash.cfg

@@ -23,7 +23,7 @@ md5sum = be9b217852da24d31ccc6d7c7a26a828
 
 [instance-theia-import.cfg.in]
 filename = instance-theia-import.cfg.in
-md5sum = 0ef11db9d5037add58b6f52030279690
+md5sum = 56c5abad2384e82f83ae8f357d6bc4c0
 
 [instance-theia-export.cfg.in]
 filename = instance-theia-export.cfg.in
@@ -33,6 +33,10 @@ md5sum = 84639ad3d48b864e5ed1a5456b6a4754
 filename = instance-theia-resilient.cfg.jinja2
 md5sum = 388ef606ccbc6d543685a53b18eb7369
 
+[transfer.py]
+filename = transfer.py
+md5sum = 6d61353ba5dea07e51d2723a4f89845a
+
 [yarn.lock]
 filename = yarn.lock
 md5sum = e9c0f6cc380b53425f521a167c2fdf66

software/theia/instance-theia-import.cfg.in

@@ -14,6 +14,12 @@ parts +=
 [slap-parameter]
 namebase = ${slap-configuration:configuration.namebase}
 
+# The resilient stack forwards the parameters to the 'export' instance
+# but not to the 'import' instance.
+# XXX We disable autorun to avoid 'AF_UNIX path too long' issues in the promise.
+[instance-parameter]
+configuration.autorun = disabled
+
 # The resilient stack calls importer:wrapper when the import instance
 # is notified that the backup files have just been pushed to it.
 [importer]

software/theia/software.cfg

@@ -359,3 +359,6 @@ output = ${buildout:directory}/instance-theia.cfg.jinja2
 
 [instance-theia-resilient.cfg.jinja2]
 <= download-base
+
+[transfer.py]
+<= download-base

software/theia/transfer.py

+import argparse
+import contextlib
+import os
+import re
+import subprocess
+import itertools
+import functools
+
+
+os.environ['LC_ALL'] = 'C'
+os.umask(0o77)
+
+regex = '^(file has vanished: |rsync warning: some files vanished before they could be transferred)'
+
+# XXX --links may not play well with --relative
+flags = (
+  '--recursive',            # recursively transfer subdirectories of source paths
+  '--relative',             # copy the full source paths in the destination directory
+  '--links',                # recreate symlinks in the destination
+  '--safe-links',           # ignore symlinks that point outside the tree
+  '--ignore-missing-args',  # ignore source paths that don't exist
+  '--delete',               # delete files in destination that are not in sources
+  '--delete-excluded',      # delete files in destination that aare excluded
+  '--perms',                # conserve file permissions
+  '--times',                # conserve times
+  '--group',                # conserve group
+  '--owner',                # conserve owner
+  '--verbose',              # be verbose
+)
+
+
+def rsync(executable, sources, dest, exclude=[]):
+  command = [executable]
+  command.extend(flags)
+  command.extend(("--exclude={}".format(x) for x in sorted(exclude)))
+  command.extend(sorted(sources))
+  command.append(dest)
+  try:
+    output = subprocess.check_output(command)
+    print(output)
+  except subprocess.CalledProcessError as e:
+    # Not all rsync errors are to be considered as errors
+    if e.returncode != 24 or re.search(regex, e.output, re.M) is None:
+      raise
+
+
+@contextlib.contextmanager
+def cwd(path):
+  old_path = os.getcwd()
+  try:
+    os.chdir(path)
+    yield
+  finally:
+    os.chdir(old_path)
+
+
+def transfer(rsync, src, dest, include, exclude=[]):
+  # Compute absolute path for dest directory
+  dest = os.path.abspath(dest)
+
+  # Call rsync from src directory in order to transfer files relatively to src
+  with cwd(src):
+    rsync(include, dest, exclude)
+
+
+def parse():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--src', required=True)
+  parser.add_argument('--dest', required=True)
+  parser.add_argument('--rsync', default='rsync')
+  parser.add_argument('--sqlite3', default='sqlite3')
+  parser.add_argument('--include', nargs='*', default=[])
+  parser.add_argument('--exclude', nargs='*', default=[])
+  parser.add_argument('--db', nargs='*', default=[])
+  return parser.parse_args()
+
+
+def main():
+  args = parse()
+  src = args.src
+
+  # Turn absolute paths into paths relative to src
+  include = (os.path.relpath(p, src) if os.path.isabs(p) else p for p in args.include)
+  exclude = (os.path.relpath(p, src) if os.path.isabs(p) else p for p in args.exclude)
+  db = (os.path.relpath(p, src) if os.path.isabs(p) else p for p in args.db)
+
+  # Transfer the included subtrees of src to dest, excepting excluded and db patterns
+  transfer(functools.partial(rsync, args.rsync), src, args.dest, include, itertools.chain(exclude, db))
+
+if __name__ == "__main__":
+  main()