slaprunner: use the new python version of runner-exporter

software/slaprunner/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = 713db528880282d568278f09458d2aab
+md5sum = bb2628810e873b4fd4b50937ef23645d
 
 [template-runner]
 filename = instance-runner.cfg
@@ -28,13 +28,9 @@ md5sum = ed2e08c07a6727b2012f15da67c0705d
 filename = instance-runner-import.cfg.in
 md5sum = f9075dab9469c69506b24e642682938c
 
-[template-runner-export-script]
-filename = template/runner-export.sh.jinja2
-md5sum = 231f9b74862f8991f54326511e76f5ec
-
 [instance-runner-export]
 filename = instance-runner-export.cfg.in
-md5sum = 2abcebc7e4610566502eca45e28c2992
+md5sum = 01395b98534066ec23a5ad5c96d5f1e7
 
 [template-resilient]
 filename = instance-resilient.cfg.jinja2

software/slaprunner/instance-runner-export.cfg.in

@@ -75,21 +75,19 @@ ip = ${slaprunner:ipv4}
 [supervisord]
 port = ${supervisord-free-port:port}
 
+[exporter-raw]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:bin}/${slap-parameter:namebase}-exporter-raw
+command-line = {{ software_release_bin }}/${slap-parameter:namebase}-exporter --srv-path ${directory:srv} --etc-path ${directory:etc} --backup-path ${directory:backup} --backup-wait-time {{ backup_wait_time }}
+environment =
+  PATH={{ rsync_bin_folder }}:/bin:/usr/bin
+
 [exporter]
-recipe = slapos.recipe.template:jinja2
-template = {{ exporter_script_path }}
-rendered = ${directory:bin}/${slap-parameter:namebase}-exporter
-# backward compatibility for resilient stack
-wrapper = ${:rendered}
-mode = 700
-context =
-  import sys sys
-  import easy_install zc.buildout.easy_install
-  section directory directory
-  raw  backup_wait_time {{ backup_wait_time }}
-  raw  output_log_file ${directory:log}/resilient.log
-  raw  shell_binary {{ bash_executable_location }}
-  raw  rsync_binary {{ rsync_executable_location }}
+recipe = slapos.cookbook:wrapper
+command-line = {{ bash_executable_location }} -c "exec ${exporter-raw:wrapper-path} > >(tee -ai "${directory:log}/resilient.log") 2>&1"
+wrapper-path = ${directory:bin}/${slap-parameter:namebase}-exporter
+# wrapper parameter is needed by resilience stack
+wrapper = ${:wrapper-path}
 
 [monitor-httpd-free-port]
 recipe = slapos.cookbook:free_port

software/slaprunner/instance.cfg

@@ -56,12 +56,12 @@ context =
   key pbsready_export_template_path template-pbsready-export:rendered
   key template_runner_path instance-base-runner:rendered
   key slapparameter_dict slap-configuration:configuration
+  raw software_release_bin ${buildout:bin-directory}
   raw backup_wait_time ${exporter-default-configuration:backup_wait_time}
-  raw exporter_script_path ${template-runner-export-script:location}/${template-runner-export-script:filename}
   raw monitor_check_resilient_feed_template_path ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename}
   raw buildout_executable_location ${buildout:executable}
   raw bash_executable_location ${bash:location}/bin/bash
-  raw rsync_executable_location ${rsync:location}/bin/rsync
+  raw rsync_bin_folder ${rsync:location}/bin
 
 [template-runner-import]
 recipe = slapos.recipe.template:jinja2

software/slaprunner/software.cfg

@@ -76,10 +76,6 @@ output = ${buildout:directory}/template-runner.cfg.in
 < = template-download-base
 filename = runner-import.sh.jinja2
 
-[template-runner-export-script]
-< = template-download-base
-filename = runner-export.sh.jinja2
-
 [instance-runner-import]
 < = download-base
 recipe = slapos.recipe.build:download

software/slaprunner/template/runner-export.sh.jinja2

-#!{{ shell_binary }}
-LC_ALL=C
-export LC_ALL
-umask 077
-
-# Exit on any error, to prevent inconsistent backup
-# Error on unset variable expansion
-set -eu
-set -o pipefail
-
-# Redirect output to log
-exec > >(tee -ai '{{ output_log_file }}')
-exec 2>&1
-
-echo -e "\n\n$0 run at : $(date)"
-
-srv_directory='{{ directory["srv"] }}'
-backup_directory='{{ directory["backup"] }}'
-etc_directory='{{ directory["etc"] }}'
-tmp_directory='{{ directory["tmp"] }}'
-
-rsync () {
-  # Workaround for bug https://bugzilla.samba.org/show_bug.cgi?id=3653
-  IGNOREEXIT=24
-  IGNOREOUT='^(file has vanished: |rsync warning: some files vanished before they could be transferred)'
-
-  set -x
-  '{{ rsync_binary }}' -rlptgov --stats --safe-links --ignore-missing-args --delete --delete-excluded "$@" 2>&1 | (egrep -v "$IGNOREOUT" || true) || [ $? = "$IGNOREEXIT" ]
-  set +x
-}
-
-(
-  # XXX: code duplication with runner-import.sh.jinja2
-  path=$srv_directory/runner
-  backup_path=$backup_directory/runner/
-  cd "$path"
-
-  if [ -d instance ]; then
-    # Concatenate the exclude file of each partition of webrunner
-    # to create a global exclude file.
-    # Also, ignore all buildout-managed files.
-    exclude=$({{ sys.executable }} - "$path" <<EOF
-if 1:
-        import glob, errno, os, sys
-        sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }}
-        from zc.buildout.configparser import parse
-        path = sys.argv[1]
-
-        def print_relative(path_list):
-            for p in path_list:
-                p = p.strip()
-                if p:
-                    print(os.path.relpath(p, path))
-        print("*.sock")
-        print("*.socket")
-        print("*.pid")
-        print(".installed*.cfg")
-        for partition in glob.glob(path + "/instance/slappart*"):
-            try:
-              os.chdir(partition)
-            except OSError as e:
-              if e.errno != errno.ENOTDIR:
-                raise
-              continue
-            try:
-                with open("srv/exporter.exclude") as f:
-                    exclude = f.readlines()
-            except IOError as e:
-                if e.errno != errno.ENOENT:
-                    raise
-            else:
-                print_relative(exclude)
-            for installed in glob.glob(".installed*.cfg"):
-                try:
-                    with open(installed) as f:
-                        installed = parse(f, installed)
-                except IOError as e:
-                    if e.errno != errno.ENOENT:
-                        raise
-                else:
-                    for section in installed.itervalues():
-                        print_relative(section.get(
-                            '__buildout_installed__', '').splitlines())
-EOF
-)
-    echo "$exclude" |rsync --exclude-from=- instance "$backup_path"
-  fi
-
-  test -d project  && rsync project "$backup_path"
-  test -d public  && rsync public "$backup_path"
-  test -f proxy.db && rsync proxy.db "$backup_path"
-)
-# We sync .* appart
-(
-  cd "$etc_directory"
-  date +%s -u > .resilient-timestamp
-  rsync config.json "$backup_directory"/etc/
-  # Hidden files are related to the webrunner's internals
-  cp -r .??* "$backup_directory/etc/"
-)
-if [ -d "$backup_directory"/runner/software ]; then
-  rm "$backup_directory"/runner/software/*
-fi
-
-(
-  cd "$backup_directory"
-  find -type f ! -name backup.signature -print0 | xargs -0 sha256sum | sort -k 66 > backup.signature
-)
-
-# Check that export didn't happen during backup of instances
-tmp_backup_sum=$(mktemp -p "$tmp_directory")
-
-remove_tmp_files () {
-  rm "$tmp_backup_sum"
-}
-trap remove_tmp_files EXIT
-
-cd {{ directory['backup'] }}
-
-# Wait a little to increase the probability to detect an ongoing backup.
-sleep 5
-
-# Getting files from runner backup directory, as instance backup files may be
-# explicitely excluded from the backup, using the srv/exporter.exclude
-find -path "./runner/instance/slappart*/srv/backup/*" -type f ! -name backup.signature -print0 |
-xargs -r0 sha256sum | sort -k 66 > "$tmp_backup_sum"
-
-# If no backup found, it's over
-if [ ! -s "$tmp_backup_sum" ]; then
-  exit 0
-fi
-
-# If the diff fails, then the notifier will restart this script
-grep "instance/slappart.*/srv/backup/" "$backup_directory/backup.signature" |
-diff "$tmp_backup_sum" - || {
-  echo "ERROR: Some backups are not consistent, exporter should be re-run."
-  echo "Let's sleep {{ backup_wait_time }} minutes, to let the backup end..."
-  sleep {{ backup_wait_time }}m
-  exit 1
-}