slaprunner: partial rewrite of import/export scrips in Python

Further rewrite should be done but at least there's no awk code anymore.

component/gawk/buildout.cfg

-[buildout]
-parts = gawk
-
-[gawk]
-recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/gawk/gawk-4.1.4.tar.gz
-md5sum = f20c94ca51b6ebfc9bffb90f95c8ffbb

software/slaprunner/buildout.hash.cfg

@@ -15,7 +15,7 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = e89d25c1bb7bf507bd96973d444934a6
+md5sum = 78e4c796b15a9c5b9b41c79a3c995ee7
 
 [template-runner]
 filename = instance-runner.cfg
@@ -23,19 +23,19 @@ md5sum = c76c66c331f1f0d9f416c56ead53e804
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2
-md5sum = feee1dc29d160e0d796dab0ba767745f
+md5sum = 40e839e76d03fabefb1859ea72383529
 
 [instance-runner-import]
 filename = instance-runner-import.cfg.in
-md5sum = 70931f214b3f7c3992e29c150a30d3cf
+md5sum = 5d49f4fe6de7bd02db0067d0aeac6de4
 
 [template-runner-export-script]
 filename = template/runner-export.sh.jinja2
-md5sum = b921435ab5a8c408026e74d9a40eea44
+md5sum = 02023f89b622c49aee42130f839846a7
 
 [instance-runner-export]
 filename = instance-runner-export.cfg.in
-md5sum = 546b368baaed84bd2d3be7d6b68a5030
+md5sum = 22a5dfe9bfc8533f8eab851e0996a6d7
 
 [template-resilient]
 filename = instance-resilient.cfg.jinja2

software/slaprunner/common.cfg

@@ -2,12 +2,10 @@
 extends =
   buildout.hash.cfg
   ../../component/bash/buildout.cfg
-  ../../component/coreutils/buildout.cfg
   ../../component/busybox/buildout.cfg
   ../../component/curl/buildout.cfg
   ../../component/dash/buildout.cfg
   ../../component/dcron/buildout.cfg
-  ../../component/gawk/buildout.cfg
   ../../component/git/buildout.cfg
   ../../component/tig/buildout.cfg
   ../../component/logrotate/buildout.cfg

software/slaprunner/instance-runner-export.cfg.in

@@ -62,11 +62,6 @@ ip = ${slaprunner:ipv4}
 [supervisord]
 port = ${supervisord-free-port:port}
 
-[exporter-configuration]
-coreutils-location = {{ dumps(parameter_dict['coreutils-location']) }}
-gawk-location = {{ dumps(parameter_dict['gawk-location']) }}
-backup_wait_time = {{ dumps(parameter_dict['backup_wait_time']) }}
-
 [exporter]
 recipe = slapos.recipe.template:jinja2
 template = {{ exporter_script_path }}
@@ -75,8 +70,10 @@ rendered = ${directory:bin}/${slap-parameter:namebase}-exporter
 wrapper = ${:rendered}
 mode = 700
 context =
+  import sys sys
+  import easy_install zc.buildout.easy_install
   section directory directory
-  section parameter_dict exporter-configuration
+  raw  backup_wait_time {{ backup_wait_time }}
   raw  output_log_file ${directory:log}/resilient.log
   raw  shell_binary {{ bash_executable_location }}
   raw  rsync_binary {{ rsync_executable_location }}

software/slaprunner/instance-runner-import.cfg.in

@@ -72,10 +72,6 @@ maximum = 22241
 [cron-entry-prepare-software]
 recipe =
 
-[importer-configuration]
-coreutils-location = ${coreutils:location}
-gawk-location = ${gawk:location}
-
 [importer]
 recipe = slapos.recipe.template:jinja2
 template = ${template-runner-import-script:location}/${template-runner-import-script:filename}
@@ -89,12 +85,13 @@ restore-error-message-file = $${directory:srv}/$${:restore-error-message-file-ba
 restore-error-message-file-basename = importer-error-message-file
 resilient-log-basename = resilient.log
 context =
+  import sys sys
+  import easy_install zc.buildout.easy_install
   key backend_url slaprunner:access-url
   key ipv4 slaprunner:ipv4
   key ipv6 slaprunner:ipv6
   key proxy_port slaprunner:proxy_port
   key instance_folder slaprunner:instance_root
-  section parameter_dict importer-configuration
   section directory directory
   section supervisord supervisord
   raw  output_log_file $${directory:log}/$${:resilient-log-basename}

software/slaprunner/instance.cfg

@@ -47,11 +47,6 @@ import-list = file parts :template-parts-destination
               file replicated :template-replicated-destination
 mode = 0644
 
-[template-runner-export-configuration]
-coreutils-location = ${coreutils:location}
-gawk-location = ${gawk:location}
-backup_wait_time = ${exporter-default-configuration:backup_wait_time}
-
 [template-runner-export]
 recipe = slapos.recipe.template:jinja2
 template = ${instance-runner-export:target}
@@ -60,7 +55,7 @@ mode = 640
 context =
   key pbsready_export_template_path template-pbsready-export:rendered
   key template_runner_path instance-base-runner:rendered
-  section parameter_dict template-runner-export-configuration
+  raw backup_wait_time ${exporter-default-configuration:backup_wait_time}
   raw exporter_script_path ${template-runner-export-script:location}/${template-runner-export-script:filename}
   raw monitor_check_resilient_feed_template_path ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename}
   raw buildout_executable_location ${buildout:executable}

software/slaprunner/template/runner-export.sh.jinja2

@@ -24,15 +24,8 @@ rsync () {
   set +x
 }
 
-relativise () {
-  while IFS= read -r line; do
-    if [ ! -z "$line" ]; then
-      '{{ parameter_dict["coreutils-location"] }}/bin/realpath' --quiet --canonicalize-missing --no-symlinks --relative-to="$1" "$line"
-    fi
-  done
-}
-
 (
+  # XXX: code duplication with runner-import.sh.jinja2
   path=$srv_directory/runner
   backup_path=$backup_directory/runner/
   cd "$path"
@@ -41,31 +34,46 @@ relativise () {
     # Concatenate the exclude file of each partition of webrunner
     # to create a global exclude file.
     # Also, ignore all buildout-managed files.
-    (
-      echo "*.sock"
-      echo "*.socket"
-      echo "*.pid"
-      echo ".installed*.cfg"
-      for partition in "$path"/instance/slappart*; do
-        # So "relativise" can handle relative paths (which are expected to be relative to partition).
-        cd "$partition"
-        exclude_file=srv/exporter.exclude
-        if [ -r "$exclude_file" ]; then
-          relativise "$path" < "$exclude_file"
-        fi
-        for installed in .installed*.cfg; do
-          if [ -r "$installed" ]; then
-            # Print every line from each __buildout_installed__ found.
-            '{{ parameter_dict["gawk-location"] }}/bin/gawk' '
-              BEGIN { do_print = 0 }
-              match($0, /^__buildout_installed__\s*=\s*(\S.*)/, ary) { do_print = 1; print ary[1]; next }
-              /^\S/ { do_print = 0; next }
-              match($0, /^\s+(\S.*)/, ary) { if (do_print) print ary[1] }
-            ' "$installed" | relativise "$path"
-          fi
-        done
-      done
-    ) | rsync --exclude-from=- instance "$backup_path"
+    exclude=$({{ sys.executable }} - "$path" <<EOF
+if 1:
+        import glob, errno, os, sys
+        sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }}
+        from zc.buildout.configparser import parse
+        path = sys.argv[1]
+
+        def print_relative(path_list):
+            for p in path_list:
+                p = p.strip()
+                if p:
+                    print(os.path.relpath(p, path))
+        print("*.sock")
+        print("*.socket")
+        print("*.pid")
+        print(".installed*.cfg")
+        for partition in glob.glob(path + "/instance/slappart*"):
+            os.chdir(partition)
+            try:
+                with open("srv/exporter.exclude") as f:
+                    exclude = f.readlines()
+            except IOError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+            else:
+                print_relative(exclude)
+            for installed in glob.glob(".installed*.cfg"):
+                try:
+                    with open(installed) as f:
+                        installed = parse(f, installed)
+                except IOError as e:
+                    if e.errno != errno.ENOENT:
+                        raise
+                else:
+                    for section in installed.itervalues():
+                        print_relative(section.get(
+                            '__buildout_installed__', '').splitlines())
+EOF
+)
+    echo "$exclude" |rsync --exclude-from=- instance "$backup_path"
   fi
 
   test -d project  && rsync project "$backup_path"
@@ -116,6 +124,6 @@ if diff "$tmp_backup_sum" "$tmp_filtered_signature"; then
   exit 0
 fi
 echo "ERROR: Some backups are not consistent, exporter should be re-run."
-echo "Let's sleep {{ parameter_dict['backup_wait_time'] }} minutes, to let the backup end..."
-sleep {{ parameter_dict["backup_wait_time"] }}m
+echo "Let's sleep {{ backup_wait_time }} minutes, to let the backup end..."
+sleep {{ backup_wait_time }}m
 exit 1

software/slaprunner/template/runner-import.sh.jinja2

@@ -41,16 +41,9 @@ rsync () {
   set +x
 }
 
-relativise () {
-  while IFS= read -r line; do
-    if [ ! -z "$line" ]; then
-      '{{ parameter_dict["coreutils-location"] }}/bin/realpath' --quiet --canonicalize-missing --no-symlinks --relative-to="$1" "$line"
-    fi
-  done
-}
-
 log_message "Restoring WebRunner content..."
 (
+  # XXX: code duplication with runner-export.sh.jinja2
   path=$srv_directory/runner
   backup_path=$backup_directory/runner/
   cd "$backup_path"
@@ -59,31 +52,46 @@ log_message "Restoring WebRunner content..."
     # Concatenate the exclude file of each partition of webrunner
     # to create a global exclude file.
     # Also, ignore all buildout-managed files.
-    (
-      echo "*.sock"
-      echo "*.socket"
-      echo "*.pid"
-      echo ".installed*.cfg"
-      for partition in "$path"/instance/slappart*; do
-        # So "relativise" can handle relative paths (which are expected to be relative to partition).
-        cd "$partition"
-        exclude_file=srv/exporter.exclude
-        if [ -r "$exclude_file" ]; then
-          relativise "$path" < "$exclude_file"
-        fi
-        for installed in .installed*.cfg; do
-          if [ -r "$installed" ]; then
-            # Print every line from each __buildout_installed__ found.
-            '{{ parameter_dict["gawk-location"] }}/bin/gawk' '
-              BEGIN { do_print = 0 }
-              match($0, /^__buildout_installed__\s*=\s*(\S.*)/, ary) { do_print = 1; print ary[1]; next }
-              /^\S/ { do_print = 0; next }
-              match($0, /^\s+(\S.*)/, ary) { if (do_print) print ary[1] }
-            ' "$installed" | relativise "$path"
-          fi
-        done
-      done
-    ) | rsync --exclude-from=- instance "$path"
+    exclude=$({{ sys.executable }} - "$path" <<EOF
+if 1:
+        import glob, errno, os, sys
+        sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }}
+        from zc.buildout.configparser import parse
+        path = sys.argv[1]
+
+        def print_relative(path_list):
+            for p in path_list:
+                p = p.strip()
+                if p:
+                    print(os.path.relpath(p, path))
+        print("*.sock")
+        print("*.socket")
+        print("*.pid")
+        print(".installed*.cfg")
+        for partition in glob.glob(path + "/instance/slappart*"):
+            os.chdir(partition)
+            try:
+                with open("srv/exporter.exclude") as f:
+                    exclude = f.readlines()
+            except IOError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+            else:
+                print_relative(exclude)
+            for installed in glob.glob(".installed*.cfg"):
+                try:
+                    with open(installed) as f:
+                        installed = parse(f, installed)
+                except IOError as e:
+                    if e.errno != errno.ENOENT:
+                        raise
+                else:
+                    for section in installed.itervalues():
+                        print_relative(section.get(
+                            '__buildout_installed__', '').splitlines())
+EOF
+)
+    echo "$exclude" |rsync --exclude-from=- instance "$path"
   fi
 
   test -d project  && rsync project "$path"

[Vincent Pelletier]

Bad raw usage, allows buildout syntax injection. Use a separate section and dumps(), as was done before.

[EDIT]: Actually, the injection is likely not at buildout syntax level (depending on expansion effect on parsing), but if not then it is at jinja2 template context declaraion level. Ex: backup_wait_time = 'dummy\n import some_value some_module'.

[Julien Muchembled]

"syntax injection" > why should we protect against it ?

Working on instantiation is a task that I find complicated enough like that, for example with all those values that have to be propagated (even in places, where I'd have preferred it was implicit). The previous form looked too much for me.

[Vincent Pelletier]

Bad raw usage, use key.

[Julien Muchembled]

Note that it's a classic template, so not using raw would be doing like before.