Slaprunner: dont run runner-import/runner-export/pbs-push if the space is full...

Slaprunner: dont run runner-import/runner-export/pbs-push if the space is full on runner1/runner0/runner1 respectively

slapos/recipe/pbs.py

@@ -75,11 +75,20 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
     # Create a simple rdiff-backup wrapper that will push
 
     template = textwrap.dedent("""\
-        #!/bin/sh
+        #!/bin/bash
         #
         # Push data to a PBS *-import instance.
         #
 
+        # check space in *-import instance
+        disk_status_url="%(disk_status_url)s"
+        status=$(wget $disk_status_url -q -O -)
+        if [[ ! -z "$status" ]] && [[ $(echo "$status" | grep '"status": "OK"' | wc -c) -eq 0  ]];
+        then
+          echo "Not enough space on the import instance as reported by monitor: $disk_status_url"
+          exit 0
+        fi
+
         LC_ALL=C
         export LC_ALL
         RDIFF_BACKUP=%(rdiffbackup_binary)s
@@ -92,6 +101,8 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
         """)
 
     template_dict = {
+      'wget': shlex.quote(self.options['wget-binary']),
+      'disk_status_url': shlex.quote(self.options['disk-status-url']),
       'rdiffbackup_binary': shlex.quote(self.options['rdiffbackup-binary']),
       'remote_schema': shlex.quote(remote_schema),
       'remote_dir': shlex.quote(remote_dir),

software/slaprunner/buildout.hash.cfg

@@ -19,23 +19,23 @@ md5sum = 713db528880282d568278f09458d2aab
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = 7df86928aeef0fbada832b0283b9a454
+md5sum = 80f67da85895cfa4717049ee0291992e
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2
-md5sum = 1edd9c7a20e208b6cb647886bfb6d1bb
+md5sum = 6b410c3488967c529bb6b9d6286c559a
 
 [instance-runner-import]
 filename = instance-runner-import.cfg.in
-md5sum = 5cfa49bcf20612844e1c50a85740d0b3
+md5sum = 163fb347303442db6d506db502d9866e
 
 [template-runner-export-script]
 filename = template/runner-export.sh.jinja2
-md5sum = 98ce179badc6af5979a64a7c3d0a2ceb
+md5sum = 92322058c4e350c765ad9eb4b7e2c1a8
 
 [instance-runner-export]
 filename = instance-runner-export.cfg.in
-md5sum = 00ddcaa6e571505d1930612d12de8fde
+md5sum = 23d951c6a7c5d26c9f935dc984aeaa43
 
 [template-resilient]
 filename = instance-resilient.cfg.jinja2

software/slaprunner/instance-runner-export.cfg.in

@@ -23,6 +23,7 @@ parts +=
   shellinabox
   slapos-cfg
   cron-entry-prepare-software
+  cron-entry-retry-export-import-on-space-full
   deploy-instance-parameters
   instance-software
   instance-software-type
@@ -31,6 +32,7 @@ parts +=
   supervisord-wrapper
   supervisord-promise
   httpd-graceful-wrapper
+  check-exporter-disk
 ## Monitoring part
 ## Monitor for runner
   monitor-base
@@ -77,6 +79,7 @@ context =
   raw  output_log_file ${directory:log}/resilient.log
   raw  shell_binary {{ bash_executable_location }}
   raw  rsync_binary {{ rsync_executable_location }}
+  raw  check_disk ${check-exporter-disk:output}
 
 [monitor-httpd-free-port]
 recipe = slapos.cookbook:free_port
@@ -108,6 +111,12 @@ configuration-file-path = ${buildout:directory}/knowledge0.cfg
 [publish-connection-information]
 <= resilient-publish-connection-parameter
 
+[check-exporter-disk]
+<= retry-if-disk-free
+
+[importer-exporter-binary]
+path = ${notifier-exporter:wrapper}
+
 [monitor-check-resilient-feed-file]
 recipe = slapos.recipe.template:jinja2
 template = {{ monitor_check_resilient_feed_template_path }}
@@ -118,3 +127,4 @@ context =
   key monitor_feed_directory monitor-directory:public
   raw base_url http://[${notifier:host}]:${notifier:port}/get/
   raw python_executable ${buildout:executable}
+

software/slaprunner/instance-runner-import.cfg.in

@@ -21,6 +21,7 @@ parts +=
   symlinks
   slapos-cfg
   cron-entry-prepare-software
+  cron-entry-retry-export-import-on-space-full
   deploy-instance-parameters
   instance-software-type
   bash-profile
@@ -92,6 +93,7 @@ context =
   key ipv6 slaprunner:ipv6
   key proxy_port slaprunner:proxy_port
   key instance_folder slaprunner:instance_root
+  key check_disk check-importer-disk:output
   section directory directory
   section supervisord supervisord
   raw  output_log_file ${directory:log}/${:resilient-log-basename}
@@ -176,6 +178,12 @@ instance-configuration =
   raw takeover-password ${resilient-web-takeover-password:passwd}
 configuration-file-path = ${buildout:directory}/knowledge0.cfg
 
+[check-importer-disk]
+<= retry-if-disk-free
+
+[importer-exporter-binary]
+path = ${importer:wrapper}
+
 [monitor-conf-parameters]
 private-path-list += 
   ${directory:logrotate-backup}

software/slaprunner/instance-runner.cfg

@@ -20,6 +20,7 @@ parts =
   shellinabox
   slapos-cfg
   cron-entry-prepare-software
+  cron-entry-retry-export-import-on-space-full
   deploy-instance-parameters
   instance-software
   instance-software-type
@@ -171,11 +172,13 @@ software_info_json = $${runnerdirectory:home}/software_info.json
 instance_info_json = $${runnerdirectory:home}/instance_info.json
 path = $${shell-environment:path}
 instance_name = $${slap-parameter:instance-name}
+retry-exporter-importer-status =  $${directory:srv}/retry-exporter-importer-status
 
 default_repository = $${slap-parameter:slapos-repository}
 default_repository_branch = $${slap-parameter:slapos-reference}
 
 
+
 #---------------------------
 #--
 #-- supervisord managing slaprunner instance processes
@@ -808,6 +811,46 @@ rendered = $${monitor-directory:reports}/$${:filename}
 filename = monitor-check-webrunner-internal-instance
 mode = 0744
 
+[retry-if-disk-free]
+recipe = collective.recipe.template
+input = inline: #!/bin/sh
+  FROM_CRON=$1
+  MONITOR_STATUS_FILE=$${monitor-directory:public}/check-free-disk-space.status.json
+  STATUS_FILE=$${slaprunner:retry-exporter-importer-status}
+
+  # Skip if there is no monitor status file
+  if [ -f $MONITOR_STATUS_FILE ]; then
+    if [ -z "$(cat $MONITOR_STATUS_FILE | grep '\"status\": \"OK\"')" ] && [ "$FROM_CRON" = false ]; then
+      # If space is full
+      echo "Disk is full or less than the expected threshold. Exiting..";
+      # Activate Export/Import Retry
+      echo 1 > $STATUS_FILE || exit 0; 
+      exit 1;
+    fi
+    if [ "$FROM_CRON" = false ]; then
+      # Deactivate Export/Import Retry
+      echo 0 > $STATUS_FILE
+      echo "Disk check passed";
+    else 
+      # This must be a call from cron
+      if [ `cat $STATUS_FILE` -eq 1 ]; then
+        exec $${importer-exporter-binary:path}
+      fi
+    fi
+  fi
+output = $${directory:bin}/check-and-retry-until-disk-free
+mode = 700
+
+[importer-exporter-binary]
+path =
+
+[cron-entry-retry-export-import-on-space-full]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = retry-export-import
+frequency = */10 * * * *
+command = $${retry-if-disk-free:output} true
+
 ## Slapuser slapos command script
 [template-slapuser-script]
 recipe = slapos.recipe.template:jinja2

software/slaprunner/template/runner-export.sh.jinja2

@@ -19,6 +19,11 @@ backup_directory='{{ directory["backup"] }}'
 etc_directory='{{ directory["etc"] }}'
 tmp_directory='{{ directory["tmp"] }}'
 
+# Check if we have disk space on this computer
+(
+  exec {{ check_disk }} false
+)
+
 rsync () {
   # Workaround for bug https://bugzilla.samba.org/show_bug.cgi?id=3653
   IGNOREEXIT=24

software/slaprunner/template/runner-import.sh.jinja2

@@ -35,6 +35,9 @@ log_message () {
 # Delete the error message file, to not keep it even after a successful build
 rm "$RESTORE_ERROR_MESSAGE_FILE" || true
 
+log_message "Checking disk space on the machine..."
+( exec {{ check_disk }} false )
+
 rsync () {
   set -x
   '{{ rsync_binary }}' -rlptgov --stats --safe-links --delete "$@"

stack/resilient/buildout.cfg

@@ -6,6 +6,7 @@ extends =
   ../../component/openssh/buildout.cfg
   ../../component/rdiff-backup/buildout.cfg
   ../../component/rsync/buildout.cfg
+  ../../component/wget/buildout.cfg
   ../monitor/buildout.cfg
 
 parts =
@@ -22,6 +23,7 @@ parts =
   gzip
   rdiff-backup
   dash
+  wget
 
 [pbs-recipe-egg]
 recipe = zc.recipe.egg

stack/resilient/buildout.hash.cfg

@@ -27,7 +27,7 @@ md5sum = c6c11db5372150019debb1ce519b907d
 
 [template-pull-backup]
 filename = instance-pull-backup.cfg.in
-md5sum = fe4811540683bfcfac5ac12de084b623
+md5sum = 505b0c0f55497f01e9f5d9d317f5b325
 
 [template-replicated]
 filename = template-replicated.cfg.in

stack/resilient/instance-pull-backup.cfg.in

@@ -139,6 +139,8 @@ ignore-known-hosts-file = $${slap-parameter:ignore-known-hosts-file}
 instance-root-name = $${instance-info-parameters:root-name}
 log-url = $${monitor-publish-parameters:monitor-base-url}/private/notifier/
 status-item-directory = $${directory:notifier-status-items}
+disk-status-url = $${monitor-publish-parameters:monitor-base-url}/public/check-free-disk-space.status.json
+wget-binary = ${wget:location}/bin/wget
 
 [pbs-resilient-status-feed]
 recipe = slapos.cookbook:wrapper