software/theia: Add import script

software/theia/buildout.hash.cfg

@@ -19,11 +19,11 @@ md5sum = aff6fec658a1d38acd4ff34615da8d4a
 
 [instance]
 filename = instance.cfg.in
-md5sum = dbb8f3edad591b3715f3f49f52035f79
+md5sum = 10d70b4383876897a661440edf67e2b3
 
 [instance-import]
 filename = instance-import.cfg.in
-md5sum = 0398a33f33dc99679da0728b2cfbcbf9
+md5sum = aa5e6f46be508d674c56e2ff20551bc6
 
 [instance-export]
 filename = instance-export.cfg.in
@@ -37,6 +37,10 @@ md5sum = 01a1bd2a0202d33433e8f08103fcba42
 filename = theia-export-script.jinja
 md5sum = 56e139e3cce8ab034487e13fa78c5fb8
 
+[theia-import-script]
+filename = theia-import-script.jinja
+md5sum = a7a148231fcf97f6fcfe87c16df208b2
+
 [yarn.lock]
 filename = yarn.lock
 md5sum = e9c0f6cc380b53425f521a167c2fdf66

software/theia/instance-import.cfg.in

@@ -22,10 +22,37 @@ name = Theia Import Frontend
 # is notified that the backup files have just been pushed to it.
 # All it expects is that a script be available in importer:wrapper.
 [importer]
-recipe = slapos.cookbook:wrapper
-command-line = echo "Import Not Implemented Yet"
-wrapper-path = $${directory:bin}/$${slap-parameter:namebase}-exporter
-wrapper = $${:wrapper-path}
+wrapper = $${theia-import-script:rendered}
+
+[theia-import-script]
+recipe = slapos.recipe.template:jinja2
+template = ${software-info:theia-import-script}
+rendered = $${directory:bin}/theia-import-script
+mode = 700
+restore-exit-code-file = $${directory:srv}/$${:restore-exit-code-file-basename}
+restore-exit-code-file-basename = importer-exit-code-file
+restore-error-message-file = $${directory:srv}/$${:restore-error-message-file-basename}
+restore-error-message-file-basename = importer-error-message-file
+resilient-log-basename = resilient.log
+context =
+  import sys sys
+  import easy_install zc.buildout.easy_install
+  key ipv4 slapos-standalone-config:ipv4
+  key ipv6 slapos-standalone-config:ipv6
+  key proxy_port slapos-standalone-config:port
+  raw instance_folder $${directory:runner}/instance
+  section directory directory
+  raw slapos ${software-info:slapos}
+  raw slapos_cfg $${directory:runner}/etc/slapos.cfg
+  raw slapos_node_software_log $${directory:runner}/var/log/slapos-node-software.log
+  raw slapos_node_instance_log $${directory:runner}/var/log/slapos-node-instance.log
+  raw  output_log_file $${directory:var}/log/$${:resilient-log-basename}
+  raw  shell_binary ${software-info:bash}
+  raw  sqlite3_binary ${software-info:sqlite3}
+  raw  rsync_binary ${software-info:rsync}
+  raw  restore_exit_code_file $${:restore-exit-code-file}
+  raw  restore_error_message_file $${:restore-error-message-file}
+
 
 # Resilient connection parameters of import instance are published
 # through the resilient stack.

software/theia/instance.cfg.in

@@ -9,11 +9,14 @@ eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 
 [software-info]
+slapos = ${buildout:bin-directory}/slapos
 python-with-eggs = ${buildout:bin-directory}/${python-with-eggs:interpreter}
 python = ${python:location}/bin/python
 rsync = ${rsync:location}/bin/rsync
 sqlite3 = ${sqlite3:location}/bin/sqlite3
+bash = ${bash:location}/bin/bash
 theia-export-script = ${theia-export-script:output}
+theia-import-script = ${theia-import-script:output}
 
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration

software/theia/software.cfg

@@ -372,3 +372,6 @@ output = ${buildout:directory}/instance.cfg
 
 [theia-export-script]
 <= download-base
+
+[theia-import-script]
+<= download-base

software/theia/theia-import-script.jinja

+#!{{ shell_binary }}
+LC_ALL=C
+export LC_ALL
+umask 077
+
+# Exit on any error, to prevent inconsistent backup
+# Error on unset variable expansion
+set -eu
+
+# Redirect output to log
+exec > >(tee -ai '{{ output_log_file }}')
+exec 2>&1
+
+echo -e "\n\n$0 run at : $(date)"
+
+srv_directory='{{ directory["srv"] }}'
+backup_directory='{{ directory["backup"] }}'
+etc_directory='{{ directory["etc"] }}'
+
+RESTORE_EXIT_CODE_FILE='{{ restore_exit_code_file }}'
+RESTORE_ERROR_MESSAGE_FILE='{{ restore_error_message_file }}'
+ERROR_MESSAGE=""
+
+fail_with_exit_code () {
+  echo 1 > $RESTORE_EXIT_CODE_FILE
+  echo -e "Failure during step : $ERROR_MESSAGE" > $RESTORE_ERROR_MESSAGE_FILE
+  exit 1
+}
+trap fail_with_exit_code ERR
+
+log_message () {
+    ERROR_MESSAGE=$1
+    echo -e $1
+}
+# Delete the error message file, to not keep it even after a successful build
+rm "$RESTORE_ERROR_MESSAGE_FILE" || true
+
+rsync () {
+  set -x
+  '{{ rsync_binary }}' -rlptgov --stats --safe-links --delete "$@"
+  set +x
+}
+
+SQLITE3="{{ sqlite3_binary }}"
+
+log_message "Restoring WebRunner content..."
+(
+  # XXX: code duplication with runner-export.sh.jinja2
+  path=$srv_directory
+  backup_path=$backup_directory/srv
+  cd "$backup_path"
+
+  if [ -d instance ]; then
+    # Concatenate the exclude file of each partition of webrunner
+    # to create a global exclude file.
+    # Also, ignore all buildout-managed files.
+    exclude=$({{ sys.executable }} - "$path" <<EOF
+if 1:
+        import glob, errno, os, sys
+        sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }}
+        from zc.buildout.configparser import parse
+        path = sys.argv[1]
+
+        def print_relative(path_list):
+            for p in path_list:
+                p = p.strip()
+                if p:
+                    print(os.path.relpath(p, path))
+        print("*.sock")
+        print("*.socket")
+        print("*.pid")
+        print(".installed*.cfg")
+        for partition in glob.glob(path + "/instance/slappart*"):
+            try:
+              os.chdir(partition)
+            except OSError as e:
+              if e.errno != errno.ENOTDIR:
+                raise
+              continue
+            try:
+                with open("srv/exporter.exclude") as f:
+                    exclude = f.readlines()
+            except IOError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+            else:
+                print_relative(exclude)
+            for installed in glob.glob(".installed*.cfg"):
+                try:
+                    with open(installed) as f:
+                        installed = parse(f, installed)
+                except IOError as e:
+                    if e.errno != errno.ENOENT:
+                        raise
+                else:
+                    for section in installed.itervalues():
+                        print_relative(section.get(
+                            '__buildout_installed__', '').splitlines())
+EOF
+)
+    echo "$exclude" |rsync --exclude-from=- instance "$path"
+  fi
+
+  test -d project  && rsync project "$path"
+  test -d frontend-static/public  && rsync frontend-static/public "$path/frontend-static"
+  test -f runner/var/proxy.db.dump && rm $path/runner/var/proxy.db && $SQLITE3 $path/runner/var/proxy.db ".read runner/var/proxy.db.dump"
+)
+
+log_message "Restoring WebRunner config (etc directory)..."
+(
+  cd "$backup_directory"/etc/
+  # Hidden files are related to the webrunner's internals
+  cp -r .??* "$etc_directory"
+)
+
+# Invoke arbitrary script to perform specific restoration
+# procedure.
+runner_import_restore=$srv_directory/runner-import-restore
+if [ -x "$runner_import_restore" ]; then
+  log_message "Running $runner_import_restore..."
+  "$srv_directory/runner-import-restore"
+fi
+
+# If no "etc/.project" neither "srv/runner/var/proxy.db", we can safely assume
+# that there is no instance deployed on runner0
+if [ ! -f "$etc_directory/.project" -a ! -f "$srv_directory/runner/var/proxy.db" ]; then
+  log_message "No Software Requested... Writing status file... End"
+  echo 0 > $RESTORE_EXIT_CODE_FILE
+  exit 0
+fi
+
+log_message "Updating slapproxy database..."
+HOME='{{ directory["home"] }}'
+# XXX Hardcoded
+export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+export MAKEFLAGS=-j4
+SLAPOS='{{ slapos }}'
+DATABASE="$HOME/srv/runner/var/proxy.db"
+db_query () {
+  # Try opening locked tables for 5 seconds to prevent "database is locked" error
+"$SQLITE3" "$DATABASE" <<EOF
+.timeout 5000
+$@
+EOF
+}
+# If slapproxy database is empty then no software release was opened
+if [ ! -s "$DATABASE" ]; then
+  log_message "Slapproxy database empty, no Software Requested... Writing status file... End"
+  echo 0 > $RESTORE_EXIT_CODE_FILE
+  exit 0
+fi
+
+# Slap proxy table contain version number, find the table name dynamically.
+# This is known to work with version 11 or 12 of tables, but it will probably
+# work with earlier versions as well.
+DB_PARTITION_TABLE=$(db_query ".table partition__")
+DB_PARTITION_NETWORK_TABLE=$(db_query ".table partition\_network__")
+DB_SOFTWARE_TABLE=$(db_query ".table software__")
+
+# Change slapproxy database to point instances to new software release
+# XXX hardcoded
+PARTITION=$(basename $HOME)
+OLD_SOFTWARE_RELEASE=$(db_query "select software_release from $DB_PARTITION_TABLE where reference='slappart0';")
+if [ "$OLD_SOFTWARE_RELEASE" == "" ];
+then
+   echo "No instance configured"
+   echo 0 > $RESTORE_EXIT_CODE_FILE
+   exit 0
+fi
+SOFTWARE_RELEASE=$({{ sys.executable }} - $OLD_SOFTWARE_RELEASE $PARTITION <<EOF
+if 1:
+  import os, re, sys
+
+  # We want to replace the last occurence only
+  old_software_release, partition = sys.argv[1], sys.argv[2]
+  for match in re.finditer("(slappart|test0-|s)[0-9][0-9]*", old_software_release):
+    start, end = match.start(), match.end()
+
+  print old_software_release[:start] + partition + old_software_release[end:]
+EOF
+)
+
+db_query "update $DB_PARTITION_TABLE set software_release='$SOFTWARE_RELEASE' where software_release NOT NULL;"
+db_query "update $DB_SOFTWARE_TABLE set url='$SOFTWARE_RELEASE' where url='$OLD_SOFTWARE_RELEASE';" || db_query "delete from $DB_SOFTWARE_TABLE where url='$OLD_SOFTWARE_RELEASE';"
+# Change slapproxy database to have all instances stopped
+db_query "update $DB_PARTITION_TABLE set requested_state='stopped';"
+# Change slapproxy database to get correct IPs
+IPV4='{{ ipv4 }}'
+IPV6='{{ ipv6 }}'
+db_query "update $DB_PARTITION_NETWORK_TABLE set address='$IPV4' where netmask='255.255.255.255';"
+db_query "update $DB_PARTITION_NETWORK_TABLE set address='$IPV6' where netmask='ffff:ffff:ffff::';"
+
+MASTERURL='http://{{ ipv4 }}:{{ proxy_port }}'
+
+log_message "Removing old supervisord service description files..."
+# XXX: Path hardcoded in slapos.core
+rm '{{ instance_folder }}'/etc/supervisord.conf.d/* || true
+
+SLAPOSCFG='{{ slapos_cfg }}'
+SLAPGRIDSRLOG='{{ slapos_node_software_log }}'
+SLAPGRIDCPLOG='{{ slapos_node_instance_log }}'
+
+contain_software_release=0
+
+SOFTWARE_RELEASES_COUNT=$(db_query "SELECT count(1) FROM $DB_SOFTWARE_TABLE WHERE url != '';")
+if [ $SOFTWARE_RELEASES_COUNT -gt 0 ]; then
+  contain_software_release=1
+fi
+
+if [ $contain_software_release -eq 0 ]; then
+  log_message "No Software Release were deployed, so skip to continue..."
+  echo 0 > $RESTORE_EXIT_CODE_FILE
+  exit 0
+fi
+
+log_message "Building newest Software Release..."
+"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 ||
+"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 ||
+"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 ||
+(tail -n 200 "$SLAPGRIDSRLOG" && false)
+
+contain_instance=0
+for folder in $srv_directory/runner/instance/slappart*/; do
+  if [ -f "$folder/buildout.cfg" ]; then
+    contain_instance=1
+  fi
+done
+
+# If instance do not contains template.cfg it means the user contains no instance.
+# so it is safer to assume that he is using slaprunner for develop buildout rather them slapos.
+if [ $contain_instance -eq 0 ]; then
+  log_message "None Instance were deployed with this software release, so skip to continue..."
+  echo 0 > $RESTORE_EXIT_CODE_FILE
+  exit 0
+fi
+
+# Remove defined scripts to force buildout to recreate them to have updated paths
+rm "$srv_directory"/runner/instance/slappart*/srv/runner-import-restore || true
+log_message "Fixing Instances as needed after import..."
+# XXX hardcoded
+"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" >/dev/null 2>&1 ||
+"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" >/dev/null 2>&1 ||
+"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" >/dev/null 2>&1 ||
+(tail -n 200 "$SLAPGRIDCPLOG" && false)
+
+# Invoke defined scripts for each partition inside of slaprunner
+log_message "Invoke custom import scripts defined by each instances..."
+for partition in "$srv_directory"/runner/instance/slappart*/
+do
+  script=$partition/srv/runner-import-restore
+  if [ -x "$script" ]; then
+    log_message "Running custom instance script : $script..."
+    "$script"
+  fi
+done
+
+# Change back slapproxy database to have all instances started
+DB_PARTITION_TABLE=$(db_query ".table partition__")
+log_message "Set instances as to start after takeover..."
+db_query "update $DB_PARTITION_TABLE set requested_state='started';"
+
+# Write exit code to an arbitrary file that will be checked by promise/monitor
+log_message "Writing status file... End"
+echo 0 > $RESTORE_EXIT_CODE_FILE
+exit 0