Merge remote-tracking branch 'origin/slaprunner'

component/nodejs/buildout.cfg

@@ -14,8 +14,8 @@ parts =
 [nodejs-0.8]
 # Server-side Javascript.
 recipe = slapos.recipe.cmmi
-url = http://nodejs.org/dist/v0.8.14/node-v0.8.14.tar.gz
-md5sum = 284fd2c7578064c339d9cf6a3a475ac7
+url = http://nodejs.org/dist/v0.8.23/node-v0.8.23.tar.gz
+md5sum = 22fe54ac365f52d3d80ecf748e7323d5
 configure-options =
   --openssl-includes=${openssl:location}/include
   --openssl-libpath=${openssl:location}/lib

setup.py

@@ -188,6 +188,8 @@ setup(name=name,
           'slapreport = slapos.recipe.slapreport:Recipe',
           'slaprunner = slapos.recipe.slaprunner:Recipe',
           'slaprunner.test = slapos.recipe.slaprunner:Test',
+          'slaprunner.export = slapos.recipe.slaprunner.backup:ExportRecipe',
+          'slaprunner.import = slapos.recipe.slaprunner.backup:ImportRecipe',
           'softwaretype = slapos.recipe.softwaretype:Recipe',
           'sphinx= slapos.recipe.sphinx:Recipe',
           'sshkeys_authority = slapos.recipe.sshkeys_authority:Recipe',

slapos/recipe/slaprunner/backup.py

+##############################################################################
+#
+# Copyright (c) 2013 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import textwrap
+
+from slapos.recipe.librecipe import GenericBaseRecipe
+
+
+
+class ExportRecipe(GenericBaseRecipe):
+    """\
+    This recipe creates an exporter script for using with the resilient stack.
+
+    Required options:
+        backup-directory
+            folder that will contain the dump file.
+        srv-directory
+            folder that contain the runner directory.
+        wrapper
+            full path of the exporter script to create.
+    """
+
+    def install(self):
+        wrapper = self.options['wrapper']
+        self.createBackupScript(wrapper)
+        return [wrapper]
+
+
+    def createBackupScript(self, wrapper):
+        """\
+        Create a script to backup the database in 'custom' format.
+        """
+        content = textwrap.dedent("""\
+                #!%(shell-binary)s
+                umask 077
+                sync_element () {
+                  path=$1
+                  backup_path=$2
+                  shift 2
+                  element_list=$*
+                  for element in $element_list
+                  do
+                    cd $path;
+                    if [ -f $element ] || [ -d $element ]; then
+                       %(rsync-binary)s -avz --safe-links --delete $element  $backup_path;
+                    fi
+                  done
+                }
+                sync_element %(srv-directory)s/runner  %(backup-directory)s/runner/ instance project  proxy.db softwareLink
+                sync_element %(etc-directory)s  %(backup-directory)s/etc/ .rcode .project .users ssh
+                if [ -d %(backup-directory)s/runner/software ]; then
+                  rm %(backup-directory)s/runner/software/*
+                fi
+                """ % self.options)
+        self.createExecutable(wrapper, content=content)
+
+
+
+class ImportRecipe(GenericBaseRecipe):
+    """\
+    This recipe creates an importer script for using with the resilient stack.
+
+    Required options:
+        backup-directory
+            folder that will contain the dump file.
+        srv-directory
+            folder that contain the runner directory.
+        wrapper
+            full path of the exporter script to create.
+    """
+
+    def install(self):
+        wrapper = self.options['wrapper']
+        self.createRestoreScript(wrapper)
+        return [wrapper]
+
+
+    def createRestoreScript(self, wrapper):
+        """\
+        Create a script to restore the database from 'custom' format.
+        """
+        content = textwrap.dedent("""\
+                #!%(shell-binary)s
+                umask 077
+                restore_element () {
+                  backup_path=$1
+                  restore_path=$2
+                  shift 2
+                  element_list=$*
+                  for element in $element_list
+                  do
+                    cd $backup_path;
+                    if [ -f $element ] || [ -d $element ]; then
+                       %(rsync-binary)s -avz --delete $backup_path/$element $restore_path;
+                    fi
+                  done
+                }
+                restore_element %(backup-directory)s/runner/ %(srv-directory)s/runner  instance project  proxy.db softwareLink
+                restore_element  %(backup-directory)s/etc/ %(etc-directory)s .rcode .project .users ssh
+                ifs=$IFS IFS=';'
+                read user pass remaining < %(etc-directory)s/.users
+                IFS=$ifs
+                %(curl-binary)s -vg6L -F clogin="$user" -F cpwd="$pass" --dump-header login_cookie  %(backend-url)s/doLogin;
+                %(curl-binary)s -vg6LX POST --cookie login_cookie --max-time 5  %(backend-url)s/runSoftwareProfile;
+                rm -f login_cookie
+                """ % self.options)
+        self.createExecutable(wrapper, content=content)
+
+

software/slaprunner/TODO.txt

+- Use https to connect to the runner. !!! Critical
+- resilient sr: Cloned instances should not launch slapgrid-sr if it was not launched on export instance

software/slaprunner/common.cfg

 [buildout]
 extends =
+  ../../component/bash/buildout.cfg
   ../../component/cloud9/buildout.cfg
+  ../../component/curl/buildout.cfg
   ../../component/dash/buildout.cfg
   ../../component/dropbear/buildout.cfg
   ../../component/git/buildout.cfg
   ../../component/lxml-python/buildout.cfg
+  ../../component/rsync/buildout.cfg
   ../../stack/flask.cfg
   ../../stack/shacache-client.cfg
+  ../../stack/resilient/buildout.cfg
   ../../stack/slapos.cfg
 
 parts =
+  slapos.cookbook-repository
+  rdiff-backup
   template
-  slapos-cookbook
   eggs
 
+  instance-runner-import
+  instance-runner-export
+
+#  slapos-cookbook
+
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
 output = ${buildout:directory}/template.cfg
 mode = 0644
-md5sum = 5307e4200f044ae57b504ad68444491c
+#md5sum = 5307e4200f044ae57b504ad68444491c
 
 [template-runner]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner.cfg
 output = ${buildout:directory}/template-runner.cfg
-md5sum = 91d6550c43b7a43a999724af4650ae40
+#md5sum = 91d6550c43b7a43a999724af4650ae40
+mode = 0644
+
+[instance-resilient]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
+rendered = ${buildout:directory}/instance-resilient.cfg
+
+context = key buildout buildout:bin-directory
+          key develop_eggs_directory buildout:develop-eggs-directory
+          key eggs_directory buildout:eggs-directory
+
+import-list = file parts template-parts:destination
+              file replicated template-replicated:destination
+
+mode = 0644
+
+[instance-runner-import]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-runner-import.cfg.in
+output = ${buildout:directory}/instance-runner-import.cfg
+mode = 0644
+
+[instance-runner-export]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-runner-export.cfg.in
+output = ${buildout:directory}/instance-runner-export.cfg
 mode = 0644
 
 [eggs]

software/slaprunner/development.cfg

@@ -10,39 +10,43 @@ extends = common.cfg
 
 parts +=
   slapos.cookbook-repository
-  slapos.core-repository
-  slapos.toolbox-repository
-  check-recipe
+
+
+#  slapos.toolbox-repository
+
+#  slapos.core-repository
+#  check-recipe
 
 develop =
   ${:parts-directory}/slapos.cookbook-repository
-  ${:parts-directory}/slapos.core-repository
-  ${:parts-directory}/slapos.toolbox-repository
+#  ${:parts-directory}/slapos.toolbox-repository
+#  ${:parts-directory}/slapos.core-repository
 
 
-[slapos.toolbox-repository]
-recipe = slapos.recipe.build:gitclone
-repository = http://git.erp5.org/repos/slapos.toolbox.git
-branch = slaprunner
-git-executable = ${git:location}/bin/git
+#[slapos.toolbox-repository]
+#recipe = slapos.recipe.build:gitclone
+#repository = http://git.erp5.org/repos/slapos.toolbox.git
+#branch = slaprunner-resiliency
+#git-executable = ${git:location}/bin/git
 
 [slapos.cookbook-repository]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/slapos.git
-branch = slaprunner
-git-executable = ${git:location}/bin/git
-
-[slapos.core-repository]
-recipe = slapos.recipe.build:gitclone
-repository = http://git.erp5.org/repos/slapos.core.git
-branch = master
+branch = slaprunner-resiliency
 git-executable = ${git:location}/bin/git
 
+#[slapos.core-repository]
+#recipe = slapos.recipe.build:gitclone
+#repository = http://git.erp5.org/repos/slapos.core.git
+#branch = master
+#git-executable = ${git:location}/bin/git
+#
 [check-recipe]
 recipe = plone.recipe.command
 stop-on-error = true
 update-command = ${:command}
 command =
   grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
-  grep parts ${buildout:develop-eggs-directory}/slapos.core.egg-link &&
   grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
+
+#  grep parts ${buildout:develop-eggs-directory}/slapos.core.egg-link &&

software/slaprunner/instance-resilient.cfg.jinja2

+# vim: set ft=cfg:
+
+{% import 'parts' as parts %}
+{% import 'replicated' as replicated %}
+
+[buildout]
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
+parts +=
+  {{ parts.replicate("runner", "3") }}
+  publish-connection-informations
+
+{{ replicated.replicate("runner", "3", "runner-export", "runner-import") }}
+
+# Bubble up the parameters
+[request-runner]
+return = url ssh-public-key ssh-url notification-id ip backend_url url cloud9_url ssh_command password_recovery_code
+config = instance-amount debug domain number authorized-key notify ip-list namebase runner1-computer-guid pbs-runner1-computer-guid runner2-computer-guid pbs-runner2-computer-guid runner3-computer-guid pbs-runner3-computer-guid
+# XXX Cedric LN Ugly hack, resilient stack and slaprunner stack sharing too much ssh sections
+config-authorized-key = ${request-pbs-runner-1:connection-ssh-key} ${request-pbs-runner-2:connection-ssh-key} ${slap-parameter:authorized-key}
+config-instance-amount = ${slap-parameter:instance-amount}
+config-debug = ${slap-parameter:debug}
+config-runner1-computer-guid = ${slap-parameter:runner1-computer-guid}
+config-pbs-runner1-computer-guid = ${slap-parameter:pbs-runner1-computer-guid}
+config-runner2-computer-guid = ${slap-parameter:runner2-computer-guid}
+config-pbs-runner2-computer-guid = ${slap-parameter:pbs-runner2-computer-guid}
+config-runner3-computer-guid = ${slap-parameter:runner3-computer-guid}
+config-pbs-runner3-computer-guid = ${slap-parameter:pbs-runner3-computer-guid}
+config-domain = ${slap-parameter:domain}
+
+
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+backend_url = ${request-runner:connection-backend_url}
+url = ${request-runner:connection-url}
+cloud9_url = ${request-runner:connection-cloud9_url}
+ssh_command = ${request-runner:connection-ssh_command}
+password_recovery_code = ${request-runner:connection-password_recovery_code}
+
+[slap-parameter]
+# Default parameters for distributed deployment
+# I.e state "backup1 of maria should go there, ..."
+# XXX-Cedric: Hardcoded number of backups. Should be dynamically generated.
+runner1-computer-guid =
+pbs-runner1-computer-guid =
+runner2-computer-guid =
+pbs-runner2-computer-guid =
+runner3-computer-guid =
+pbs-runner3-computer-guid =
+# XXX-Cedric: Hardcoded parameters. Should be dynamically generated.
+domain =
+authorized-key =
+instance-amount = 10
+debug = false

software/slaprunner/instance-runner-export.cfg.in

+[buildout]
+extends = ${template-runner:output}
+          ${pbsready-export:output}
+
+parts +=
+    urls
+    slaprunner
+    cron-entry-backup
+
+[exporter]
+recipe = slapos.cookbook:slaprunner.export
+wrapper = $${directory:bin}/$${slap-parameter:namebase}-exporter
+srv-directory = $${directory:srv}
+etc-directory = $${directory:etc}
+backup-directory = $${directory:backup}
+shell-binary = ${dash:location}/bin/dash
+rsync-binary = ${rsync:location}/bin/rsync
+
+
+# Extends publish section with resilient parameters
+[urls]
+<= resilient-publish-connection-parameter
+backend_url = $${slaprunner:access-url}
+url =  $${request-frontend:connection-site_url}
+cloud9_url = $${cloud9:access-url}
+ssh_command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port}
+password_recovery_code = $${recovery-code:passwd}

software/slaprunner/instance-runner-import.cfg.in

+[buildout]
+extends = ${template-runner:output}
+          ${pbsready-import:output}
+
+
+parts +=
+    slaprunner
+# have to repeat the next one, as it's not inherited from pbsready-import
+    import-on-notification
+
+[importer]
+recipe = slapos.cookbook:slaprunner.import
+wrapper = $${directory:bin}/$${slap-parameter:namebase}-importer
+srv-directory = $${directory:srv}
+etc-directory = $${directory:etc}
+backup-directory = $${directory:backup}
+shell-binary = ${dash:location}/bin/dash
+rsync-binary = ${rsync:location}/bin/rsync
+curl-binary = ${curl:location}/bin/curl
+backend-url = $${slaprunner:access-url}

software/slaprunner/instance-runner.cfg

@@ -3,7 +3,7 @@ parts =
   cloud9
   slaprunner
   test-runner
-  sshkeys-dropbear
+  sshkeys-dropbear-runner
   dropbear-server-add-authorized-key
   sshkeys-authority
   publish-connection-informations
@@ -13,7 +13,6 @@ parts =
   dropbear-promise
   symlinks
 
-
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
@@ -82,9 +81,9 @@ instance-directory = $${runnerdirectory:instance-root}
 etc_dir = $${directory:etc}
 log_dir =  $${directory:log}
 run_dir = $${directory:run}
-ssh_client = $${sshkeys-dropbear:wrapper}
-public_key = $${sshkeys-dropbear:public-key}
-private_key = $${sshkeys-dropbear:private-key}
+ssh_client = $${sshkeys-dropbear-runner:wrapper}
+public_key = $${sshkeys-dropbear-runner:public-key}
+private_key = $${sshkeys-dropbear-runner:private-key}
 ipv4 = $${slap-network-information:local-ipv4}
 ipv6 = $${slap-network-information:global-ipv6}
 proxy_port = 50000
@@ -121,28 +120,28 @@ keys-directory = $${sshkeys-directory:keys}
 wrapper = $${directory:services}/sshkeys_authority
 keygen-binary = ${dropbear:location}/bin/dropbearkey
 
-[dropbear-server]
+[dropbear-runner-server]
 recipe = slapos.cookbook:dropbear
 host = $${slap-network-information:global-ipv6}
-port = 2222
+port = 22222
 home = $${directory:ssh}
-wrapper = $${directory:bin}/raw_sshd
-shell = /bin/bash
+wrapper = $${directory:bin}/runner_sshd
+shell = ${bash:location}/bin/bash
 rsa-keyfile = $${directory:ssh}/server_key.rsa
 dropbear-binary = ${dropbear:location}/sbin/dropbear
 
-[sshkeys-dropbear]
+[sshkeys-dropbear-runner]
 <= sshkeys-authority
 recipe = slapos.cookbook:sshkeys_authority.request
 name = dropbear
 type = rsa
-executable = $${dropbear-server:wrapper}
-public-key = $${dropbear-server:rsa-keyfile}.pub
-private-key = $${dropbear-server:rsa-keyfile}
-wrapper = $${directory:services}/sshd
+executable = $${dropbear-runner-server:wrapper}
+public-key = $${dropbear-runner-server:rsa-keyfile}.pub
+private-key = $${dropbear-runner-server:rsa-keyfile}
+wrapper = $${directory:services}/runner_sshd
 
 [dropbear-server-add-authorized-key]
-<= dropbear-server
+<= dropbear-runner-server
 recipe = slapos.cookbook:dropbear.add_authorized_key
 key = $${slap-parameter:authorized-key}
 
@@ -166,7 +165,7 @@ recipe = slapos.cookbook:publish
 backend_url = $${slaprunner:access-url}
 url =  $${request-frontend:connection-site_url}
 cloud9_url = $${cloud9:access-url}
-ssh_command = ssh $${dropbear-server:host} -p $${dropbear-server:port}
+ssh_command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port}
 password_recovery_code = $${recovery-code:passwd}
 
 
@@ -194,8 +193,8 @@ curl_path = ${curl:location}/bin/curl
 [dropbear-promise]
 recipe = slapos.cookbook:check_port_listening
 path = $${directory:promises}/dropbear
-hostname = $${dropbear-server:host}
-port = $${dropbear-server:port}
+hostname = $${dropbear-runner-server:host}
+port = $${dropbear-runner-server:port}
 
 [symlinks]
 recipe = cns.recipe.symlink

software/slaprunner/instance.cfg

@@ -9,3 +9,10 @@ offline = true
 [switch_softwaretype]
 recipe = slapos.cookbook:softwaretype
 default = ${template-runner:output}
+resilient = ${instance-resilient:rendered}
+runner = ${template-runner:output}
+runner-import = ${instance-runner-import:output}
+runner-export = ${instance-runner-export:output}
+
+frozen = ${instance-frozen:output}
+pull-backup = ${template-pull-backup:output}

software/slaprunner/software.cfg

@@ -5,7 +5,7 @@
 # 2/ Define list of trusted certificates for the cache.
 
 [buildout]
-extends = common.cfg
+extends = development.cfg
 
 [networkcache]
 # signature certificates of the following uploaders.
@@ -56,12 +56,12 @@ netaddr = 0.7.10
 plone.recipe.command = 1.1
 pycrypto = 2.6
 pytz = 2012j
-slapos.cookbook = 0.71.1
-slapos.core = 0.34
-slapos.libnetworkcache = 0.13.3
+#slapos.cookbook = 0.71.1
+slapos.core = 0.35.1
+slapos.libnetworkcache = 0.13.4
 slapos.recipe.build = 0.11.5
 slapos.recipe.template = 2.4.2
-slapos.toolbox = 0.34.0
+#slapos.toolbox = 0.34.0
 smmap = 0.8.2
 xml-marshaller = 0.9.7
 z3c.recipe.scripts = 1.0.1

stack/resilient/README.txt

@@ -127,6 +127,27 @@ wrapper. I suggest you only add options and specify your export/import recipe.
 
 
 
+
+Checking that it works
+----------------------
+
+To check that your software instance is resilient you can proceed this way:
+Once all instances are successfully deployed, go to your export instance, connect as the instance user and run:
+$ ~/bin/exporter
+It is the script responsible for triggering the resiliency stack on your instance. After doing a backup of your data, it will notify the pull-backup instances of a new backup, triggering the transfer of this data to the import instances.
+
+Once this script is run successfully, go to your import instance, connect as its instance user and check ~/srv/backup/"your sofwtare"/, the location of the data you wanted to receive. The last part of the resiliency is up to your import script.
+
+DEBUGGING:
+Here is a partial list of things you can check to understand what is causing the problem:
+
+- Check that your import script does not fail and successfully places your data in ~/srv/backup/"your software" (as the import instance user) by runnig:
+$ ~/bin/"your software"-exporter
+- Check the export instance script is run successfully as this instance user by running:
+$ ~/bin/exporter
+- Check the pull-instance system did its job by going to one of your pull-backup instance, connect as its user and check the log : ~/var/log/equeue.log
+
+
 -----------------------------------------------------------------------------------------
 
 Finally, instance-mysoftware-import.cfg.in and

stack/resilient/buildout.cfg

 [buildout]
+extends =
+  ../../component/gzip/buildout.cfg
+  ../../component/rdiff-backup/buildout.cfg
 
 parts =
+  rdiff-backup
   pbsready
   pbsready-import
   pbsready-export
@@ -9,6 +13,9 @@ parts =
   instance-frozen
   template-resilient
 
+  # needed tools for resiliency
+  gzip
+
 
 #----------------
 #--