Merge branch 'master' into erp5-component

CHANGES.txt

 Changes
 =======
 
+0.84.2 (2013-10-04)
+-------------------
+
+ * sshkeys_authority: don't allow to return None as parameter. [9e340a0]
+
 0.84.1 (2013-10-03)
 -------------------
 

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '0.84.1'
+version = '0.84.2'
 name = 'slapos.cookbook'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"

slapos/recipe/kvm/template/kvm_controller_run.in

@@ -6,6 +6,8 @@
 import socket
 import time
 
+# XXX: to be factored with slapos.toolbox qemu qmp wrapper.
+
 socket_path = '%(socket-path)s'
 vnc_password = '%(vnc-passwd)s'
 

software/kvm/common.cfg

@@ -21,6 +21,7 @@ parts =
 
 # XXX: we have to manually add this for resilience
   rdiff-backup
+  collective.recipe.template-egg
 
 #XXX-Cedric : Currently, one can only access to KVM using noVNC.
 #             Ideally one should be able to access KVM by using either NoVNC or VNC.
@@ -79,7 +80,7 @@ command =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-#md5sum = bdd0495ef729e7272ec9c97aca919c09
+md5sum = 8617a8cc345a55688c5449528daef4d1
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
@@ -94,14 +95,14 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
 mode = 644
-#md5sum = 6753004b582c0470bd028253ce1964ad
+md5sum = 45a846378215eded6c001d0dd729a1ec
 download-only = true
 on-update = true
 
 [template-kvm-resilient-test]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-resilient-test.cfg.jinja2
-#md5sum = 027d68d9decbc6aec59365fa723975d7
+md5sum = b58427f93d5fcca94bdc90661fe6080b
 mode = 0644
 download-only = true
 on-update = true
@@ -132,7 +133,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/kvm-export.sh.in
 filename = kvm-export.sh.in
-md5sum = 3e878b3343c76f0d6950986fffcb6a8c
+md5sum = bf03a90f6960b37cba812ee936a13342
 download-only = true
 mode = 0755
 

software/kvm/instance-kvm-resilient-input-schema.json

@@ -29,5 +29,11 @@
       "description": "Periodicity of backup, in cron format.",
       "type": "string"
     }
+    "remove-backup-older-than": {
+      "title": "Remove backups older than...",
+      "description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible."
+      "type": "string",
+      "default": "3B"
+    }
   }
 }

software/kvm/instance-kvm-resilient-test.cfg.jinja2

@@ -50,12 +50,10 @@ config-{{ key }} = {{ dumps(value) }}
 config-virtual-hard-drive-url = ${slap-parameter:virtual-hard-drive-url}
 config-virtual-hard-drive-md5sum = ${slap-parameter:virtual-hard-drive-md5sum}
 config-resiliency-backup-periodicity = */5
-# We don't use url parameter, but we want it to be there to make sure root instance is ready.
-return = url
 # XXX What to do?
 sla = computer_guid
 sla-computer_guid = ${slap-connection:computer-id}
 
 [slap-parameter]
 virtual-hard-drive-url = https://softinst43236.host.vifib.net/data/public/8e2138.php?dl=true
-virtual-hard-drive-md5sum = de0f10c7c6538e9928879332afd9be7a
+virtual-hard-drive-md5sum = 465e1024447997e7b86ee2e5151e031b

software/kvm/instance-kvm-resilient.cfg.jinja2

@@ -12,6 +12,8 @@ offline = true
 parts +=
   {{ parts.replicate("kvm", "3") }}
   publish-connection-informations
+  kvm-frontend-url-promise
+  kvm-backend-url-promise
 
 {{ replicated.replicate("kvm", "3", "kvm-export", "kvm-import", slapparameter_dict=slapparameter_dict) }}
 
@@ -29,3 +31,22 @@ recipe = slapos.cookbook:publish
 backend-url = ${request-kvm:connection-backend-url}
 url = ${request-kvm:connection-url}
 ipv6 = ${request-kvm:connection-ipv6}
+
+[kvm-frontend-url-promise]
+# Check that url parameter is complete
+recipe = collective.recipe.template
+input = inline:#!/bin/sh
+  URL="${request-kvm:connection-url}"
+  if [[ ! "$URL" == https://* ]]; then
+    exit 1
+  fi
+output = ${resilient-directory:promise}/kvm-frontend-url
+mode = 700
+
+[kvm-backend-url-promise]
+# Check that backend url is reachable
+recipe = slapos.cookbook:check_url_available
+path = ${directory:promises}/frontend_promise
+url = ${publish-connection-information:url}
+dash_path = /bin/sh
+curl_path = {{ curl_executable_location }}

software/kvm/instance.cfg.in

@@ -49,10 +49,10 @@ mode = 0644
 recipe = slapos.recipe.template:jinja2
 template = ${template-kvm-resilient-test:location}/instance-kvm-resilient-test.cfg.jinja2
 rendered = $${buildout:directory}/template-kvm-resilient-test.cfg
-bin-directory = ${buildout:bin-directory}
 context =
-    key bin_directory dynamic-template-kvm-resilient-test:bin-directory
     key develop_eggs_directory buildout:develop-eggs-directory
     key eggs_directory buildout:eggs-directory
     key slapparameter_dict slap-configuration:configuration
+    raw bin_directory ${buildout:bin-directory}
+    raw curl-executable-location ${curl:location}/bin/curl
 mode = 0644

software/kvm/software.cfg

@@ -122,34 +122,35 @@ Werkzeug = 0.9.4
 apache-libcloud = 0.13.2
 async = 0.6.1
 buildout-versions = 1.7
+collective.recipe.template = 1.10
 erp5.util = 0.4.36
 gitdb = 0.5.4
 itsdangerous = 0.23
 lxml = 3.2.3
 meld3 = 0.6.10
 plone.recipe.command = 1.1
-psutil = 1.1.0
+psutil = 1.1.1
 pycrypto = 2.6
 rdiff-backup = 1.0.5
-slapos.cookbook = 0.84.1
+slapos.cookbook = 0.84.2
 slapos.recipe.cmmi = 0.2
 slapos.recipe.download = 1.0.dev-r4053
-slapos.toolbox = 0.37.1
+slapos.toolbox = 0.37.2
 smmap = 0.8.2
 websockify = 0.5.1
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
 # slapos.core==0.35.1
-# slapos.toolbox==0.37.1
+# slapos.toolbox==0.37.2
 Flask = 0.10.1
 
 # Required by:
-# slapos.toolbox==0.37.1
+# slapos.toolbox==0.37.2
 GitPython = 0.3.2.RC1
 
 # Required by:
-# slapos.toolbox==0.37.1
+# slapos.toolbox==0.37.2
 atomize = 0.1.1
 
 # Required by:
@@ -157,19 +158,19 @@ atomize = 0.1.1
 ecdsa = 0.9
 
 # Required by:
-# slapos.toolbox==0.37.1
+# slapos.toolbox==0.37.2
 feedparser = 5.1.3
 
 # Required by:
-# slapos.cookbook==0.84.1
+# slapos.cookbook==0.84.2
 inotifyx = 0.2.0-1
 
 # Required by:
-# slapos.cookbook==0.84.1
+# slapos.cookbook==0.84.2
 lock-file = 2.0
 
 # Required by:
-# slapos.cookbook==0.84.1
+# slapos.cookbook==0.84.2
 netaddr = 0.7.10
 
 # Required by:
@@ -180,9 +181,8 @@ netifaces = 0.8-1
 # websockify==0.5.1
 numpy = 1.7.1
 
-
 # Required by:
-# slapos.toolbox==0.37.1
+# slapos.toolbox==0.37.2
 paramiko = 1.12.0
 
 # Required by:
@@ -190,12 +190,12 @@ paramiko = 1.12.0
 pyflakes = 0.7.3
 
 # Required by:
-# slapos.cookbook==0.84.1
+# slapos.cookbook==0.84.2
 pytz = 2013.7
 
 # Required by:
-# slapos.cookbook==0.84.1
-# slapos.toolbox==0.37.1
+# slapos.cookbook==0.84.2
+# slapos.toolbox==0.37.2
 slapos.core = 0.35.1
 
 # Required by:
@@ -207,11 +207,10 @@ supervisor = 3.0
 unittest2 = 0.5.1
 
 # Required by:
-# slapos.cookbook==0.84.1
-# slapos.toolbox==0.37.1
+# slapos.cookbook==0.84.2
+# slapos.toolbox==0.37.2
 xml-marshaller = 0.9.7
 
 # Required by:
 # slapos.core==0.35.1
 zope.interface = 4.0.5
-

software/kvm/template/kvm-export.sh.in

@@ -4,13 +4,17 @@ QEMU_IMG=${kvm-instance:qemu-img-path}
 SNAPSHOT_NAME=$(date +%s)
 DISK_PATH=${kvm-instance:disk-path}
 BACKUP_PATH=${:backup-disk-path}
+QMP_CLIENT=${buildout:directory}/software_release/bin/qemu-qmp-client
 
 if [ ! -f $DISK_PATH ]; then
   echo "Nothing to backup, disk image doesn't exist yet."
   exit 0;
 fi
 
+$QMP_CLIENT ${kvm-instance:socket-path} suspend && \
 $QEMU_IMG snapshot -c $SNAPSHOT_NAME $DISK_PATH
+$QMP_CLIENT ${kvm-instance:socket-path} resume
+
 if [ -f $BACKUP_PATH ]; then
   rm $BACKUP_PATH
 fi

software/slaprunner/common.cfg

@@ -15,7 +15,6 @@ extends =
   ../../stack/slapos.cfg
 
 parts =
-  rdiff-backup
   template
   eggs
   nginx
@@ -27,6 +26,10 @@ parts =
   instance-runner-export
   slapos-cookbook
 
+# XXX: we have to manually add this for resilience
+  rdiff-backup
+  collective.recipe.template-egg
+
 ####################
 ## Node JS proxy
 ####################

stack/resilient/buildout.cfg

 [buildout]
 extends =
+  ../../component/dash/buildout.cfg
   ../../component/dropbear/buildout.cfg
   ../../component/gzip/buildout.cfg
   ../../component/rdiff-backup/buildout.cfg
   ../../component/rsync/buildout.cfg
 
 parts =
-  rdiff-backup
+  collective.recipe.template-egg
   pbsready
   pbsready-import
   pbsready-export
@@ -16,7 +17,12 @@ parts =
 
   # needed tools for resiliency
   gzip
+  rdiff-backup
+  dash
 
+[collective.recipe.template-egg]
+recipe = zc.recipe.egg
+eggs = collective.recipe.template
 
 #----------------
 #--
@@ -30,7 +36,7 @@ parts =
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready.cfg.in
 output = ${buildout:directory}/pbsready.cfg
-md5sum = 570e0b54c97d510befa2ea981c1e90e0
+#md5sum = 46f9d33e642467a72c599c8dc767e6c3
 mode = 0644
 
 [pbsready-import]
@@ -39,7 +45,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready-import.cfg.in
 output = ${buildout:directory}/pbsready-import.cfg
-md5sum = cc9c776500ccd07cb51969beb68ffcda
+md5sum = cb562bd954b9e809c8748d0f96de4116
 mode = 0644
 
 [pbsready-export]
@@ -48,7 +54,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready-export.cfg.in
 output = ${buildout:directory}/pbsready-export.cfg
-md5sum = 25d05b3929fb4c6cf275866bad678d6a
+md5sum = 8fb619622a08aff2321497895e04df16
 mode = 0644
 
 [template-pull-backup]
@@ -61,14 +67,14 @@ mode = 0644
 [template-replicated]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/template-replicated.cfg.in
-md5sum = c762a625f65193bc8a570b4d56a0d08c
+md5sum = 9e236726678d89a5359e1571a91e59e8
 mode = 0644
 destination = ${buildout:directory}/template-replicated.cfg.in
 
 [template-parts]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/template-parts.cfg.in
-md5sum = c942f82552fcb42fc74a5f896e0cd5f3
+md5sum = a3f55a20881c3f1ec4416662146c06f7
 mode = 0644
 destination = ${buildout:directory}/template-parts.cfg.in
 

stack/resilient/pbsready-export.cfg.in

@@ -14,6 +14,7 @@ parts =
   sshkeys-authority
   dropbear-server
   sshkeys-dropbear
+  resilient-sshkeys-dropbear-promise
   dropbear-server-pbs-authorized-key
   notifier
 

stack/resilient/pbsready-import.cfg.in

@@ -14,6 +14,7 @@ parts =
   sshkeys-authority
   dropbear-server
   sshkeys-dropbear
+  resilient-sshkeys-dropbear-promise
   dropbear-server-pbs-authorized-key
   notifier
 

stack/resilient/pbsready.cfg.in

@@ -10,6 +10,7 @@ parts =
   sshkeys-authority
   dropbear-server
   sshkeys-dropbear
+  resilient-sshkeys-dropbear-promise
   dropbear-server-pbs-authorized-key
   notifier
 
@@ -179,6 +180,27 @@ server-binary = ${buildout:bin-directory}/pubsubserver
 notifier-binary = ${buildout:bin-directory}/pubsubnotifier
 
 
+#----------------
+#--
+#-- Dropbear.
+
+[dropbear-server]
+recipe = slapos.cookbook:dropbear
+host = $${slap-network-information:global-ipv6}
+# Explicitely excludes to define "port" argument. It will be defined in
+# pbs-ready-import.cfg.in and pbs-ready-export.cfg.in
+home = $${directory:ssh}
+wrapper = $${rootdirectory:bin}/raw_sshd
+shell = $${rdiff-backup-server:wrapper}
+rsa-keyfile = $${directory:ssh}/server_key.rsa
+dropbear-binary = ${dropbear:location}/sbin/dropbear
+
+[dropbear-server-pbs-authorized-key]
+<= dropbear-server
+recipe = slapos.cookbook:dropbear.add_authorized_key
+key = $${slap-parameter:authorized-key}
+
+
 #----------------
 #--
 #-- sshkeys
@@ -205,31 +227,21 @@ public-key = $${dropbear-server:rsa-keyfile}.pub
 private-key = $${dropbear-server:rsa-keyfile}
 wrapper = $${basedirectory:services}/sshd
 
-
-#----------------
-#--
-#-- Dropbear.
-
-[dropbear-server]
-recipe = slapos.cookbook:dropbear
-host = $${slap-network-information:global-ipv6}
-# Explicitely excludes to define "port" argument. It will be defined in
-# pbs-ready-import.cfg.in and pbs-ready-export.cfg.in
-home = $${directory:ssh}
-wrapper = $${rootdirectory:bin}/raw_sshd
-shell = $${rdiff-backup-server:wrapper}
-rsa-keyfile = $${directory:ssh}/server_key.rsa
-dropbear-binary = ${dropbear:location}/sbin/dropbear
-
-[dropbear-server-pbs-authorized-key]
-<= dropbear-server
-recipe = slapos.cookbook:dropbear.add_authorized_key
-key = $${slap-parameter:authorized-key}
+[resilient-sshkeys-dropbear-promise]
+# Check that public key file exists and is not empty
+recipe = collective.recipe.template
+input = inline:#!${dash:location}/bin/dash
+  PUBLIC_KEY_CONTENT="$${sshkeys-dropbear:public-key-value}"
+  if [ ! -n "$PUBLIC_KEY_CONTENT" ]; then
+    exit 1
+  fi
+output = $${basedirectory:promises}/public-key-existence
+mode = 700
 
 
 #----------------
 #--
-#-- Conncetion informations to re-use.
+#-- Connection informations to re-use.
 # XXX-Cedric: when "aggregation" system is done in libslap, directly publish.
 [resilient-publish-connection-parameter]
 recipe = slapos.cookbook:publish

stack/resilient/template-parts.cfg.in

@@ -4,18 +4,21 @@
 
   request-{{namebase}}
   request-{{namebase}}-2
+  resilient-request-{{namebase}}-public-key-promise
   
-  {% for i in range(1,nbbackup|int) %}
-  request-{{namebase}}-pseudo-replicating-{{i}}
-  request-{{namebase}}-pseudo-replicating-{{i}}-2
+  {% for id in range(1,nbbackup|int) %}
+  request-{{namebase}}-pseudo-replicating-{{id}}
+  request-{{namebase}}-pseudo-replicating-{{id}}-2
+  resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key-promise
   {% endfor %}
   
-  {% for i in range(1,nbbackup|int) %}
-  request-pbs-{{namebase}}-{{i}}
-  request-pull-backup-server-{{namebase}}-{{i}}
-  request-pull-backup-server-{{namebase}}-backup-{{i}}
+  {% for id in range(1,nbbackup|int) %}
+  request-pbs-{{namebase}}-{{id}}
+  resilient-request-pbs-{{namebase}}-{{id}}-public-key-promise
+  request-pull-backup-server-{{namebase}}-{{id}}
+  request-pull-backup-server-{{namebase}}-backup-{{id}}
   {% endfor %}
-  
+
 
 
   {% endmacro %}

stack/resilient/template-replicated.cfg.in

@@ -11,6 +11,13 @@
 {% endif -%}
 
 
+[resilient-directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc
+promise = ${:etc}/promise
+
+
 ## Tells the Backupable recipe that we want a backup
 [resilient]
 recipe = slapos.cookbook:request
@@ -28,7 +35,6 @@ software-url = ${slap-connection:software-release-url}
 software-type = {{typeexport}}
 name = {{namebase}}0
 return = ssh-public-key ssh-url notification-id ip
-
 config =
 # Resilient related parameters
   number authorized-key notify ip-list namebase
@@ -66,6 +72,7 @@ sla-{{ key }} = {{ value }}
 {%   endif -%}
 {% endif -%}
 
+
 {% for id in range(1,nbbackup|int) %}
 
 [request-{{namebase}}-pseudo-replicating-{{id}}]
@@ -107,9 +114,9 @@ sla-{{ key }} = {{ value }}
 {%   endif %}
 {% endif %}
 
-
 {% endfor -%}
 
+
 [iplist]
 config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbbackup|int) %} ${request-{{namebase}}-pseudo-replicating-{{j}}:connection-ip}{% endfor %}
 
@@ -117,11 +124,37 @@ config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbback
 <= request-{{namebase}}
    iplist
 
+[resilient-request-{{namebase}}-public-key-promise]
+# Check that public-key-value parameter exists and is not empty
+# XXX: maybe we should consider empty values to be non-nexistent.
+recipe = collective.recipe.template
+# XXX: don't use system executable
+input = inline:#!/bin/sh
+  PUBLIC_KEY_CONTENT="${request-{{namebase}}-2:connection-ssh-public-key})"
+  if [[ ! -n "$PUBLIC_KEY_CONTENT" -o "$PUBLIC_KEY_CONTENT" == None ]]; then
+    exit 1
+  fi
+output = ${resilient-directory:promise}/resilient-request-{{namebase}}-public-key
+mode = 700
+
 {% for id in range(1,nbbackup|int) %}
 [request-{{namebase}}-pseudo-replicating-{{id}}-2]
 <= request-{{namebase}}-pseudo-replicating-{{id}}
    iplist
 
+[resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key-promise]
+# Check that public-key-value parameter exists and is not empty
+# XXX: maybe we should consider empty values to be non-nexistent.
+recipe = collective.recipe.template
+# XXX: don't use system executable
+input = inline:#!/bin/sh
+  PUBLIC_KEY_CONTENT="${request-{{namebase}}-pseudo-replicating-{{id}}-2:connection-ssh-public-key})"
+  if [ ! -n "$PUBLIC_KEY_CONTENT" -a  "$PUBLIC_KEY_CONTENT" == None ]; then
+    exit 1
+  fi
+output = ${resilient-directory:promise}/resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key
+mode = 700
+
 {% endfor %}
 
 
@@ -167,6 +200,19 @@ sla-{{ key }} = {{ value }}
 {%   endif %}
 {% endif %}
 
+[resilient-request-pbs-{{namebase}}-{{id}}-public-key-promise]
+# Check that public-key-value parameter exists and is not empty
+# XXX: maybe we should consider empty values to be non-nexistent.
+recipe = collective.recipe.template
+# XXX: don't use system executable
+input = inline:#!/bin/sh
+  PUBLIC_KEY_CONTENT="${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}:connection-ssh-key})"
+  if [ ! -n "$PUBLIC_KEY_CONTENT" -a  "$PUBLIC_KEY_CONTENT" == None ]; then
+    exit 1
+  fi
+output = ${resilient-directory:promise}/resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key
+mode = 700
+
 
 [request-pull-backup-server-{{namebase}}-{{id}}]
 <= request-pbs-common