Merge remote-tracking branch 'origin/master' into erp5-cluster

Conflicts:
	stack/erp5/instance-tidstorage.cfg.in

.gitignore

@@ -9,3 +9,5 @@ eggs/
 parts/
 slapos.cookbook.egg-info
 .*.swp
+*~
+\#*\#
\ No newline at end of file

CHANGES.txt

 Changes
 =======
 
-0.78.1 (2013-05-31)
+0.78.5 (2013-08-06)
+-----------------------
+
+ * check_url_available: add option to check secure links [6cbce4d8231]
+
+0.78.4 (2013-08-06)
+-----------------------
+
+ * slapos.cookbook:slaprunner: Update to use https. [Cedric Le Ninivin]
+
+
+0.78.3 (2013-07-18)
+-------------------
+
+ * slapos.cookbook:publish: Add support to publish information for slaves. [Cedric Le Ninivin]
+
+0.78.2 (2013-07-18)
 -------------------
 
  * Fix slapos.cookbook:request: Add backward compatiblity about getInstanceGuid(). [Cedric de Saint Martin]
  * slapos.cookbook:check_* promises: Add timeout to curl that is not otherwise killed by slapos promise subsystem. [Cedric de Saint Martin]
+ * Cloudooo: Allow any environment variables. [Yusei Tahara]
+ * ERP5: disable MariaDB query cache completely by 'query_cache_type = 0' for ERP5. [Kazuhiko Shiozaki]
+ * ERP5: enable haproxy admin socket and install haproxyctl script. [Kazuhiko Shiozaki]
+ * ERP5: increase the maximum number of open file descriptors before starting mysqld. [Kazuhiko Shiozaki]
+ * python 2.7: updated to 2.7.5 [Cedric de Saint Martin]
+
+0.78.1 (2013-05-31)
+-------------------
+
  * Add boinc recipe: Allow to deploy an empty BOINC project. [Alain Takoudjou]
  * Add boinc.app recipe: Allow to deploy and update a BOINC application into existing BOINC server instance . [Alain Takoudjou]
  * Add boinc.client recipe: Allow to deploy a BOINC Client instance on SlapOS. [Alain Takoudjou]
@@ -15,10 +40,6 @@ Changes
  * Add trac recipe: for deploying Trac and manage project with support of SVN and GIT. [Alain Takoudjou]
  * Add bonjourgrid recipe: for deploying BonjourGrid Master and submit BOINC or Condor project. [Alain Takoudjou]
  * Add bonjourgrid.client recipe: for deploying BonjourGrid Worker instance and execute BOINC or Condor Jobs. [Alain Takoudjou]
- * Cloudooo: Allow any environment variables. [Yusei Tahara]
- * ERP5: disable MariaDB query cache completely by 'query_cache_type = 0' for ERP5. [Kazuhiko Shiozaki]
- * ERP5: enable haproxy admin socket and install haproxyctl script. [Kazuhiko Shiozaki]
- * ERP5: increase the maximum number of open file descriptors before starting mysqld. [Kazuhiko Shiozaki]
 
 0.78.0 (2013-04-28)
 -------------------

component/apache/buildout.cfg

@@ -120,8 +120,9 @@ make-targets =
 [apache-2.2]
 # inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/
 recipe = slapos.recipe.cmmi
-url = http://mir2.ovh.net/ftp.apache.org/dist//httpd/httpd-2.2.24.tar.gz
-md5sum = 64a3392018ad60583209a16d728180d3
+version = 2.2.25
+url = http://mir2.ovh.net/ftp.apache.org/dist/httpd/httpd-${:version}.tar.bz2
+md5sum = 9ebe3070c0bb4311f21a0cd0e34f0045
 patch-options = -p1
 configure-options = --disable-static
                     --enable-authn-alias
@@ -182,6 +183,8 @@ environment =
 recipe = slapos.recipe.cmmi
 url = http://sourceforge.net/projects/mod-antiloris/files/mod_antiloris-0.4.tar.bz2/download
 md5sum = 66862bf10e9be3a023e475604a28a0b4
+depends =
+  ${apache-2.2:version}
 configure-command = ${apache-2.2:location}/bin/apxs
 configure-options = -c mod_antiloris.c
 make-binary = ${:configure-command}

component/cloud9/buildout.cfg

@@ -44,6 +44,13 @@ filename = cloud9-session-directory.patch
 download-only = true
 md5sum = 5dc8cc28447ed3747b8a53c768d872aa
 
+[cloud9-socket.patch]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/${:filename}
+filename = cloud9-socket.patch
+download-only = true
+#md5sum = 5dc8cc28447ed3747b8a53c768d872aa
+
 [cloud9-git]
 # Online IDE written in javascript/node.js
 # URL : c9.io
@@ -55,7 +62,7 @@ commit = f7d102bc225c922f116d2cea52a746d64343ea59
 repository = https://github.com/ajaxorg/cloud9.git
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 environment = export GIT_SSL_NO_VERIFY=true; export PATH=${git:location}/bin:${nodejs:location}/bin:${node-sm:location}/node_modules/sm/bin:$PATH; export CPPFLAGS="-I${libxml2:location}/include -I${nodejs:location}/include"; export LDFLAGS="-L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib"; export HOME=${:location};
-command = ${:environment} (git clone --quiet ${:repository} ${:location} && cd ${:location} && git reset --hard ${:commit} && ${node-sm:location}/node_modules/.bin/sm install && patch -p1 < ${cloud9-session-directory.patch:location}/${cloud9-session-directory.patch:filename}) || (rm -fr ${:location}; exit 1)
+command = ${:environment} (git clone --quiet ${:repository} ${:location} && cd ${:location} && git reset --hard ${:commit} && ${node-sm:location}/node_modules/.bin/sm install && patch -p1 < ${cloud9-session-directory.patch:location}/${cloud9-session-directory.patch:filename} && ${node-sm:location}/node_modules/.bin/sm install && patch -p1 < ${cloud9-socket.patch:location}/${cloud9-socket.patch:filename}) || (rm -fr ${:location}; exit 1)
 update-command =
 executable = ${:location}/server.js
 

component/cloud9/cloud9-socket.patch

+diff --git a/node_modules/smith.io/node_modules/engine.io/node_modules/engine.io-client/dist/engine.io-dev.js b/node_modules/smith.io/node_modules/engine.io/node_modules/engine.io-client/dist/engine.io-dev.js
+index fa7e54a..14b8e67 100644
+--- a/node_modules/smith.io/node_modules/engine.io/node_modules/engine.io-client/dist/engine.io-dev.js
++++ b/node_modules/smith.io/node_modules/engine.io/node_modules/engine.io-client/dist/engine.io-dev.js
+@@ -2126,7 +2126,7 @@ Polling.prototype.uri = function () {
+     query = '?' + query;
+   }
+ 
+-  return schema + '://' + this.host + port + this.path + query;
++  return this.path + query;
+ };
+ 
+ });require.register("transports/websocket.js", function(module, exports, require, global){

component/coreutils/buildout.cfg

@@ -2,7 +2,7 @@
 extends =
   ../xz-utils/buildout.cfg
 parts =
-  coreutils
+  coreutils-output
 
 [coreutils]
 recipe = slapos.recipe.cmmi
@@ -13,3 +13,13 @@ configure-options =
 environment =
   PATH=${xz-utils:location}/bin:%(PATH)s
   LDFLAGS =-Wl,--as-needed
+
+[coreutils-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${:test} -x ${:test} -a -x ${:cat} -a -x ${:rm}
+test = ${coreutils:location}/bin/test
+cat = ${coreutils:location}/bin/cat
+rm = ${coreutils:location}/bin/rm

component/dash/buildout.cfg

 [buildout]
+extends = 
+  ../coreutils/buildout.cfg
 
-parts = dash
+parts = dash-output
 
 [dash]
 recipe = slapos.recipe.cmmi
@@ -10,3 +12,11 @@ configure-options =
   --disable-static
   --disable-fnmatch
   --disable-glob
+
+[dash-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${coreutils-output:test} -x ${:dash}
+dash = ${dash:location}/bin/dash

component/dcron/buildout.cfg

 [buildout]
-parts = dcron
+extends = 
+  ../coreutils/buildout.cfg
+
+parts = dcron-output
+
+[dcron-hooks-download]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/${:filename}
+md5sum = 860e914dff4108b47565965fe5ebc7b5
+download-only = true
+filename = dcron-hooks.py
 
 [dcron-patch-nonroot]
 recipe = hexagonit.recipe.download
-md5sum = 2f5b22dc1cbe81060a9c28e6f5c06e8b
+md5sum = d5408ab682b65cc1eda40d588fcd7db8
 url = ${:_profile_base_location_}/${:filename}
 filename = dcron-4.4.noroot.no.globals.patch
 download-only = true
@@ -18,3 +28,13 @@ patches =
 patch-options = -p1
 make-options =
   PREFIX=${buildout:parts-directory}/${:_buildout_section_name_}
+post-make-hook = ${dcron-hooks-download:location}/${dcron-hooks-download:filename}:post_make_hook
+
+[dcron-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${coreutils-output:test} -x ${:crond} -a -x ${:crontab} -a ! -u ${:crontab}
+crond = ${dcron:location}/sbin/crond
+crontab = ${dcron:location}/bin/crontab

component/dcron/dcron-4.4.noroot.no.globals.patch

-# Patch for making dcron usable without root user, as a local service
 diff -ru dcron-4.4.org/chuser.c dcron-4.4/chuser.c
 --- dcron-4.4.org/chuser.c	2010-01-18 16:27:31.000000000 +0100
-+++ dcron-4.4/chuser.c	2011-04-01 11:19:19.000000000 +0200
++++ dcron-4.4/chuser.c	2013-07-18 18:17:16.342147418 +0200
 @@ -14,47 +14,6 @@
  int
  ChangeUser(const char *user, char *dochdir)
@@ -53,7 +52,16 @@ diff -ru dcron-4.4.org/chuser.c dcron-4.4/chuser.c
  
 diff -ru dcron-4.4.org/crontab.c dcron-4.4/crontab.c
 --- dcron-4.4.org/crontab.c	2010-01-18 16:27:31.000000000 +0100
-+++ dcron-4.4/crontab.c	2011-04-01 11:19:19.000000000 +0200
++++ dcron-4.4/crontab.c	2013-07-18 18:18:07.768535485 +0200
+@@ -88,7 +88,7 @@
+ 				break;
+ 			case 'c':
+ 				/* getopt guarantees optarg != 0 here */
+-				if (*optarg != 0 && getuid() == geteuid()) {
++				if (*optarg != 0) {
+ 					CDir = optarg;
+ 				} else {
+ 					printlogf(0, "-c option: superuser only");
 @@ -316,9 +316,6 @@
  
  	close(filedes[0]);
@@ -75,7 +83,7 @@ diff -ru dcron-4.4.org/crontab.c dcron-4.4/crontab.c
  				ptr = PATH_VI;
 diff -ru dcron-4.4.org/job.c dcron-4.4/job.c
 --- dcron-4.4.org/job.c	2010-01-18 16:27:31.000000000 +0100
-+++ dcron-4.4/job.c	2011-04-01 11:19:19.000000000 +0200
++++ dcron-4.4/job.c	2013-07-18 18:17:16.342147418 +0200
 @@ -62,14 +62,6 @@
  		 * Change running state to the user in question
  		 */
@@ -108,7 +116,7 @@ diff -ru dcron-4.4.org/job.c dcron-4.4/job.c
  		/*
 diff -ru dcron-4.4.org/Makefile dcron-4.4/Makefile
 --- dcron-4.4.org/Makefile	2010-01-18 16:27:31.000000000 +0100
-+++ dcron-4.4/Makefile	2011-04-01 11:19:35.000000000 +0200
++++ dcron-4.4/Makefile	2013-07-18 18:17:16.342147418 +0200
 @@ -3,7 +3,6 @@
  
  # these variables can be configured by e.g. `make SCRONTABS=/different/path`

component/dcron/dcron-hooks.py

+import os
+import shutil
+def post_make_hook(options, buildout):
+  crontab_path = os.path.join(options['location'], 'bin', 'crontab')
+  os.chmod(crontab_path, 0750)

component/dropbear/buildout.cfg

@@ -7,9 +7,10 @@
 [buildout]
 extends = 
   ../zlib/buildout.cfg
+  ../coreutils/buildout.cfg
 
 parts =
-  dropbear
+  dropbear-output
 
 [dropbear-userspace-patch]
 recipe = hexagonit.recipe.download
@@ -63,3 +64,12 @@ patches=
 
 patch-options=
   -p1
+
+[dropbear-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${coreutils-output:test} -x ${:ssh} -a -x ${:keygen}
+ssh = ${dropbear:location}/bin/dbclient
+keygen = ${dropbear:location}/bin/dropbearkey

component/ffmpeg/buildout.cfg

@@ -2,6 +2,7 @@
 extends = 
   ../bzip2/buildout.cfg
   ../libpng/buildout.cfg
+  ../patch/buildout.cfg
   ../pkgconfig/buildout.cfg
   ../zlib/buildout.cfg
 
@@ -28,16 +29,25 @@ environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${libogg:location}/lib/pkgconfig
 
+[libtheora-png_sizeof.patch]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/${:filename}
+filename = libtheora-png_sizeof.patch
+md5sum = eaa1454081b50f05b59495a12f52b0d5
+download-only = true
+
 [libtheora]
 recipe = slapos.recipe.cmmi
 url = http://downloads.xiph.org/releases/theora/libtheora-1.1.1.tar.bz2
 md5sum = 292ab65cedd5021d6b7ddd117e07cd8e
 depends =
   ${libpng:so_version}
+patches = ${libtheora-png_sizeof.patch:location}/${libtheora-png_sizeof.patch:filename}
+patch-options = -p1
 configure-options =
   --disable-static
 environment = 
-  PATH=${pkgconfig:location}/bin:%(PATH)s
+  PATH=${patch:location}/bin:${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${libogg:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:${libvorbis:location}/lib/pkgconfig
 
 [yasm]

component/ffmpeg/libtheora-png_sizeof.patch

+--- libtheora-1.1.1/examples/png2theora.c.orig        2009-08-23 03:14:04.000000000 +0900
++++ libtheora-1.1.1/examples/png2theora.c     2013-07-16 12:40:07.629087870 +0900
+@@ -462,9 +462,9 @@
+   png_set_strip_alpha(png_ptr);
+ 
+   row_data = (png_bytep)png_malloc(png_ptr,
+-    3*height*width*png_sizeof(*row_data));
++    3*height*width*sizeof(*row_data));
+   row_pointers = (png_bytep *)png_malloc(png_ptr,
+-    height*png_sizeof(*row_pointers));
++    height*sizeof(*row_pointers));
+   for(y = 0; y < height; y++) {
+     row_pointers[y] = row_data + y*(3*width);
+   }

component/gcc/buildout.cfg

@@ -33,7 +33,7 @@ environment =
 [ppl]
 recipe = slapos.recipe.cmmi
 # we should use version 0.10.x for gcc-4.5
-url = http://www.cs.unipr.it/ppl/Download/ftp/releases/0.10.2/ppl-0.10.2.tar.bz2
+url = http://bugseng.com/products/ppl/download/ftp/releases/0.10.2/ppl-0.10.2.tar.bz2
 md5sum = 5667111f53150618b0fa522ffc53fc3e
 configure-options =
   --with-libgmp-prefix=${gmp:location}
@@ -85,9 +85,6 @@ patches =
   ${gcc-multiarch.patch:location}/${gcc-multiarch.patch:filename}
 patch-options = -p2
 configure-command = make clean \\; make distclean \\; ./configure
-# GMP does not correctly detect achitecture so it have to be given
-# as slapos.recipe.cmmi is using shell expansion in subproceses
-# backticks are working
 configure-options =
   --disable-bootstrap
   --enable-languages="c,c++"
@@ -97,12 +94,30 @@ configure-options =
   --with-mpc=${mpc:location}
   --with-ppl=${ppl:location}
   --with-cloog=${cloog-ppl:location}
-  --with-ecj-jar=${ecj:location}/${ecj:filename}
   --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
-
 environment =
   LDFLAGS=-Wl,-rpath=${mpfr:location}/lib -Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${ppl:location}/lib -Wl,-rpath=${cloog-ppl:location}/lib
-  PATH=${zip:location}/bin:%(PATH)s
+# make install does not work when several core are used
+make-targets = install -j1
+
+[gcc-minimal]
+recipe = slapos.recipe.cmmi
+url = http://ftp.gnu.org/gnu/gcc/gcc-4.5.4/gcc-core-4.5.4.tar.bz2
+md5sum = ca62e442629a9a7710f5d797bf1b521c
+patches =
+  ${gcc-multiarch.patch:location}/${gcc-multiarch.patch:filename}
+patch-options = -p2
+configure-options =
+  --disable-bootstrap
+  --enable-languages=c
+  --disable-multilib
+  --with-gmp=${gmp:location}
+  --with-mpfr=${mpfr:location}
+  --with-mpc=${mpc:location}
+  --without-ppl
+  --without-cloog
+environment =
+  LDFLAGS=-Wl,-rpath=${mpfr:location}/lib -Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${mpc:location}/lib
 # make install does not work when several core are used
 make-targets = install -j1
 

component/git/buildout.cfg

@@ -4,6 +4,7 @@
 [buildout]
 extends =
   ../curl/buildout.cfg
+  ../gettext/buildout.cfg
   ../libexpat/buildout.cfg
   ../openssl/buildout.cfg
   ../zlib/buildout.cfg

component/glib/buildout.cfg

@@ -17,3 +17,7 @@ environment =
   CPPFLAGS=-I${zlib:location}/include -I${gettext:location}/include
   LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib
   PATH=${gettext:location}/bin:%(PATH)s
+
+[x86-cygwin-glib]
+patches =
+  ${:_profile_base_location_}/cygwin-glib-2.28.8-src.patch

component/glib/cygwin-glib-2.28.8-src.patch

+--- glib/gstrfuncs.c~	2012-12-30 14:51:30.000000000 +0800
++++ glib/gstrfuncs.c	2012-12-30 14:51:50.203125000 +0800
+@@ -1423,7 +1423,7 @@
+ #ifdef HAVE_STRSIGNAL
+   const char *msg_locale;
+ 
+-#if defined(G_OS_BEOS) || defined(G_WITH_CYGWIN)
++#if defined(G_OS_BEOS) 
+ extern const char *strsignal(int);
+ #else
+   /* this is declared differently (const) in string.h on BeOS */

component/gnutls/buildout.cfg

 [buildout]
 extends =
-  ../readline/buildout.cfg
+  ../gmp/buildout.cfg
+  ../nettle/buildout.cfg
   ../ncurses/buildout.cfg
+  ../readline/buildout.cfg
   ../zlib/buildout.cfg
 
 parts = gnutls
@@ -22,14 +24,13 @@ environment =
   LDFLAGS=-lgpg-error -L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib
 
 [gnutls]
-# XXX-Cedric : update to latest gnutls
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.gnutls.org/gcrypt/gnutls/v2.8/gnutls-2.8.6.tar.bz2
-md5sum = eb0a6d7d3cb9ac684d971c14f9f6d3ba
+url = ftp://ftp.gnutls.org/gcrypt/gnutls/v3.2/gnutls-3.2.0.tar.xz
+md5sum = e0cba4ddd923420026ff9739b3bc069a
 configure-options =
   --with-libgcrypt-prefix=${gcrypt:location}
   --disable-static
 environment =
-  CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${gcrypt:location}/include -I${gpg-error:location}/include
-  LDFLAGS=-lgcrypt -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${gcrypt:location}/lib -Wl,-rpath=${gcrypt:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib
-  PKG_CONFIG=${zlib:location}/lib/pkgconfig
+  CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${gmp:location}/include -I${gcrypt:location}/include -I${gpg-error:location}/include -I${nettle:location}/include
+  LDFLAGS=-lgcrypt -L${gmp:location}/lib -Wl,-rpath=${gmp:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${gcrypt:location}/lib -Wl,-rpath=${gcrypt:location}/lib -L${nettle:location}/lib -Wl,-rpath=${nettle:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib
+

component/grep/buildout.cfg

 [buildout]
 extends =
   ../pcre/buildout.cfg
+  ../coreutils/buildout.cfg
   ../xz-utils/buildout.cfg
 parts =
-  grep
+  grep-output
 
 [grep]
 recipe = slapos.recipe.cmmi
@@ -13,3 +14,11 @@ environment =
   PATH=${xz-utils:location}/bin:%(PATH)s
   CPPFLAGS=-I${pcre:location}/include
   LDFLAGS=-L${pcre:location}/lib -Wl,-rpath=${pcre:location}/lib
+
+[grep-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${coreutils-output:test} -x ${:grep}
+grep = ${grep:location}/bin/grep

component/mariadb/buildout.cfg

@@ -25,10 +25,10 @@ download-only = true
 
 [mariadb]
 recipe = slapos.recipe.cmmi
-version = 5.5.31
+version = 5.5.32
 revision = 1
 url = http://downloads.askmonty.org/f/mariadb-${:version}/kvm-tarbake-jaunty-x86/mariadb-${:version}.tar.gz/from/http://ftp.osuosl.org/pub/mariadb
-md5sum = 3fe756bc76f0e7a3af2757e48ce0f3f4
+md5sum = 565c2dce6a2fb027c9d0ffbae4934135
 # compile directory is required to build mysql plugins.
 keep-compile-dir = true
 patch-options = -p0

component/nettle/buildout.cfg

+[buildout]
+extends =
+  ../gmp/buildout.cfg
+  ../m4/buildout.cfg
+
+[nettle-lib-location.patch]
+recipe = hexagonit.recipe.download
+download-only = true
+filename = ${:_buildout_section_name_}
+url = ${:_profile_base_location_}/${:filename}
+md5sum = 41dd0ce2a73487929bdc637b75dd62c9
+
+[nettle]
+recipe = slapos.recipe.cmmi
+url = http://www.lysator.liu.se/~nisse/archive/nettle-2.7.1.tar.gz
+md5sum = 003d5147911317931dd453520eb234a5
+patches =
+  ${nettle-lib-location.patch:location}/${nettle-lib-location.patch:filename}
+configure-option =
+  --disable-static
+  --disable-assembler
+  --disable-openssl
+environment =
+  PATH=${m4:location}/bin:%(PATH)s
+  CPPFLAGS=-I${gmp:location}/include
+  LDFLAGS=-L${gmp:location}/lib -Wl,-rpath=${gmp:location}/lib

component/nettle/nettle-lib-location.patch

+--- configure.orig	2013-07-05 15:37:28.000000000 +0200
++++ configure	2013-07-05 15:47:48.000000000 +0200
+@@ -4680,52 +4680,6 @@
+ if test "x$ABI" != xstandard ; then
+   { $as_echo "$as_me:${as_lineno-$LINENO}: Compiler uses $ABI-bit ABI. To change, set CC." >&5
+ $as_echo "$as_me: Compiler uses $ABI-bit ABI. To change, set CC." >&6;}
+-  if test "$libdir" = '${exec_prefix}/lib' ; then
+-    # Try setting a better default
+-    case "$host_cpu:$host_os:$ABI" in
+-      *:solaris*:32|*:sunos*:32)
+-	libdir='${exec_prefix}/lib'
+-	;;
+-      *:solaris*:64|*:sunos*:64)
+-	libdir='${exec_prefix}/lib/64'
+-	;;
+-      # Linux conventions are a mess... According to the Linux File
+-      # Hierarchy Standard, all architectures except IA64 puts 32-bit
+-      # libraries in lib, and 64-bit in lib64. Some distributions,
+-      # e.g., Fedora and Gentoo, adhere to this standard, while at
+-      # least Debian has decided to put 64-bit libraries in lib and
+-      # 32-bit libraries in lib32.
+-
+-      # We try to figure out the convention, except if we're cross
+-      # compiling. We use lib${ABI} if /usr/lib${ABI} exists and
+-      # appears to not be a symlink to a different name.
+-      *:linux*:32|*:linux*:64)
+-	if test "$cross_compiling" = yes ; then
+-          { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Cross compiling for linux. Can't guess if libraries go in lib${ABI} or lib." >&5
+-$as_echo "$as_me: WARNING: Cross compiling for linux. Can't guess if libraries go in lib${ABI} or lib." >&2;}; 	else
+-	  # The dash builtin pwd tries to be "helpful" and remember
+-	  # symlink names. Use -P option, and hope it's portable enough.
+-	  test -d /usr/lib${ABI} \
+-	    && (cd /usr/lib${ABI} && pwd -P | grep >/dev/null "/lib${ABI}"'$') \
+-	    && libdir='${exec_prefix}/'"lib${ABI}"
+-	fi
+-	;;
+-      # On freebsd, it seems 32-bit libraries are in lib32,
+-      # and 64-bit in lib. Don't know about "kfreebsd", does
+-      # it follow the Linux fhs conventions?
+-      *:freebsd*:32)
+-	libdir='${exec_prefix}/lib32'
+-	;;
+-      *:freebsd*:64)
+-	libdir='${exec_prefix}/lib'
+-	;;
+-      *)
+-        { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Don't know where to install $ABI-bit libraries on this system." >&5
+-$as_echo "$as_me: WARNING: Don't know where to install $ABI-bit libraries on this system." >&2;};
+-    esac
+-    { $as_echo "$as_me:${as_lineno-$LINENO}: Libraries to be installed in $libdir." >&5
+-$as_echo "$as_me: Libraries to be installed in $libdir." >&6;}
+-  fi
+ fi
+ 
+ # Select assembler code

component/nginx/buildout.cfg

@@ -6,8 +6,9 @@ extends =
 
 [nginx]
 recipe = slapos.recipe.cmmi
-url = http://nginx.org/download/nginx-1.2.7.tar.gz
-md5sum = d252f5c689a14a668e241c744ccf5f06
+#url = http://nginx.org/download/nginx-1.2.7.tar.gz
+url = http://nginx.org/download/nginx-1.4.2.tar.gz
+#md5sum = d252f5c689a14a668e241c744ccf5f06
 configure-options=
   --with-ipv6
   --with-http_ssl_module

component/nodejs/buildout.cfg

@@ -14,8 +14,8 @@ parts =
 [nodejs-0.8]
 # Server-side Javascript.
 recipe = slapos.recipe.cmmi
-url = http://nodejs.org/dist/v0.8.14/node-v0.8.14.tar.gz
-md5sum = 284fd2c7578064c339d9cf6a3a475ac7
+url = http://nodejs.org/dist/v0.8.23/node-v0.8.23.tar.gz
+md5sum = 22fe54ac365f52d3d80ecf748e7323d5
 configure-options =
   --openssl-includes=${openssl:location}/include
   --openssl-libpath=${openssl:location}/lib

component/openssl/buildout.cfg

@@ -51,4 +51,4 @@ configure-options =
 make-options =
   -j1
 make-targets =
-  install && rm -f ${buildout:parts-directory}/${:_buildout_section_name_}/etc/ssl/certs/* && for i in ${ca-certificates:location}/certs/*/*.crt; do ln -sv $i ${buildout:parts-directory}/${:_buildout_section_name_}/etc/ssl/certs/`${buildout:parts-directory}/${:_buildout_section_name_}/bin/openssl x509 -hash -noout -in $i`.0; done; true
+  all install_sw && rm -f ${buildout:parts-directory}/${:_buildout_section_name_}/etc/ssl/certs/* && for i in ${ca-certificates:location}/certs/*/*.crt; do ln -sv $i ${buildout:parts-directory}/${:_buildout_section_name_}/etc/ssl/certs/`${buildout:parts-directory}/${:_buildout_section_name_}/bin/openssl x509 -hash -noout -in $i`.0; done; true

component/popt/buildout.cfg

@@ -8,3 +8,6 @@ url = ftp://anduin.linuxfromscratch.org/BLFS/svn/p/popt-1.16.tar.gz
 md5sum = 3743beefa3dd6247a73f8f7a32c14c33
 configure-options =
   --disable-static
+
+[x86-cygwin-popt]
+patches = ${:_profile_base_location_}/cygwin-popt-1.16-src.patch
\ No newline at end of file

component/popt/cygwin-popt-1.16-src.patch

+--- poptconfig.c~	2011-09-10 22:49:30.802250000 +0800
++++ poptconfig.c	2011-09-10 23:46:30.572048000 +0800
+@@ -45,8 +45,9 @@ extern int glob_pattern_p (const char *_
+ #if !defined(__GLIBC__)
+ /* Return nonzero if PATTERN contains any metacharacters.
+    Metacharacters can be quoted with backslashes if QUOTE is nonzero.  */
++#define glob_pattern_p glob_pattern_p_s
+ static int
+-glob_pattern_p (const char * pattern, int quote)
++glob_pattern_p_s(const char * pattern, int quote)
+ 	/*@*/
+ {
+     const char * p;

component/python-openssl/buildout.cfg

@@ -8,6 +8,8 @@ parts =
 [python-openssl]
 recipe = zc.recipe.egg:custom
 egg = pyOpenSSL
+include-dirs =
+  ${openssl:location}/include/
 library-dirs =
   ${openssl:location}/lib/
 rpath =

component/qemu-kvm/buildout.cfg

 [buildout]
-extends = 
+extends =
   ../../component/gnutls/buildout.cfg
   ../../component/libpng/buildout.cfg
   ../../component/libuuid/buildout.cfg
+  ../../component/pkgconfig/buildout.cfg
   ../../component/xorg/buildout.cfg
   ../../component/zlib/buildout.cfg
 
-[kvm]
-# Backward compatibility
-<= qemu-kvm
+# XXX Change all reference to kvm section to qemu section, then
+# use qemu as main name section.
+[qemu]
+<= kvm
 
-[qemu-kvm]
+[kvm]
 recipe = slapos.recipe.cmmi
 # qemu-kvm and qemu are now the same since 1.3.
-url = http://wiki.qemu-project.org/download/qemu-1.4.1.tar.bz2
-md5sum = eb2d696956324722b5ecfa46e41f9a75
+url = http://wiki.qemu-project.org/download/qemu-1.5.1.tar.bz2
+md5sum = b56e73bdcfdb214d5c68e13111aca96f
 depends =
   ${libpng:so_version}
 configure-options =
-  --target-list=""
+  --target-list=x86_64-softmmu
+  --enable-system
+  --with-system-pixman
   --disable-sdl
   --disable-xen
   --enable-vnc-tls
@@ -36,4 +40,27 @@ configure-options =
 environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${gnutls:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig
+  LDFLAGS=-L${pixman:location}/lib -Wl,-rpath=${pixman:location}/lib
+
+# The following is only available in buildout2, which we don't use yet.
+[kvm-bits64]
+configure-options =
+  --target-list=x86_64-softmmu
+  ${kvm:configure-options}
+
+[kvm-bits32]
+configure-options =
+  --target-list=i386-softmmu
+  ${kvm:configure-options}
+
+
+[debian-amd64-netinst.iso]
+# Download the installer of Debian 7 (Wheezy)
+recipe = slapos.recipe.download
+url = http://cdimage.debian.org/debian-cd/7.1.0/amd64/iso-cd/debian-7.1.0-amd64-netinst.iso
+filename = ${:_buildout_section_name_}
+md5sum = 80f498a1f9daa76bc911ae13692e4495
+download-only = true
+mode = 0644
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
 

component/rdiff-backup/buildout.cfg

 [buildout]
 extends =
   ../librsync/buildout.cfg
+  ../coreutils/buildout.cfg
 parts =
-  rdiff-backup
+  rdiff-backup-output
 
 [rdiff-backup-build]
 recipe = zc.recipe.egg:custom
@@ -24,3 +25,11 @@ eggs =
 entry-points =
   rdiff-backup=rdiff_backup.Main:Main
 arguments = sys.argv[1:]
+
+[rdiff-backup-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${coreutils-output:test} -x ${:rdiff-backup}
+rdiff-backup = ${buildout:directory}/bin/rdiff-backup

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '0.78.2.dev'
+version = '0.78.5'
 name = 'slapos.cookbook'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"
@@ -165,8 +165,6 @@ setup(name=name,
           'publish.serialised = slapos.recipe.publish:Serialised',
           'publishsection = slapos.recipe.publish:PublishSection',
           'publishurl = slapos.recipe.publishurl:Recipe',
-          'pwgen = slapos.recipe.pwgen:Recipe',
-          'pwgen.stable = slapos.recipe.pwgen:StablePasswordGeneratorRecipe',
           'redis.server = slapos.recipe.redis:Recipe',
           'request = slapos.recipe.request:Recipe',
           'request.serialised = slapos.recipe.request:Serialised',
@@ -188,6 +186,8 @@ setup(name=name,
           'slapreport = slapos.recipe.slapreport:Recipe',
           'slaprunner = slapos.recipe.slaprunner:Recipe',
           'slaprunner.test = slapos.recipe.slaprunner:Test',
+          'slaprunner.export = slapos.recipe.slaprunner.backup:ExportRecipe',
+          'slaprunner.import = slapos.recipe.slaprunner.backup:ImportRecipe',
           'softwaretype = slapos.recipe.softwaretype:Recipe',
           'sphinx= slapos.recipe.sphinx:Recipe',
           'sshkeys_authority = slapos.recipe.sshkeys_authority:Recipe',

slapos/recipe/README.generic_varnish.txt

@@ -12,15 +12,14 @@ On slap console, you can instanciate varnish like this:
 instance = request(
   software_type='varnish',
   partition_parameter_kw={
-     'tidstorage-url':'http://[your tidstrage address]:your tid strage port',
+     'backend-url':'https://[your_backend_address]:your_backend_port',
      'web-checker-frontend-url':'http://www.example.com',
      'web-checker-mail-address':'web-checker-result@example.com',
      'web-checker-smtp-host':'mail.example.com',
   }
 )
 
-tidstrage-url is the backend url that varnish will cache. It is expected that
-the backend is created by tidstorage recipe.
+backend-url is the backend url that varnish will cache.
 
 web-checker-frontend-url is the entry-point-url that web checker will check
 the HTTP headers of all the pages in the web site.

slapos/recipe/README.kvm.txt

@@ -31,10 +31,10 @@ KVM with Remote and gzipped Image
     gzip = true
     
     # Use -hda instead -drive arg
-    # Default is drive (see Options bellow)
+    # Default is drive (see Options below)
     image_type = hda
 
-    ### Common Configuration bellow. ###
+    ### Common Configuration below. ###
     
     # VNC is optional
     kvm_vnc = <SOME-IP>:<VNC-DISPLAY>

slapos/recipe/addresiliency/renamer.py

@@ -59,8 +59,6 @@ class Renamer(object):
 
         cp_broken.rename(new_name=broken_new_ref)
 
-        cp_broken.stopped()
-
         log.debug("Renaming {}: {}".format(cp_winner.getId(), cp_exporter_ref))
 
         # update name (and later, software type) for the partition that will take over

slapos/recipe/check_url_available/__init__.py

@@ -37,6 +37,7 @@ class Recipe(GenericBaseRecipe):
       'url': self.options['url'],
       'shell_path': self.options['dash_path'],
       'curl_path': self.options['curl_path'],
+      'check_secure': self.options.get('check-secure', 0)
     }
 
     # XXX-Cedric in this script, curl won't check certificate

slapos/recipe/check_url_available/template/check_url.in

@@ -31,6 +31,13 @@ if [ $CODE -eq 000 ]; then
   exit 1
 fi
 
+if [ %(check_secure)s -eq 1 ]; then
+  if [ $CODE -eq 401 ]; then
+    echo "$URL is protected (returned $CODE)." >&2
+    exit 0
+  fi
+fi
+
 if ! [ $CODE -eq 200 ]; then
   echo "$URL is not available (returned $CODE)." >&2
   exit 2

slapos/recipe/davstorage/__init__.py

@@ -27,6 +27,8 @@
 import subprocess
 import httplib
 import base64
+import os
+import shutil
 
 from slapos.recipe.librecipe import GenericBaseRecipe
 
@@ -50,13 +52,29 @@ class Recipe(GenericBaseRecipe):
                            user, password
                           ])
 
-
+    htdocs_location = self.options['htdocs']
+    if not (os.path.exists(htdocs_location) and os.listdir(htdocs_location)):
+      try:
+        os.rmdir(htdocs_location)
+      except:
+        pass
+      shutil.copytree(self.options['source'], htdocs_location)
+
+    # Install php.ini
+    php_ini = self.createFile(os.path.join(self.options['php-ini-dir'],
+                                             'php.ini'),
+                                self.substituteTemplate(self.getTemplateFilename('php.ini.in'),
+                                dict(tmp_directory=self.options['tmp-dir']))
+                )
+    path_list.append(php_ini)
+    
     apache_config = dict(
       pid_file=self.options['pid-file'],
       lock_file=self.options['lock-file'],
       davlock_db=self.options['davdb-lock'],
       ip=self.options['ip'],
-      port=self.options['port'],
+      port_webdav=self.options['port_webdav'],
+      port_ajax=self.options['port_ajax'],
       error_log=self.options['error-log'],
       access_log=self.options['access-log'],
       document_root=self.options['htdocs'],
@@ -67,6 +85,7 @@ class Recipe(GenericBaseRecipe):
       htpasswd_file=htpasswd_file,
       ssl_certificate=self.options['cert-file'],
       ssl_key=self.options['key-file'],
+      php_ini_dir=self.options['php-ini-dir']
     )
 
     # Create logfiles
@@ -86,7 +105,7 @@ class Recipe(GenericBaseRecipe):
 
     promise = self.createPythonScript(self.options['promise'],
       __name__ + '.promise',
-      dict(host=self.options['ip'], port=int(self.options['port']),
+      dict(host=self.options['ip'], port=int(self.options['port_webdav']),
            user=self.options['user'], password=self.options['password'])
                                      )
     path_list.append(promise)

slapos/recipe/davstorage/template/httpd.conf.in

 ServerRoot "%(server_root)s"
 
-Listen [%(ip)s]:%(port)s
+Listen [%(ip)s]:%(port_webdav)s
+Listen [%(ip)s]:%(port_ajax)s
+
+NameVirtualHost [%(ip)s]:%(port_webdav)s
+NameVirtualHost [%(ip)s]:%(port_ajax)s
 
 # Needed modules
-LoadModule unixd_module modules/mod_unixd.so
-LoadModule access_compat_module modules/mod_access_compat.so
-LoadModule authz_core_module modules/mod_authz_core.so
+LoadModule unixd_module "%(modules_dir)s/mod_unixd.so"
+LoadModule access_compat_module "%(modules_dir)s/mod_access_compat.so"
+LoadModule authn_core_module "%(modules_dir)s/mod_authn_core.so"
+LoadModule authz_core_module "%(modules_dir)s/mod_authz_core.so"
 LoadModule authn_file_module "%(modules_dir)s/mod_authn_file.so"
 LoadModule authz_host_module "%(modules_dir)s/mod_authz_host.so"
 LoadModule authz_user_module "%(modules_dir)s/mod_authz_user.so"
 LoadModule auth_basic_module "%(modules_dir)s/mod_auth_basic.so"
-LoadModule auth_digest_module "%(modules_dir)s/mod_auth_digest.so"
+# Comment auth_digest since we don't use it
+#LoadModule auth_digest_module "%(modules_dir)s/mod_auth_digest.so"
 LoadModule log_config_module "%(modules_dir)s/mod_log_config.so"
 LoadModule headers_module "%(modules_dir)s/mod_headers.so"
 LoadModule setenvif_module "%(modules_dir)s/mod_setenvif.so"
-LoadModule socache_shmcb_module modules/mod_socache_shmcb.so
+LoadModule socache_shmcb_module "%(modules_dir)s/mod_socache_shmcb.so"
 LoadModule ssl_module "%(modules_dir)s/mod_ssl.so"
 LoadModule mime_module "%(modules_dir)s/mod_mime.so"
 LoadModule dav_module "%(modules_dir)s/mod_dav.so"
 LoadModule dav_fs_module "%(modules_dir)s/mod_dav_fs.so"
 LoadModule dir_module "%(modules_dir)s/mod_dir.so"
+LoadModule php5_module "%(modules_dir)s/libphp5.so"
 
 ServerAdmin %(email_address)s
 
 # Quiet Server header (if not, Apache give its life history)
 # It's safer
 ServerTokens ProductOnly
+PidFile "%(pid_file)s"
+PHPINIDir "%(php_ini_dir)s"
+
+SSLRandomSeed startup builtin
+SSLRandomSeed connect builtin
+
+<VirtualHost [%(ip)s]:%(port_ajax)s>
+  #ServerName www.example.com
+  
+  # Directory protection
+  <Directory />
+      Options FollowSymLinks
+      AllowOverride None
+      Require all denied
+  </Directory>
+  
+  <Directory %(document_root)s>
+    Options FollowSymLinks
+    AllowOverride All
+    Require all granted
+  </Directory>
+ 
+  DocumentRoot "%(document_root)s"
+  DirectoryIndex index.html index.php
+
+SSLEngine on
+SSLCertificateFile "%(ssl_certificate)s"
+SSLCertificateKeyFile "%(ssl_key)s"
+
+</VirtualHost>
 
+<VirtualHost [%(ip)s]:%(port_webdav)s>
 DocumentRoot "%(document_root)s"
-PidFile "%(pid_file)s"
 DavLockDB "%(davlock_db)s"
 
 <Directory />
@@ -66,6 +103,12 @@ DavLockDB "%(davlock_db)s"
 
 </Directory>
 
+SSLEngine on
+SSLCertificateFile "%(ssl_certificate)s"
+SSLCertificateKeyFile "%(ssl_key)s"
+
+</VirtualHost>
+
 ErrorLog "%(error_log)s"
 LogLevel warn
 
@@ -77,9 +120,5 @@ DefaultType text/plain
 TypesConfig "%(mime_types)s"
 AddType application/x-compress .Z
 AddType application/x-gzip .gz .tgz
-
-SSLRandomSeed startup builtin
-SSLRandomSeed connect builtin
-SSLEngine on
-SSLCertificateFile "%(ssl_certificate)s"
-SSLCertificateKeyFile "%(ssl_key)s"
+AddType application/x-httpd-php .php .phtml .php5 .php4
+AddType application/x-httpd-php-source .phps
\ No newline at end of file

slapos/recipe/davstorage/template/php.ini.in

+[PHP]
+engine = On
+safe_mode = Off
+expose_php = Off
+error_reporting = E_ALL & ~(E_DEPRECATED|E_NOTICE|E_WARNING)
+display_errors = On
+display_startup_errors = Off
+log_errors = On
+log_errors_max_len = 1024
+ignore_repeated_errors = Off
+ignore_repeated_source = Off
+session.save_path = "%(tmp_directory)s"
+session.auto_start = 0
+date.timezone = Europe/Paris
+file_uploads = On
+upload_max_filesize = 8M
+post_max_size = 8M
+magic_quotes_gpc=Off
+output_buffering=Off
\ No newline at end of file

slapos/recipe/generatepassword.py

@@ -26,27 +26,69 @@
 #
 ##############################################################################
 
-import binascii
+import errno
 import os
+import random
+import string
 
-from slapos.recipe.librecipe import GenericBaseRecipe
+def generatePassword(length):
+  return ''.join(random.SystemRandom().sample(string.ascii_lowercase, length))
 
 
-class Recipe(GenericBaseRecipe):
+class Recipe(object):
+  """Generate a password that is only composed of lowercase letters
+
+    This recipe only makes sure that ${:passwd} does not end up in `.installed`
+    file, which is world-readable by default. So be careful not to spread it
+    throughout the buildout configuration by referencing it directly: see
+    recipes like slapos.recipe.template:jinja2 to safely process the password.
+
+    Options:
+    - bytes: password length (default: 8 characters)
+    - storage-path: plain-text persistent storage for password,
+                    that can only be accessed by the user
+      (default: ${buildout:parts-directory}/${:_buildout_section_name_})
+  """
 
   def __init__(self, buildout, name, options):
-    if os.path.exists(options['storage-path']):
-      open_file = open(options['storage-path'], 'r')
-      options['passwd'] = open_file.read()
-      open_file.close()
+    options_get = options.get
+    try:
+      self.storage_path = options['storage-path']
+    except KeyError:
+      self.storage_path = options['storage-path'] = os.path.join(
+        buildout['buildout']['parts-directory'], name)
+    try:
+      with open(self.storage_path) as f:
+        passwd = f.read()
+    except IOError, e:
+      if e.errno != errno.ENOENT:
+        raise
+      passwd = None
+    if not passwd:
+      passwd = self.generatePassword(int(options_get('bytes', '8')))
+      self.update = self.install
+    self.passwd = passwd
+    # Password must not go into .installed file, for 2 reasons:
+    # security of course but also to prevent buildout to always reinstall.
+    options.get = lambda option, *args, **kw: passwd \
+      if option == 'passwd' else options_get(option, *args, **kw)
 
-    if options.get('passwd', '') == '':
-      options['passwd'] = binascii.hexlify(os.urandom(
-                            int(options.get('bytes', '24'))))
-    return GenericBaseRecipe.__init__(self, buildout, name, options)
+  generatePassword = staticmethod(generatePassword)
 
   def install(self):
-    with open(self.options['storage-path'], 'w') as fout:
-      fout.write(self.options['passwd'])
-    return [self.options['storage-path']]
+    if self.storage_path:
+      try:
+        os.unlink(self.storage_path)
+      except OSError, e:
+        if e.errno != errno.ENOENT:
+          raise
+      fd = os.open(self.storage_path,
+        os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0600)
+      try:
+        os.write(fd, self.passwd)
+      finally:
+        os.close(fd)
+    return self.storage_path
 
+  def update(self):
+    return ()

slapos/recipe/generic_varnish/__init__.py

@@ -37,7 +37,12 @@ class Recipe(GenericSlapRecipe):
   """
   def _install(self):
     ip = self.options['ip']
-    backend_url = self.parameter_dict['tidstorage-url']
+    backend_url = self.options.get('backend-url',
+      # BBB: Peeking in partition parameters directly. Eew.
+      self.parameter_dict.get('backend-url',
+        self.parameter_dict.get('tidstorage-url') # BBB
+      )
+    )
     backend_server, backend_port = self._getBackendServer(backend_url)
     path_list = []
     if backend_url.startswith('https://'):
@@ -70,6 +75,7 @@ class Recipe(GenericSlapRecipe):
       varnishd_pid_file=self.options['pid-file'],
       varnish_instance_name=self.options['varnish-instance-name'],
       varnish_data=self.options['varnish-data'],
+      gcc_location=self.options['gcc-location'],
       shell_path=self.options['shell-path'],
       vcl_file=self.options['vcl-file'],
       backend_port=backend_port,

slapos/recipe/generic_varnish/template/varnishd.in

@@ -4,11 +4,12 @@ DAEMON_OPTS="-F \
              -a %(varnish_ip)s:%(varnishd_server_port)s \
              -T %(varnish_ip)s:%(varnishd_manager_port)s \
              -t 0 \
+             -p nuke_limit=500 \
              -n %(varnish_instance_name)s \
              -f %(vcl_file)s \
              -s file,%(varnish_data)s/varnish_storage.bin,1G"
 
 PIDFILE=%(varnishd_pid_file)s
 # exporting PATH here so that we will pass the PATH variable to the subprocess
-export PATH
+export PATH="%(gcc_location)s:$PATH"
 exec %(varnishd_binary)s -P ${PIDFILE} ${DAEMON_OPTS} 2>&1

slapos/recipe/generic_zope/template/zope.conf.in

@@ -25,6 +25,9 @@ rest-input-encoding utf-8
 rest-output-encoding utf-8
 default-zpublisher-encoding utf-8
 
+# Disable ownership checking to execute codes generated by alarm
+skip-ownership-checking on
+
 # Temporary storage database (for sessions)
 <zodb_db temporary>
      <temporarystorage>

slapos/recipe/generic_zope_zeo_client/template/zope.conf.in

@@ -25,6 +25,9 @@ rest-input-encoding utf-8
 rest-output-encoding utf-8
 default-zpublisher-encoding utf-8
 
+# Disable ownership checking to execute codes generated by alarm
+skip-ownership-checking on
+
 # Temporary storage database (for sessions)
 <zodb_db temporary>
      <temporarystorage>

slapos/recipe/haproxy/__init__.py

@@ -87,7 +87,11 @@ class Recipe(GenericBaseRecipe):
       'haproxy-listen-snippet.cfg.in')
     server_snippet = ""
     ip = self.options['ip']
-    server_check_path = self.options['server-check-path']
+    server_check_path = self.options.get('server-check-path', None)
+    if server_check_path:
+      httpchk = 'option httpchk GET %s' % server_check_path
+    else:
+      httpchk = ''
     # FIXME: maxconn must be provided per-backend, not globally
     maxconn = self.options['maxconn']
     i = 0
@@ -97,7 +101,7 @@ class Recipe(GenericBaseRecipe):
           'name': name,
           'ip': ip,
           'port': port,
-          'server_check_path': server_check_path,
+          'httpchk': httpchk,
         })
       for address in backend_list:
         i += 1

slapos/recipe/haproxy/template/haproxy-listen-snippet.cfg.in

 listen %(name)s %(ip)s:%(port)s
   cookie  SERVERID insert
   balance roundrobin
-  option httpchk GET %(server_check_path)s
+  %(httpchk)s
   stats uri /haproxy
   stats realm Global\ statistics

slapos/recipe/kvm/__init__.py

@@ -61,7 +61,8 @@ class Recipe(GenericBaseRecipe):
       shell_path=self.options['shell-path'],
       qemu_path=self.options['qemu-path'],
       qemu_img_path=self.options['qemu-img-path'],
-      vnc_passwd=self.options['passwd']
+      vnc_passwd=self.options['passwd'],
+      default_disk_image=self.options['default-disk-image'],
     )
 
     # Runners

slapos/recipe/kvm/template/kvm_run.in

@@ -7,6 +7,9 @@ import os
 import socket
 import subprocess
 
+# XXX: give all of this through parameter, don't use this as template
+default_disk_image = '%(default_disk_image)s'
+
 def getSocketStatus(host, port):
   s = None
   for res in socket.getaddrinfo(host, port,
@@ -26,7 +29,8 @@ def getSocketStatus(host, port):
     break
   return s
 
-# create disk if doesn't exist
+# Create disk if doesn't exist
+# XXX: move to Buildout profile
 disk_path = '%(disk_path)s'
 if not os.path.exists(disk_path):
   subprocess.Popen(['%(qemu_img_path)s', 'create' ,'-f', 'qcow2',
@@ -58,4 +62,10 @@ for nbd_ip, nbd_port in (
           '-drive',
           'file=nbd:[%%s]:%%s,media=cdrom' %% (nbd_ip, nbd_port)])
 
+# If no NBD is specified/available: use internal disk image
+else:
+  kvm_argument_list.extend([
+      '-drive', 'file=%%s,media=cdrom' %% default_disk_image
+  ])
+
 os.execv('%(qemu_path)s', kvm_argument_list)

slapos/recipe/librecipe/generic.py

@@ -183,17 +183,13 @@ class GenericBaseRecipe(object):
         'template/%s' % template_name)
 
   def generatePassword(self, len_=32):
-    """
-    The purpose of this method is to generate a password which doesn't change
-    from one execution to the next, so the generated password doesn't change
-    on each slapgrid-cp execution.
-
-    Currently, it returns a hardcoded password because no decision has been
-    taken on where a generated password should be kept (so it is generated
-    once only).
-    """
-    # TODO: implement a real password generator which remember the last
-    # call.
+    # TODO: Consider having generate.password recipe inherit this class,
+    #       so that it can be easily inheritable.
+    #       In the long-term, it's probably better that passwords are provided
+    #       by software requesters, to avoid keeping unhashed secrets in
+    #       partitions when possible.
+    self.logger.warning("GenericBaseRecipe.generatePassword is deprecated."
+                        " Use generate.password recipe instead.")
     return "insecure"
 
   def isTrueValue(self, value):
@@ -247,7 +243,8 @@ class GenericBaseRecipe(object):
       destination = self.location
     if os.path.exists(destination):
         # leftovers from a previous failed attempt, removing it.
-        log.warning('Removing already existing directory %s' % destination)
+        self.logger.warning('Removing already existing directory %s',
+                            destination)
         shutil.rmtree(destination)
     os.mkdir(destination)
 

slapos/recipe/neoppod.py

@@ -50,8 +50,6 @@ class NeoBaseRecipe(GenericBaseRecipe):
       #'-n', options['name'],
       '-c', options['cluster'],
     ]
-    if options['verbose']:
-      option_list.append('-v')
     option_list.extend(self._getOptionList())
     return [self.createPythonScript(
       options['wrapper'],

slapos/recipe/publish.py

@@ -35,20 +35,22 @@ class Recipe(GenericSlapRecipe):
     publish_dict = dict()
     options = self.options.copy()
     del options['recipe']
-
+    slave_reference = options.pop('-slave-reference', None)
     for k, v in options.iteritems():
+      if k[:1] == '-':
+        continue
       publish_dict[k] = v
-    self._setConnectionDict(publish_dict)
+    self._setConnectionDict(publish_dict, slave_reference)
     return []
 
-  def _setConnectionDict(self, publish_dict):
-    return self.setConnectionDict(publish_dict)
+  def _setConnectionDict(self, publish_dict, slave_reference=None):
+    return self.setConnectionDict(publish_dict, slave_reference)
 
 SERIALISED_MAGIC_KEY = '_'
 
 class Serialised(Recipe):
-  def _setConnectionDict(self, publish_dict):
-    return super(Serialised, self)._setConnectionDict(wrap(publish_dict))
+  def _setConnectionDict(self, publish_dict, slave_reference=None):
+    return super(Serialised, self)._setConnectionDict(wrap(publish_dict), slave_reference)
 
 
 

slapos/recipe/pwgen.py

-##############################################################################
-#
-# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-import subprocess
-import os
-
-from slapos.recipe.librecipe import GenericBaseRecipe
-
-class Recipe(GenericBaseRecipe):
-
-  def _options(self, options):
-    if not os.path.exists(self.options['file']):
-      password = subprocess.check_output([self.options['pwgen-binary'], '-1']).strip()
-      with open(self.options['file'], 'w') as password_file:
-        password_file.write(password)
-    else:
-      with open(self.options['file'], 'r') as password_file:
-        password = password_file.read()
-    options['password'] = password
-
-  def install(self):
-    os.chmod(self.options['file'], 0600)
-    return []
-
-class StablePasswordGeneratorRecipe(GenericBaseRecipe):
-  """
-  The purpose of this class is to generate a password which doesn't change
-  from one execution to the next (hence "stable"), so the generated password
-  doesn't change on each slapgrid-cp execution.
-
-  See GenericBaseRecipe.generatePassword .
-  """
-
-  def _options(self, options):
-    options['password'] = self.generatePassword()
-
-  update = install = lambda self: []

slapos/recipe/slaprunner/__init__.py

@@ -40,8 +40,8 @@ class Recipe(GenericBaseRecipe):
     self.partition_amount = options['partition-amount'].strip()
     self.cloud9_url = options.get('cloud9-url', '').strip()
     self.log_file = os.path.join(options['log_dir'].strip(), 'slaprunner.log')
-    # Set slaprunner access URL
-    options['access-url'] = 'http://[%s]:%s' % (self.ipv6, self.runner_port)
+    # Set slaprunner access URL, CLN Beware ipv6 access is made throught nginx
+    options['access-url'] = 'https://[%s]:%s' % (self.ipv6, self.runner_port)
 
   def install(self):
     path_list = []
@@ -62,7 +62,7 @@ class Recipe(GenericBaseRecipe):
         etc_dir=self.options['etc_dir'],
         run_dir=self.options['run_dir'],
         log_dir=self.options['log_dir'],
-        runner_host=self.ipv6,
+        runner_host=self.ipv4,
         runner_port=self.runner_port,
         ipv4_address=self.ipv4,
         ipv6_address=self.ipv6,
@@ -132,7 +132,7 @@ class Test(GenericBaseRecipe):
         etc_dir=self.options['etc_dir'],
         run_dir=self.options['etc_dir'],
         log_dir=self.workdir,
-        runner_host=self.ipv6,
+        runner_host=self.ipv4,
         runner_port=self.runner_port,
         ipv4_address=self.ipv4,
         ipv6_address=self.ipv6,

slapos/recipe/slaprunner/backup.py

+##############################################################################
+#
+# Copyright (c) 2013 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import textwrap
+
+from slapos.recipe.librecipe import GenericBaseRecipe
+
+
+
+class ExportRecipe(GenericBaseRecipe):
+    """\
+    This recipe creates an exporter script for using with the resilient stack.
+
+    Required options:
+        backup-directory
+            folder that will contain the dump file.
+        srv-directory
+            folder that contain the runner directory.
+        wrapper
+            full path of the exporter script to create.
+    """
+
+    def install(self):
+        wrapper = self.options['wrapper']
+        self.createBackupScript(wrapper)
+        return [wrapper]
+
+
+    def createBackupScript(self, wrapper):
+        """\
+        Create a script to backup the database in 'custom' format.
+        """
+        content = textwrap.dedent("""\
+                #!%(shell-binary)s
+                umask 077
+                sync_element () {
+                  path=$1
+                  backup_path=$2
+                  shift 2
+                  element_list=$*
+                  for element in $element_list
+                  do
+                    cd $path;
+                    if [ -f $element ] || [ -d $element ]; then
+                       %(rsync-binary)s -avz --safe-links --delete $element  $backup_path;
+                    fi
+                  done
+                }
+                sync_element %(srv-directory)s/runner  %(backup-directory)s/runner/ instance project  proxy.db softwareLink
+                sync_element %(etc-directory)s  %(backup-directory)s/etc/ .rcode .project .users .htpasswd ssh
+                if [ -d %(backup-directory)s/runner/software ]; then
+                  rm %(backup-directory)s/runner/software/*
+                fi
+                """ % self.options)
+        self.createExecutable(wrapper, content=content)
+
+
+
+class ImportRecipe(GenericBaseRecipe):
+    """\
+    This recipe creates an importer script for using with the resilient stack.
+
+    Required options:
+        backup-directory
+            folder that will contain the dump file.
+        srv-directory
+            folder that contain the runner directory.
+        wrapper
+            full path of the exporter script to create.
+    """
+
+    def install(self):
+        wrapper = self.options['wrapper']
+        self.createRestoreScript(wrapper)
+        return [wrapper]
+
+
+    def createRestoreScript(self, wrapper):
+        """\
+        Create a script to restore the database from 'custom' format.
+        """
+        content = textwrap.dedent("""\
+                #!%(shell-binary)s
+                umask 077
+                restore_element () {
+                  backup_path=$1
+                  restore_path=$2
+                  shift 2
+                  element_list=$*
+                  for element in $element_list
+                  do
+                    cd $backup_path;
+                    if [ -f $element ] || [ -d $element ]; then
+                       %(rsync-binary)s -avz --delete $backup_path/$element $restore_path;
+                    fi
+                  done
+                }
+                restore_element %(backup-directory)s/runner/ %(srv-directory)s/runner  instance project  proxy.db softwareLink
+                restore_element  %(backup-directory)s/etc/ %(etc-directory)s .rcode .project .users .htpasswd ssh
+                ifs=$IFS IFS=';'
+                read user pass remaining < %(etc-directory)s/.users
+                IFS=$ifs
+                %(curl-binary)s -vg6L -F clogin="$user" -F cpwd="$pass" --dump-header login_cookie  %(backend-url)s/doLogin;
+                %(curl-binary)s -vg6LX POST --cookie login_cookie --max-time 5  %(backend-url)s/runSoftwareProfile;
+                rm -f login_cookie
+                """ % self.options)
+        self.createExecutable(wrapper, content=content)
+
+

slapos/recipe/softwaretype.py

@@ -87,8 +87,6 @@ class Recipe:
       computer_partition_id)
     self.parameter_dict = self.computer_partition.getInstanceParameterDict()
     software_type = self.parameter_dict['slap_software_type']
-    self.logger.info('Deploying instance with software type %s' % \
-        software_type)
 
     # Raise if request software_type does not exist ...
     if software_type not in self.options:

slapos/recipe/web_checker/__init__.py

@@ -34,11 +34,15 @@ class Recipe(GenericSlapRecipe):
   """
   def _install(self):
     path_list = []
-    web_checker_mail_address = self.parameter_dict['web-checker-mail-address']
-    web_checker_smtp_host = self.parameter_dict['web-checker-smtp-host']
-    web_checker_frontend_url = self.parameter_dict.get(
-                                 'web-checker-frontend-url',
-                                 self.options['frontend-url'])
+    try:
+      web_checker_mail_address = self.options['mail-address']
+      web_checker_smtp_host = self.options['smtp-host']
+      web_checker_frontend_url = self.options['frontend-url']
+    except KeyError:
+      # BBB
+      web_checker_mail_address = self.parameter_dict['web-checker-mail-address']
+      web_checker_smtp_host = self.parameter_dict['web-checker-smtp-host']
+      web_checker_frontend_url = self.parameter_dict['web-checker-frontend-url']
     web_checker_working_directory = \
       self.options['web-checker-working-directory']
     config = dict(

software/backupserver/instance-pullrdiffbackup.cfg.in

+{% set part_list = [] -%}
+{% set crontab_line_list = [] -%}
+###############################
+#
+# Instanciate dcron
+#
+###############################
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+bin = $${buildout:directory}/bin
+srv = $${buildout:directory}/srv
+var = $${buildout:directory}/var
+log = $${:var}/log
+# scripts = $${:etc}/run
+services = $${:etc}/service
+cron-entries = $${:etc}/cron.d
+cron-lines = $${:etc}/cron.lines
+crontabs = $${:etc}/crontabs
+cronstamps = $${:etc}/cronstamps
+backup = $${:srv}/backup
+backupscript = $${:etc}/backup
+home = $${:etc}/home
+ssh = $${:home}/.ssh
+
+#################################
+# Cron service
+#################################
+[dcron-service]
+recipe = slapos.recipe.template
+url = ${template-dcron-service:output}
+output = $${directory:services}/crond
+mode = 0700
+logfile = $${directory:log}/crond.log
+
+#################################
+# Slave backup scripts and crontab
+#################################
+# Go throught slave list to set their configuration
+{% for slave_instance in slave_instance_list -%}
+{%   set slave_reference = slave_instance.get('slave_reference') -%}
+{%   set frequency = slave_instance.get('frequency', '') -%}
+{%   set hostname = slave_instance.get('hostname', '') -%}
+{%   set connection = slave_instance.get('connection', '') -%}
+{%   set include = slave_instance.get('include', '') -%}
+{%   set include_string = "' --include='".join(include.split(' ')) -%}
+{%   set exclude = slave_instance.get('exclude', '') -%}
+{%   set exclude_string = '' -%}
+{%   set sudo = slave_instance.get('sudo', 'False') -%}
+{%   set remote_schema = 'rdiff-backup --server --restrict-read-only / -- "$@"' -%}
+
+{%   if (exclude != '') -%}
+{%     set exclude_string = "' --exclude='".join(exclude.split(' ')) -%}
+{%     set exclude_string = "--exclude='" + exclude_string + "'" -%}
+{%   endif -%}
+
+{%   if (sudo == 'True') -%}
+{%     set remote_schema = 'sudo backupagent_rdiff-backup' -%}
+{%   endif -%}
+
+{%   if (frequency != '') and (hostname != '') and (connection != '') and (include != '') -%}
+
+[{{ slave_reference }}-backup-directory]
+recipe = slapos.cookbook:mkdirectory
+directory = $${directory:backup}/$${:_buildout_section_name_}
+
+[{{ slave_reference }}-backup-private_key]
+recipe = plone.recipe.command
+stop-on-error = true
+command = ${dropbear-output:keygen} -t $${:type} -s 2048 -f $${:key} 
+key = $${directory:ssh}/$${:_buildout_section_name_}
+location = $${:key}
+type = rsa
+
+[{{ slave_reference }}-backup-public_key]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = $${:command}
+command = ${coreutils-output:rm} -f $${:key} && ${dropbear-output:keygen} -y -f {{ '$${' ~ slave_reference }}-backup-private_key:key} | ${grep-output:grep} {{ '$${' ~ slave_reference }}-backup-private_key:type} > $${:key}
+key = {{ '$${' ~ slave_reference }}-backup-private_key:key}.pub
+location = $${:key}
+
+[{{ slave_reference }}-backup-check-public_key]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = $${:command}
+command = grep ssh-{{ '$${' ~ slave_reference }}-backup-private_key:type} {{ '$${' ~ slave_reference }}-backup-public_key:key}
+
+[{{ slave_reference }}-backup-read-public_key]
+recipe = slapos.cookbook:generate.password
+storage-path = {{ '$${' ~ slave_reference }}-backup-public_key:key}
+bytes = 8
+
+# Publish slave {{ slave_reference }} information
+[{{ slave_reference }}-backup-publish]
+recipe = slapos.cookbook:publish
+-slave-reference = {{ slave_reference }}
+authorized_key = {{ '$${' ~ slave_reference }}-backup-read-public_key:passwd}
+{%   do part_list.append("%s-backup-publish" % slave_reference) -%}
+
+[{{ slave_reference }}-backup-script]
+recipe = slapos.recipe.template
+url = ${template-backup-script:output}
+output = $${directory:backupscript}/$${:_buildout_section_name_}
+mode = 0700
+datadirectory = {{ '$${' ~ slave_reference }}-backup-directory:directory}
+sshkey = {{ '$${' ~ slave_reference }}-backup-private_key:key}
+connection = {{ connection }}
+hostname = {{ hostname }}
+include = {{ include_string }}
+exclude_string = {{ exclude_string }}
+remote_schema = {{ remote_schema }}
+
+[{{ slave_reference }}-backup-crontab-line]
+recipe = slapos.recipe.template
+url = ${template-crontab-line:output}
+output = $${directory:cron-lines}/$${:_buildout_section_name_}
+mode = 0600
+script = {{ '$${' ~ slave_reference }}-backup-script:output}
+frequency = {{ frequency }}
+{%   do crontab_line_list.append("$${%s-backup-crontab-line:output}" % slave_reference) -%}
+
+{%   endif -%}
+{% endfor -%}
+
+#################################
+# Generate crontab file
+#################################
+
+{% set crontab_line_list_string = " ".join(crontab_line_list) -%}
+[activate-crontab-file]
+# XXX File is never removed
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = $${:command}
+command = ${coreutils-output:cat} ${template-crontab:output} {{ crontab_line_list_string }} | ${dcron-output:crontab} -c $${directory:crontabs} -
+
+
+# Add parts generated by template
+[buildout]
+parts =
+  dcron-service
+  activate-crontab-file
+{% for part in part_list -%}
+{{ '    %s' % part }}
+{% endfor -%}
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true

software/backupserver/instance.cfg.in

+[buildout]
+parts =
+  switch-softwaretype
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[dynamic-template-pullrdiffbackup]
+recipe = slapos.recipe.template:jinja2
+template = ${template-pullrdiffbackup:output}
+rendered = $${buildout:parts-directory}/$${:_buildout_section_name_}/$${:filename}
+filename = instance-pullrdiffbackup.cfg
+extensions = jinja2.ext.do
+context =
+    key slave_instance_list instance-parameter:slave-instance-list
+
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = $${:pullrdiffbackup}
+# pullrdiffbackup = ${template-pullrdiffbackup:output}
+pullrdiffbackup = $${dynamic-template-pullrdiffbackup:rendered}
+
+[slap-connection]
+# part to migrate to new - separated words
+computer-id = $${slap_connection:computer_id}
+partition-id = $${slap_connection:partition_id}
+server-url = $${slap_connection:server_url}
+software-release-url = $${slap_connection:software_release_url}
+key-file = $${slap_connection:key_file}
+cert-file = $${slap_connection:cert_file}
+
+# [slap-parameter]
+# slave-instance-list = []
+
+[instance-parameter]
+# Fetches parameters defined in SlapOS Master for this instance.
+# Always the same.
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = $${slap_connection:computer_id}
+partition = $${slap_connection:partition_id}
+url = $${slap_connection:server_url}
+key = $${slap_connection:key_file}
+cert = $${slap_connection:cert_file}

software/backupserver/template-backup-script.sh.in

+#!${dash-output:dash}
+
+# trap "echo Backing up failed for $${:hostname}" ERR
+
+export HOME=$${directory:home}
+
+set -e
+cd $${:datadirectory}
+${rdiff-backup-output:rdiff-backup} \
+  $${:exclude_string} \
+  --include='$${:include}' \
+  --exclude='**' \
+  --remote-schema '${dropbear-output:ssh} -T -y -i $${:sshkey} %s $${:remote_schema}' \
+  $${:connection}::/ ./
+
+# python scripts/verify_with_sudo.py ./ $${:connection}:/
+

software/backupserver/template-crontab-line.in

+# $${:_buildout_section_name_}
+$${:frequency} $${:script}

software/backupserver/template-crontab.in

+# min(0-59)  hours(0-23) day(1-31) month(1-12) dow(0-7)   command
+MAILTO=admins@erp5.org
+

software/backupserver/template-dcron-service.sh.in

+#!${dash-output:dash}
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+
+exec ${dcron-output:crond} \
+  -s $${directory:cron-entries} \
+  -c $${directory:crontabs} \
+  -t $${directory:cronstamps} \
+  -f -l 5 \
+  -L $${dcron-service:logfile}
+  # -M cron_simplelogger

software/cloudooo/instance-cloudoo-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "extends": "./schema-definitions.json#",
+  "properties": {
+    "tcpv4-port": {
+      "allOf": [{
+        "$ref": "#/definitions/tcpv4port"
+      }, {
+        "description": "Start allocating ports at this value, going upward",
+        "default": 23000
+      }]
+    },
+    "font-url-list": {
+      "description": "List of URLs from which fonts are to be downloaded",
+      "default": [],
+      "items": {
+        "type": "string"
+      },
+      "type": "array"
+    },
+    "backend-count": {
+      "description": "Number of backend cloudooo instances",
+      "default": 1,
+      "type": "integer"
+    }
+  }
+}

software/cloudooo/instance-cloudoo-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Values returned by Cloudooo instanciation",
+  "properties": {
+    "url": {
+      "description": "Conversion service access information",
+      "type": "string"
+    }
+  },
+  "type": "object"
+}

software/cloudooo/instance-cloudoo.cfg.in

+{% set ipv4 = (ipv4_set | list)[0] -%}
 {% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
+{% set section_list = [] -%}
+{% macro section(name) %}{% do section_list.append(name) %}{{ name }}{% endmacro -%}
 [buildout]
 parts =
   publish-cloudooo-connection-information
@@ -9,35 +12,53 @@ develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
 [publish-cloudooo-connection-information]
-recipe = slapos.cookbook:publishurl
-url = cloudooo://${cloudooo-instance:ip}:${cloudooo-instance:port}/
+recipe = slapos.cookbook:publish.serialised
+url = cloudooo://${haproxy:ip}:${haproxy:port}/
 
-[cloudooo-instance]
+[cloudooo-base]
 recipe = slapos.cookbook:generic.cloudooo
-
-# Network options
-ip = ${slap-network-information:local-ipv4}
-port = 23000
-openoffice-port = 23060
-
-# Paths
-configuration-file = ${rootdirectory:etc}/cloudooo.cfg
-wrapper = ${basedirectory:services}/cloudooo
-
-# Paths: Data
-data-directory = ${directory:cloudooo-data}
-
+ip = {{ ipv4 }}
 environment =
   LD_LIBRARY_PATH = {{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['zlib'] }}/lib
   FONTCONFIG_FILE = ${fontconfig-instance:conf-path}
   PATH = ${binary-link:target-directory}
-
 # Binary information
 # cloudooo specific configuration
 ooo-binary-path = {{ parameter_dict['libreoffice-bin'] }}/program
 ooo-paster = {{ bin_directory }}/cloudooo_paster
 ooo-uno-path = {{ parameter_dict['libreoffice-bin'] }}/basis-link/program
 
+{% set cloudooo_port = slapparameter_dict.get('tcpv4_port', 23000) | int -%}
+{% set backend_count = slapparameter_dict.get('backend-count', 1) | int -%}
+{% for index in range(backend_count) -%}
+{%   set name = 'cloudooo-' ~ index -%}
+[{{ section(name) }}]
+< = cloudooo-base
+port = {{ cloudooo_port }}
+openoffice-port = {{ cloudooo_port + 1 }}
+configuration-file = ${directory:etc}/{{ name }}.cfg
+data-directory = ${directory:srv}/{{ name }}
+wrapper = ${directory:services}/{{ name }}
+
+{%   set cloudooo_port = cloudooo_port + 2 -%}
+{% endfor -%}
+
+[haproxy]
+recipe = slapos.cookbook:haproxy
+name = cloudooo
+conf-path = ${directory:etc}/haproxy.cfg
+socket-path = ${directory:run}/haproxy.sock
+ip = {{ ipv4 }}
+port = 8001
+maxconn = 1
+wrapper-path = ${directory:services}/haproxy
+binary-path = {{ parameter_dict['haproxy'] }}/sbin/haproxy
+ctl-path = ${directory:bin}/haproxy-ctl
+backend-list =
+{%- for section_name in section_list %}
+  {{ "${" ~ section_name ~ ":ip}:${" ~ section_name ~ ":port}" }}
+{%- endfor %}
+
 [cloudooo-test-runner]
 recipe = slapos.cookbook:cloudooo.test
 
@@ -45,24 +66,25 @@ prepend-path = ${buildout:bin-directory}
 run-unit-test = ${buildout:bin-directory}/runUnitTest
 run-test-suite = ${buildout:bin-directory}/runTestSuite
 
-ooo-paster = ${cloudooo-instance:ooo-paster}
-configuration-file = ${cloudooo-instance:configuration-file}
+ooo-paster = ${cloudooo-0:ooo-paster}
+configuration-file = ${cloudooo-0:configuration-file}
 
 run-unit-test-binary = {{ bin_directory }}/runCloudoooUnitTest
 run-test-suite-binary = {{ bin_directory }}/runCloudoooTestSuite
 
 [fontconfig-instance]
 recipe = slapos.cookbook:fontconfig
-conf-path = ${rootdirectory:etc}/font.conf
+conf-path = ${directory:etc}/font.conf
 font-system-folder = {{ parameter_dict['fonts'] }}
 font-folder = ${directory:font}
-url-list =
-service-folder = ${basedirectory:services}
+{# XXX: violates "instanciation happens offline" rule -#}
+url-list = {{ slapparameter_dict.get('font-url-list', []) | join(' ') }}
+service-folder = ${directory:services}
 onetimedownload_path = {{ bin_directory }}/onetimedownload
 
 [binary-link]
 recipe = slapos.cookbook:symbolic.link
-target-directory = ${rootdirectory:bin}
+target-directory = ${directory:bin}
 link-binary =
   {{ parameter_dict['coreutils'] }}/bin/basename
   {{ parameter_dict['coreutils'] }}/bin/cat
@@ -78,17 +100,12 @@ link-binary =
   {{ parameter_dict['poppler'] }}/bin/pdftohtml
 
 # rest of parts are candidates for some generic stuff
-[basedirectory]
-recipe = slapos.cookbook:mkdirectory
-services = ${rootdirectory:etc}/run
-
 [directory]
 recipe = slapos.cookbook:mkdirectory
-cloudooo-data = ${rootdirectory:srv}/cloudooo
-font = ${rootdirectory:srv}/font
-
-[rootdirectory]
-recipe = slapos.cookbook:mkdirectory
+bin = ${buildout:directory}/bin
 etc = ${buildout:directory}/etc
+font = ${:srv}/font
+run = ${:var}/run
+services = ${:etc}/run
 srv = ${buildout:directory}/srv
-bin = ${buildout:directory}/bin
+var = ${buildout:directory}/var

software/cloudooo/instance.cfg.in

@@ -6,6 +6,14 @@ eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
+[slap-parameters]
+recipe = slapos.cookbook:slapconfiguration
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
 rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
@@ -13,12 +21,14 @@ extra-context =
 context =
     key eggs_directory buildout:eggs-directory
     key develop_eggs_directory buildout:develop-eggs-directory
+    key slapparameter_dict slap-parameters:configuration
     ${:extra-context}
 
 [dynamic-template-cloudooo-parameters]
 file = {{ file_location }}
 fontconfig = {{ fontconfig_location }}
 freetype = {{ freetype_location }}
+haproxy = {{ haproxy_location }}
 libICE = {{ libICE_location }}
 libpng12 = {{ libpng12_location }}
 libSM = {{ libSM_location }}
@@ -40,8 +50,10 @@ buildout-bin-directory = {{ buildout_bin_directory }}
 < = jinja2-template-base
 template = {{ template_cloudooo }}
 filename = instance-cloudoo.cfg
+extensions = jinja2.ext.do
 extra-context =
     section parameter_dict dynamic-template-cloudooo-parameters
+    key ipv4_set slap-parameters:ipv4
 
 [switch-softwaretype]
 recipe = slapos.cookbook:softwaretype

software/cloudooo/software.cfg

@@ -16,6 +16,7 @@ parts +=
   check-recipe
 # Create instance template
   template
+  slapos-cookbook
 
 # XXX: Workaround of SlapOS limitation
 # Unzippig of eggs is required, as SlapOS do not yet provide nicely working
@@ -24,11 +25,10 @@ unzip = true
 
 # Local development
 [slapos.cookbook-repository]
-recipe = plone.recipe.command
-stop-on-error = true
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-command = ${git:location}/bin/git clone --branch cloudooo --quiet http://git.erp5.org/repos/slapos.git ${:location}
-update-command = cd ${:location} && ${git:location}/bin/git pull --quiet
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.git
+branch = master
+git-executable = ${git:location}/bin/git
 
 [check-recipe]
 recipe = plone.recipe.command
@@ -36,6 +36,14 @@ stop-on-error = true
 update-command = ${:command}
 command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
 
+[slap-parameters]
+recipe = slapos.cookbook:slapconfiguration
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
 [template-jinja2-base]
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/${:filename}.in
@@ -54,7 +62,7 @@ context =
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 filename = template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 694205787e78c5d615d72d7b4b26d174
+md5sum = 425cb2e76d46d53bb0b0eebdb8c1aa95
 extra-context =
     key buildout_bin_directory buildout:bin-directory
     key dcron_location dcron:location
@@ -63,6 +71,7 @@ extra-context =
     key fonts_location fonts:location
     key freetype_location freetype:location
     key git_location git:location
+    key haproxy_location haproxy:location
     key imagemagick_location imagemagick:location
     key libICE_location libICE:location
     key libSM_location libSM:location
@@ -83,10 +92,9 @@ extra-context =
 [template-cloudooo]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-cloudoo.cfg.in
-md5sum = 4c8608f9525be0f01a09d60b240315a9
+md5sum = bbe84b4c9022db62c926e8a8a4bf02a1
 mode = 640
 
-
 [networkcache]
 # signature certificates of the following uploaders.
 #   Romain Courteaud

software/davstorage/common.cfg

+[buildout]
+
+versions = versions
+
+extends =
+  ../../component/apache-php/buildout.cfg  
+  ../../component/apache/buildout.cfg
+  ../../component/curl/buildout.cfg
+  ../../component/dash/buildout.cfg
+  ../../component/dcron/buildout.cfg
+  ../../component/logrotate/buildout.cfg
+  ../../stack/shacache-client.cfg
+  ../../component/lxml-python/buildout.cfg
+  ../../component/python-2.7/buildout.cfg
+  ../../component/gzip/buildout.cfg
+
+find-links +=
+  http://www.nexedi.org/static/packages/source/slapos.buildout/
+  http://www.nexedi.org/static/packages/source/
+  http://www.nexedi.org/static/packages/source/hexagonit.recipe.download/
+
+# Use only quite well working sites.
+allow-hosts +=
+  *.googlecode.com
+  *.nexedi.org
+  *.python.org
+  *.sourceforge.net
+  alastairs-place.net
+  bitbucket.org
+  dist.repoze.org
+  effbot.org
+  github.com
+  launchpad.net
+  peak.telecommunity.com
+  sourceforge.net
+  www.dabeaz.com
+  www.owlfish.com
+
+parts =
+  apache-php
+  application
+  template
+  lxml-python
+  eggs
+  instance-recipe-egg
+
+unzip= true
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+
+[instance-recipe]
+egg = slapos.cookbook
+module = davstorage
+
+[instance-recipe-egg]
+recipe = zc.recipe.egg
+python = python2.7
+eggs = ${instance-recipe:egg}
+
+[application]
+recipe = hexagonit.recipe.download
+url = http://garr.dl.sourceforge.net/project/ajaxplorer/ajaxplorer/dev-channel/4.3.4/ajaxplorer-core-4.3.4.tar.gz
+md5sum = 2f2ff8bda7bbe841ef0e870c724eb74f
+strip-top-level-dir = true
+
+[template]
+# Default template for the instance.
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg
+md5sum = bed788dee6daf05349c4577e7a7f1299
+output = ${buildout:directory}/template.cfg
+mode = 0644
+
+[instance-davstorage]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-davstorage.cfg
+md5sum = 699ecf4678386667f58a3391bab7af0f
+output = ${buildout:directory}/template-davstorage.cfg
+mode = 0644
+
+[lxml-python]
+python = python2.7

software/davstorage/development.cfg

+[buildout]
+extends =
+  ../../component/git/buildout.cfg
+  common.cfg
+
+parts +=
+  slapos.cookbook-repository
+  slapos.toolbox-repository
+  check-recipe
+
+develop =
+  ${:parts-directory}/slapos.cookbook-repository
+  ${:parts-directory}/slapos.toolbox-repository
+
+[slapos.cookbook-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.git
+branch = davstorage-ajaxplorer
+git-executable = ${git:location}/bin/git
+
+[slapos.toolbox-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.toolbox.git
+branch = master
+git-executable = ${git:location}/bin/git
+
+[check-recipe]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command =
+  grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
+  grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
+
+[versions]
+zc.buildout = 1.6.0-dev-SlapOS-002
+Jinja2 = 2.6
+Werkzeug = 0.8.3
+buildout-versions = 1.7
+hexagonit.recipe.cmmi = 1.6
+hexagonit.recipe.download = 1.6nxd002
+meld3 = 0.6.10
+openssl = 1.0.1c
+# Required by:
+# slapos.core
+Flask = 0.9
+
+# Required by:
+# slapos.cookbook
+PyXML = 0.8.4
+
+slapos.recipe.template = 2.4.2
+
+# Required by:
+# slapos.cookbook==0.24
+# slapos.core==0.14
+# xml-marshaller==0.9.7
+lxml = 3.1.0
+
+# Required by:
+# slapos.cookbook==0.24
+netaddr = 0.7.10
+
+# Required by:
+# slapos.core==0.14
+netifaces = 0.8
+
+# Required by:
+# slapos.cookbook==0.24
+# slapos.core==0.14
+# zc.buildout==1.5.3-dev-SlapOS-009
+# zc.recipe.egg==1.3.2
+setuptools = 0.6c12dev-r88846
+
+# Required by:
+# slapos.cookbook==0.73.1
+slapos.core = 0.35.1
+
+# Required by:
+# slapos.core==0.35.1
+supervisor = 3.0b1
+
+# Required by:
+# slapos.cookbook==0.73.1
+xml-marshaller = 0.9.7
+
+# Required by:
+# slapos.cookbook==0.24
+zc.recipe.egg = 1.3.2
+
+# Required by:
+# slapos.core==0.35.1
+zope.interface = 4.0.5
+

software/davstorage/instance-davstorage.cfg

 [buildout]
 parts =
   davstorage
-  url
+  publish-connection-informations
   certificate-authority
   ca-davstorage
   cron
   cron-entry-logrotate
   logrotate
   logrotate-entry-davstorage
+  request-frontend
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
-[url]
-recipe = slapos.cookbook:publishurl
-scheme = webdavs
-user = $${davstorage:user}
-port = $${davstorage:port}
-host = $${davstorage:ip}
-password = $${davstorage:password}
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+webdav_access = https://$${request-frontend:connection-domain}
+url = https://$${request-frontend-ajaxupload:connection-domain}
+webdav_user = $${davstorage:user}
+webdav_password = $${davstorage:password}
 
 [davstorage]
 recipe = slapos.cookbook:davstorage
 user = user
-port = 8080
+password = insecure
+port_webdav = 8080
+port_ajax = 8070
 ip = $${slap-network-information:global-ipv6}
 
 # Path
@@ -42,12 +44,15 @@ root = $${buildout:directory}
 email-address = admin+davstorage@vifib.net
 htpasswd-file = $${directory:davstorage-conf}/davstorage.htpasswd
 promise = $${basedirectory:promises}/davstorage
+php-ini-dir = $${directory:php-ini-dir}
+tmp-dir = $${directory:tmp-php}
 
 # Binaries
 apache-binary = ${apache:location}/bin/httpd
 apache-modules-dir = ${apache:location}/modules/
 apache-mime-file = ${apache:location}/conf/mime.types
 apache-htpasswd = ${apache:location}/bin/htpasswd
+source = ${application:location}
 
 [certificate-authority]
 recipe = slapos.cookbook:certificate_authority
@@ -66,7 +71,7 @@ dcrond-binary = ${dcron:location}/sbin/crond
 cron-entries = $${directory:cron-entries}
 crontabs = $${directory:crontabs}
 cronstamps = $${directory:cronstamps}
-catcher = $${cron-simplelogger:binary}
+catcher = $${cron-simplelogger:wrapper}
 binary = $${basedirectory:services}/crond
 
 [logrotate]
@@ -91,8 +96,8 @@ command = $${logrotate:wrapper}
 
 [cron-simplelogger]
 recipe = slapos.cookbook:simplelogger
-binary = $${rootdirectory:bin}/cron_simplelogger
-output = $${directory:cronoutput}
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
 
 
 [logrotate-entry-davstorage]
@@ -128,6 +133,7 @@ etc = $${buildout:directory}/etc/
 var = $${buildout:directory}/var/
 srv = $${buildout:directory}/srv/
 bin = $${buildout:directory}/bin/
+tmp = $${buildout:directory}/tmp/
 
 [basedirectory]
 recipe = slapos.cookbook:mkdirectory
@@ -149,3 +155,38 @@ cron-entries = $${rootdirectory:etc}/cron.d/
 crontabs = $${rootdirectory:etc}/crontabs/
 cronstamps = $${rootdirectory:etc}/cronstamps/
 cronoutput = $${basedirectory:log}/cron/
+php-ini-dir = $${rootdirectory:etc}/php
+tmp-php = $${rootdirectory:tmp}/php
+
+# Request frontend
+[request-frontend-ajaxupload]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Frontend Ajax
+# XXX We have hardcoded SR URL here.
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config = url https-only
+config-https-only = true
+config-url = https://[$${davstorage:ip}]:$${davstorage:port_ajax}/
+return = domain
+
+[request-frontend]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Frontend Webdav
+# XXX We have hardcoded SR URL here.
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config = url https-only
+config-https-only = true
+config-url = https://$${davstorage:user}:$${davstorage:password}@[$${davstorage:ip}]:$${davstorage:port_webdav}/
+return = domain
+
+# XXX Vivien: promise not working for now
+#[frontend-ajaxupload-promise]
+#recipe = slapos.cookbook:check_url_available
+#path = $${basedirectory:promises}/frontend-ajaxupload
+#url = $${request-frontend-ajaxupload:connection-site_url}
+#dash_path = ${dash:location}/bin/dash
+#curl_path = ${curl:location}/bin/curl
\ No newline at end of file

software/davstorage/software.cfg

 [buildout]
-
-find-links +=
-    http://www.nexedi.org/static/packages/source/slapos.buildout/
-
-versions = versions
-
 extends =
-  ../../component/apache/buildout.cfg
-  ../../component/dcron/buildout.cfg
-  ../../component/logrotate/buildout.cfg
-  ../../stack/shacache-client.cfg
-  ../../component/lxml-python/buildout.cfg
-  ../../component/python-2.7/buildout.cfg
-  ../../component/gzip/buildout.cfg
-
-# Use only quite well working sites.
-allow-hosts =
-  *.nexedi.org
-  *.python.org
-  *.sourceforge.net
-  dist.repoze.org
-  effbot.org
-  github.com
-  peak.telecommunity.com
-  psutil.googlecode.com
-  www.dabeaz.com
-  alastairs-place.net
-
-
-parts =
-  template
-  lxml-python
-  apache
-  logrotate
-  dcron
-  eggs
-  gzip
-  instance-recipe-egg
-
-unzip= true
-
-[eggs]
-recipe = zc.recipe.egg
-eggs =
-
-[instance-recipe]
-egg = slapos.cookbook
-module = davstorage
-
-[instance-recipe-egg]
-recipe = zc.recipe.egg
-python = python2.7
-eggs = ${instance-recipe:egg}
-
-[template]
-# Default template for the instance.
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg
-md5sum = bed788dee6daf05349c4577e7a7f1299
-output = ${buildout:directory}/template.cfg
-mode = 0644
-
-[instance-davstorage]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-davstorage.cfg
-md5sum = 49c8e049a78e233d3a553e64c8914592
-output = ${buildout:directory}/template-davstorage.cfg
-mode = 0644
-
-[lxml-python]
-python = python2.7
+  ../../component/git/buildout.cfg
+  common.cfg
+
+parts +=
+  slapos.cookbook-repository
+  slapos.toolbox-repository
+  check-recipe
+
+develop =
+  ${:parts-directory}/slapos.cookbook-repository
+  ${:parts-directory}/slapos.toolbox-repository
+
+[slapos.cookbook-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.git
+branch = davstorage-ajaxplorer
+git-executable = ${git:location}/bin/git
+
+[slapos.toolbox-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.toolbox.git
+branch = master
+git-executable = ${git:location}/bin/git
+
+[check-recipe]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command =
+  grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
+  grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
 
 [versions]
 zc.buildout = 1.6.0-dev-SlapOS-002
 Jinja2 = 2.6
-Werkzeug = 0.7.1
-buildout-versions = 1.6
-hexagonit.recipe.cmmi = 1.5.0
+Werkzeug = 0.8.3
+buildout-versions = 1.7
+hexagonit.recipe.cmmi = 1.6
 hexagonit.recipe.download = 1.6nxd002
-meld3 = 0.6.7
-slapos.cookbook = 0.26
-
+meld3 = 0.6.10
+openssl = 1.0.1c
 # Required by:
-# slapos.core==0.14
-Flask = 0.7.2
+# slapos.core
+Flask = 0.9
 
 # Required by:
-# slapos.cookbook==0.24
+# slapos.cookbook
 PyXML = 0.8.4
 
-# Required by:
-# slapos.recipe.template==1.1
-collective.recipe.template = 1.9
+slapos.recipe.template = 2.4.2
 
 # Required by:
 # slapos.cookbook==0.24
 # slapos.core==0.14
 # xml-marshaller==0.9.7
-lxml = 2.3
+lxml = 3.1.0
 
 # Required by:
 # slapos.cookbook==0.24
-netaddr = 0.7.6
+netaddr = 0.7.10
 
 # Required by:
 # slapos.core==0.14
-#netifaces = 0.4
+netifaces = 0.8
 
 # Required by:
 # slapos.cookbook==0.24
@@ -115,15 +73,15 @@ netaddr = 0.7.6
 setuptools = 0.6c12dev-r88846
 
 # Required by:
-# slapos.cookbook==0.24
-slapos.core = 0.14
+# slapos.cookbook==0.73.1
+slapos.core = 0.35.1
 
 # Required by:
-# slapos.core==0.14
-supervisor = 3.0a10
+# slapos.core==0.35.1
+supervisor = 3.0b1
 
 # Required by:
-# slapos.cookbook==0.24
+# slapos.cookbook==0.73.1
 xml-marshaller = 0.9.7
 
 # Required by:
@@ -131,5 +89,6 @@ xml-marshaller = 0.9.7
 zc.recipe.egg = 1.3.2
 
 # Required by:
-# slapos.core==0.14
-zope.interface = 3.7.0
+# slapos.core==0.35.1
+zope.interface = 4.0.5
+

software/davstorage/todo.txt

+TODO:
+
++ Move Ajaxplorer to a separate directory (currently in the www/ directory
+  accessed by webdav client, and so might delete files)
+
++ Make configuration part disappear ! (configuration when instance deploy, 
+  no need for user to do it at each start of a new instance)

software/erp5testnode/instance-default.cfg

@@ -7,7 +7,6 @@ offline = true
 parts =
   connection-dict
   testnode
-  pwgen
   shell
   shellinabox
   certificate-authority
@@ -16,12 +15,11 @@ parts =
 [connection-dict]
 recipe = slapos.cookbook:publish
 url = http://[$${shellinabox:ipv6}]:$${shellinabox:port}/
-password = $${pwgen:password}
+password = $${pwgen:passwd}
 
 [pwgen]
-recipe = slapos.cookbook:pwgen
-file = $${buildout:directory}/.password
-pwgen-binary = ${pwgen:location}/bin/pwgen
+recipe = slapos.cookbook:generate.password
+storage-path = $${buildout:directory}/.password
 
 [testnode]
 recipe = slapos.cookbook:erp5testnode
@@ -82,7 +80,7 @@ port = 8080
 shell = $${shell:wrapper}
 wrapper = $${rootdirectory:bin}/shellinaboxd
 shellinabox-binary = ${shellinabox:location}/bin/shellinaboxd
-password = $${pwgen:password}
+password = $${pwgen:passwd}
 directory = $${buildout:directory}/
 login-shell = $${rootdirectory:bin}/login
 certificate-directory = $${directory:shellinabox}

software/erp5testnode/software.cfg

@@ -20,7 +20,6 @@ extends =
   ../../component/zip/buildout.cfg
   ../../component/busybox/buildout.cfg
   ../../component/shellinabox/buildout.cfg
-  ../../component/pwgen/buildout.cfg
 
 # Local development
 develop =

software/etherpad-lite/common.cfg

+[buildout]
+
+versions = versions
+
+extends =
+  ../../stack/slapos.cfg
+  ../../component/gcc/buildout.cfg
+  ../../component/openssl/buildout.cfg
+  ../../component/curl/buildout.cfg
+  ../../component/dash/buildout.cfg
+  ../../component/dcron/buildout.cfg
+  ../../component/logrotate/buildout.cfg
+  ../../component/lxml-python/buildout.cfg
+  ../../component/python-2.7/buildout.cfg
+  ../../component/gzip/buildout.cfg
+  ../../component/git/buildout.cfg
+  ../../component/nodejs/buildout.cfg
+  ../../component/postgresql/buildout.cfg
+
+parts =
+  postgresql
+  nodejs
+  etherpad-lite-repository
+  install-deps
+  template
+  lxml-python
+  eggs
+  instance-recipe-egg
+
+unzip= true
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+
+[instance-recipe]
+egg = slapos.cookbook
+module = etherpad-lite
+
+[instance-recipe-egg]
+recipe = zc.recipe.egg
+python = python2.7
+eggs = ${instance-recipe:egg}
+
+[etherpad-lite-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://github.com/ether/etherpad-lite.git
+branch = develop
+git-executable = ${git:location}/bin/git
+
+[template]
+# Default template for the instance.
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg
+md5sum = 7ab2a242df988bf5c10bf8002acac3bd
+output = ${buildout:directory}/template.cfg
+mode = 0644
+
+[instance-etherpad-lite]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-etherpad-lite.cfg
+md5sum = fd7249be8988155110234c7bb877abb9
+output = ${buildout:directory}/template-etherpad-lite.cfg
+mode = 0644
+
+[template-conf]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/templates/${:filename}
+mode = 0644
+filename = settings.json.in
+md5sum = 19ab39e6b3256c82fd54ce074488b136
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+
+[template-run-script]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/templates/${:filename}
+mode = 0644
+filename = run.sh.in
+md5sum = eac870b5f30e735e109a48913af2fae3
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+
+[template-deps-script]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/templates/${:filename}
+etherpad-location = ${etherpad-lite-repository:location}
+nodejs-location = ${nodejs:location}
+curl-location = ${curl:location}
+postgre-location = ${postgresql:location}
+mode = 0755
+md5sum = 53d0d53d419bd9ee592d3e1a1c84c758
+filename = installDeps.sh.in
+output = ${etherpad-lite-repository:location}/bin/installDeps.sh
+
+[install-deps]
+recipe = plone.recipe.command
+command = ${template-deps-script:output}
+update-command = command
+
+[lxml-python]
+python = python2.7

software/etherpad-lite/development.cfg

+[buildout]
+extends =
+  common.cfg
+
+parts +=
+  slapos.cookbook-repository
+  slapos.toolbox-repository
+  check-recipe
+
+develop =
+  ${:parts-directory}/slapos.cookbook-repository
+  ${:parts-directory}/slapos.toolbox-repository
+
+[slapos.cookbook-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.git
+branch = etherpad-lite
+git-executable = ${git:location}/bin/git
+
+[slapos.toolbox-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.toolbox.git
+branch = master
+git-executable = ${git:location}/bin/git
+
+[check-recipe]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command =
+  grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
+  grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
\ No newline at end of file

software/etherpad-lite/instance-etherpad-lite.cfg

+[buildout]
+parts =
+  etherpad-lite
+  publish-connection-informations
+#  frontend-etherpad
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc/
+var = $${buildout:directory}/var/
+srv = $${buildout:directory}/srv/
+bin = $${buildout:directory}/bin/
+tmp = $${buildout:directory}/tmp/
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log/
+services = $${rootdirectory:etc}/run/
+run = $${rootdirectory:etc}/run/
+backup = $${rootdirectory:srv}/backup/
+promises = $${rootdirectory:etc}/promise/
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etherpad-conf = $${rootdirectory:etc}/etherpad/
+etherpad-repository-location = $${buildout:directory}/parts/etherpad-lite-repository
+
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+url = $${request-frontend:connection-site_url}
+
+[etherpad-conf-generation]
+recipe = slapos.recipe.template
+url = ${template-conf:location}/${template-conf:filename}
+ip = $${slap-network-information:global-ipv6}
+dirtydb-location = $${rootdirectory:var}/dirty.db
+port = 9001
+mode = 0644
+output = $${directory:etherpad-conf}/settings.json
+
+[etherpad-run-script]
+recipe = slapos.recipe.template
+url = ${template-run-script:location}/${template-run-script:filename}
+etherpad-location = ${etherpad-lite-repository:location}
+etherpad-repository-location = $${directory:etherpad-repository-location}
+nodejs-location = ${nodejs:location}
+etherpad-deps-script-location = ${template-deps-script:output}
+etherpad-conf-location = $${etherpad-conf-generation:output}
+etherpad-conf-name = settings-$${slap-connection:partition-id}.json
+mode = 0755
+output = $${rootdirectory:bin}/run.sh
+
+# Command line comes from the run script of etherpad-lite
+[etherpad-lite]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${basedirectory:run}/etherpad-lite 
+command-line = $${etherpad-run-script:output} -s $${etherpad-run-script:etherpad-conf-name}
+
+[request-frontend]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Frontend
+# XXX We have hardcoded SR URL here.
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config = url
+config-url = http://$${etherpad-conf-generation:ip}:$${etherpad-conf-generation:port}
+return = site_url
+
+[frontend-etherpad]
+recipe = slapos.cookbook:check_url_available
+path = $${basedirectory:promises}/frontend-etherpad
+url = $${request-frontend:connection-site_url}
+dash_path = ${dash:location}/bin/dash
+curl_path = ${curl:location}/bin/curl
\ No newline at end of file

software/etherpad-lite/instance.cfg

+[buildout]
+
+parts =
+  switch_softwaretype
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[switch_softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = ${instance-etherpad-lite:output}

software/etherpad-lite/software.cfg

+[buildout]
+extends =
+  common.cfg
+
+parts +=
+  slapos.cookbook-repository
+  slapos.toolbox-repository
+  check-recipe
+
+develop =
+  ${:parts-directory}/slapos.cookbook-repository
+  ${:parts-directory}/slapos.toolbox-repository
+
+[slapos.cookbook-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.git
+branch = etherpad-lite
+git-executable = ${git:location}/bin/git
+
+[slapos.toolbox-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.toolbox.git
+branch = master
+git-executable = ${git:location}/bin/git
+
+[check-recipe]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command =
+  grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
+  grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
\ No newline at end of file

software/etherpad-lite/templates/installDeps.sh.in

+#!/bin/sh
+
+# Ugly setup
+cd ${:etherpad-location}
+export PATH=$PATH:${:postgre-location}/bin
+
+#Is gnu-grep (ggrep) installed on SunOS (Solaris)
+if [ $(uname) = "SunOS" ]; then
+  hash ggrep > /dev/null 2>&1 || { 
+    echo "Please install ggrep (pkg install gnu-grep)" >&2
+    exit 1 
+  }
+fi
+
+#Is wget installed?
+hash ${:curl-location}/bin/curl > /dev/null 2>&1 || { 
+  echo "Please install curl" >&2
+  exit 1 
+}
+
+#Is node installed?
+hash ${:nodejs-location}/bin/node > /dev/null 2>&1 || { 
+  echo "Please install node.js ( http://nodejs.org )" >&2
+  exit 1 
+}
+
+#Is npm installed?
+hash ${:nodejs-location}/bin/npm > /dev/null 2>&1 || { 
+  echo "Please install npm ( http://npmjs.org )" >&2
+  exit 1 
+}
+
+echo "Ensure that all dependencies are up to date...  If this is the first time you have run Etherpad please be patient."
+(
+  mkdir -p node_modules
+  cd node_modules
+  [ -e ep_etherpad-lite ] || ln -s ../src ep_etherpad-lite
+  cd ep_etherpad-lite
+  ${:nodejs-location}/bin/npm install --loglevel warn
+) || { 
+  rm -rf node_modules
+  exit 1 
+}
+
+echo "Ensure jQuery is downloaded and up to date..."
+DOWNLOAD_JQUERY="true"
+NEEDED_VERSION="1.9.1"
+
+if [ $DOWNLOAD_JQUERY = "true" ]; then
+  ${:curl-location}/bin/curl -lo src/static/js/jquery.js http://code.jquery.com/jquery-$NEEDED_VERSION.js || exit 1
+fi
+
+#Remove all minified data to force node creating it new
+echo "Clear minfified cache..."
+rm -f var/minified*
+
+echo "ensure custom css/js files are created..."
+
+for f in "index" "pad" "timeslider"
+do
+  if [ ! -f "src/static/custom/$f.js" ]; then
+    cp "src/static/custom/js.template" "src/static/custom/$f.js" || exit 1
+  fi
+  
+  if [ ! -f "src/static/custom/$f.css" ]; then
+    cp "src/static/custom/css.template" "src/static/custom/$f.css" || exit 1
+  fi
+done
+
+exit 0

software/etherpad-lite/templates/run.sh.in

+#!/bin/sh
+
+# Copy repository content from the software release
+mkdir ${:etherpad-repository-location}
+cp -R ${:etherpad-location}/* ${:etherpad-repository-location}
+
+#Move to the folder where ep-lite is installed
+cd ${:etherpad-repository-location}
+
+# XXX Vivien: very ugly, definitely need to find a cleaner way
+cp ${:etherpad-conf-location} ${:etherpad-repository-location}/${:etherpad-conf-name}
+
+#Move to the node folder and start
+echo "start..."
+${:nodejs-location}/bin/node node_modules/ep_etherpad-lite/node/server.js $*

software/etherpad-lite/templates/settings.json.in

+/*
+  This file must be valid JSON. But comments are allowed
+
+  Please edit settings.json, not settings.json.template
+*/
+{
+  // Name your instance!
+  "title": "Etherpad",
+
+  // favicon default name
+  // alternatively, set up a fully specified Url to your own favicon
+  "favicon": "favicon.ico",
+  
+  //IP and port which etherpad should bind at
+  "ip": "${:ip}",
+  "port" : "${:port}",
+
+  // Session Key, used for reconnecting user sessions
+  // Set this to a secure string at least 10 characters long.  Do not share this value.
+  "sessionKey" : "",
+
+  /*  
+  // Node native SSL support
+  // this is disabled by default
+  //
+  // make sure to have the minimum and correct file access permissions set
+  // so that the Etherpad server can access them
+
+  "ssl" : {
+            "key"  : "/path-to-your/epl-server.key",
+            "cert" : "/path-to-your/epl-server.crt"
+          },
+
+  */
+
+  //The Type of the database. You can choose between dirty, postgres, sqlite and mysql
+  //You shouldn't use "dirty" for for anything else than testing or development
+  "dbType" : "dirty",
+  //the database specific settings
+  "dbSettings" : {
+                   "filename" : "${:dirtydb-location}"
+                 },
+                 
+  /* An Example of MySQL Configuration
+   "dbType" : "mysql",
+   "dbSettings" : {
+                    "user"    : "root", 
+                    "host"    : "localhost", 
+                    "password": "", 
+                    "database": "store"
+                  },
+  */
+  
+  //the default text of a pad
+  "defaultPadText" : "Welcome to Etherpad!\n\nThis pad text is synchronized as you type, so that everyone viewing this page sees the same text. This allows you to collaborate seamlessly on documents!\n\nGet involved with Etherpad at http:\/\/etherpad.org\n",
+  
+  /* Users must have a session to access pads. This effectively allows only group pads to be accessed. */
+  "requireSession" : false,
+
+  /* Users may edit pads but not create new ones. Pad creation is only via the API. This applies both to group pads and regular pads. */
+  "editOnly" : false,
+  
+  /* if true, all css & js will be minified before sending to the client. This will improve the loading performance massivly, 
+     but makes it impossible to debug the javascript/css */
+  "minify" : true,
+
+  /* How long may clients use served javascript code (in seconds)? Without versioning this
+     may cause problems during deployment. Set to 0 to disable caching */
+  "maxAge" : 21600, // 60 * 60 * 6 = 6 hours
+  
+  /* This is the path to the Abiword executable. Setting it to null, disables abiword.
+     Abiword is needed to advanced import/export features of pads*/  
+  "abiword" : null,
+ 
+  /* This setting is used if you require authentication of all users.
+     Note: /admin always requires authentication. */
+  "requireAuthentication": false,
+
+  /* Require authorization by a module, or a user with is_admin set, see below. */
+  "requireAuthorization": false,
+
+  /* Users for basic authentication. is_admin = true gives access to /admin.
+     If you do not uncomment this, /admin will not be available! */
+  /*
+  "users": {
+    "admin": {
+      "password": "changeme1",
+      "is_admin": true
+    },
+    "user": {
+      "password": "changeme1",
+      "is_admin": false
+    }
+  },
+  */
+
+  // restrict socket.io transport methods
+  "socketTransportProtocols" : ["xhr-polling"],
+  
+  /* The log level we are using, can be: DEBUG, INFO, WARN, ERROR */
+  "loglevel": "INFO",
+  
+  //Logging configuration. See log4js documentation for further information
+  // https://github.com/nomiddlename/log4js-node
+  // You can add as many appenders as you want here:
+  "logconfig" :
+	{ "appenders": [
+	    { "type": "console",
+      //  "category": "access"// only logs pad access
+	    }
+    /*
+	  , { "type": "file"
+      , "filename": "your-log-file-here.log"
+      , "maxLogSize": 1024
+      , "backups": 3 // how many log files there're gonna be at max
+      //, "category": "test" // only log a specific category
+      }*/
+    
+	    /*{ "type": "logLevelFilter",
+	      "level": "warn" // filters out all log messages that have a lower level than "error"
+	    , "appender":
+	      {  Use whatever appender you want here  }
+      }*/
+    /*
+	  , { "type": "logLevelFilter"
+	    , "level": "error" // filters out all log messages that have a lower level than "error"
+	    , "appender":
+	      { "type": "smtp"
+        , "subject": "An error occured in your EPL instance!"
+        , "recipients": "bar@blurdybloop.com, baz@blurdybloop.com"
+        , "sendInterval": 60*5 // in secs -- will buffer log messages; set to 0 to send a mail for every message
+        , "transport": "SMTP", "SMTP": { // see https://github.com/andris9/Nodemailer#possible-transport-methods
+            "host": "smtp.example.com", "port": 465,
+            "secureConnection": true,
+            "auth": {
+                "user": "foo@example.com",
+                "pass": "bar_foo"
+            }
+        }
+        }
+      }*/
+	] }
+}

software/etherpad-lite/todo.txt

+TODO:
+
++ Modify etherpad-lite to read absolute path correctly, so that we aren't
+  forced to move everything to the software release directory for it to work

software/gitrepo/instance-gitrepo.cfg

@@ -13,14 +13,13 @@ parts =
   gitdaemon
   git-http-backend-cgi
   htpasswd
-  pwgen
   git-repos
 
 [publish]
 recipe = slapos.cookbook:publish
 url = http://[$${slap-network-information:global-ipv6}]:$${httpd-conf:port}/
 user = $${pwgen:user}
-password = $${pwgen:password}
+password = $${pwgen:passwd}
 
 [httpd]
 recipe = slapos.cookbook:wrapper
@@ -79,14 +78,12 @@ output = $${basedirectory:services}/git-daemon
 recipe = collective.recipe.cmd
 output = $${rootdirectory:etc}/httpd.htpasswd
 on_install = true
-on_udptae = true
+on_update = true
 cmds =
-  ${apache:location}/bin/htpasswd -cb $${:output} $${pwgen:user} $${pwgen:password}
+  ${apache:location}/bin/htpasswd -cb $${:output} $${pwgen:user} $${pwgen:passwd}
 
 [pwgen]
-recipe = slapos.cookbook:pwgen
-file = $${buildout:directory}/.password
-pwgen-binary = ${pwgen:location}/bin/pwgen
+recipe = slapos.cookbook:generate.password
 user = slapos
 
 [rootdirectory]

software/gitrepo/software.cfg

@@ -4,7 +4,6 @@ extends =
   ../../component/apache/buildout.cfg
   ../../component/perl/buildout.cfg
   ../../component/git/buildout.cfg
-  ../../component/pwgen/buildout.cfg
   ../../stack/slapos.cfg
 
 parts =

software/kvm/common.cfg

@@ -9,13 +9,18 @@ extends =
   ../../component/logrotate/buildout.cfg
   ../../component/noVNC/buildout.cfg
   ../../component/openssl/buildout.cfg
+  ../../component/dcron/buildout.cfg
   ../../stack/nodejs.cfg
+  ../../stack/resilient/buildout.cfg
   ../../stack/slapos.cfg
 
 parts =
   template
   eggs
 
+# XXX: we have to manually add this for resilience
+  rdiff-backup
+
 #XXX-Cedric : Currently, one can only access to KVM using noVNC.
 #             Ideally one should be able to access KVM by using either NoVNC or VNC.
 #             Problem is : no native crypto support in web browsers. So we have to disable ssl
@@ -67,13 +72,58 @@ command =
   ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install pkginfo@0.2.3
 
 
+# Create all templates that will be used to deploy instances
+
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg.in
+md5sum = 0e84223169661462f439c164d62c2a6a
+output = ${buildout:directory}/template.cfg
+mode = 0644
+
 [template-kvm]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-kvm.cfg.in
-md5sum = 87197471aa93863c310204e8865b5ac1
+md5sum = f7c0e2172dac4ee70daae50f38d610ef
 output = ${buildout:directory}/template-kvm.cfg
 mode = 0644
 
+[template-kvm-resilient]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja
+mode = 644
+md5sum = 3ee64c654aae503b93b39e9ccd6d3643
+
+[template-kvm-import]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-kvm-import.cfg.in
+md5sum = 7b36d6c61154b7ec3113a1bfaa25a904
+output = ${buildout:directory}/template-kvm-import.cfg
+mode = 0644
+
+[template-kvm-import-script]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/template/kvm-import.sh.in
+filename = kvm-import.sh.in
+md5sum = e03ed049cddd8d157228b09e1ebc071a
+download-only = true
+mode = 0755
+
+[template-kvm-export]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-kvm-export.cfg.in
+md5sum = 64a1a505aff9fde52afac46240811047
+output = ${buildout:directory}/template-kvm-export.cfg
+mode = 0644
+
+[template-kvm-export-script]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/template/kvm-export.sh.in
+filename = kvm-export.sh.in
+md5sum = 08cd8da2221f09095b14e35e6acd2a56
+download-only = true
+mode = 0755
+
 [template-nbd]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-nbd.cfg.in
@@ -87,10 +137,3 @@ url = ${:_profile_base_location_}/instance-frontend.cfg.in
 md5sum = cdb690495e9eb007d2b7d2f8e12f5c59
 output = ${buildout:directory}/template-frontend.cfg
 mode = 0644
-
-[template]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 0a98e34aaec7097a84066c0665e3a49a
-output = ${buildout:directory}/template.cfg
-mode = 0644

software/kvm/development.cfg

@@ -31,3 +31,6 @@ update-command = ${:command}
 command =
   grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
   grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
+
+[versions]
+websockify = 0.3.0
\ No newline at end of file

software/kvm/instance-kvm-export.cfg.in

+[buildout]
+extends = ${template-kvm:output}
+          ${pbsready-export:output}
+
+parts +=
+  cron-entry-backup
+
+  certificate-authority
+  publish-connection-information
+  kvm-promise
+  websockify-sighandler
+  novnc-promise
+  cron
+  frontend-promise
+
+# Create the exporter executable, which is a simple shell script
+[exporter]
+recipe = slapos.recipe.template
+url = ${template-kvm-export-script:location}/${template-kvm-export-script:filename}
+output = $${directory:bin}/$${slap-parameter:namebase}-exporter
+mode = 0755
+backup-disk-path = $${directory:backup}/virtual.qcow2
+# Resilient stack wants a "wrapper" parameter
+wrapper = $${:output}
+
+# Extends publish section with resilient parameters
+[publish-connection-information]
+<= resilient-publish-connection-parameter
+

software/kvm/instance-kvm-import.cfg.in

+[buildout]
+# Here, we don't need KVM to run to import data, so we don't
+# even extend the kvm instance profile.
+extends = ${pbsready-import:output}
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+bin = $${buildout:directory}/bin
+srv = $${buildout:directory}/srv
+var = $${buildout:directory}/var
+log = $${:var}/log
+scripts = $${:etc}/run
+services = $${:etc}/service
+promises = $${:etc}/promise
+novnc-conf = $${:etc}/novnc
+run = $${:var}/run
+ca-dir = $${:srv}/ssl
+cron-entries = $${:etc}/cron.d
+crontabs = $${:etc}/crontabs
+cronstamps = $${:etc}/cronstamps
+
+[importer]
+recipe = slapos.recipe.template
+url = ${template-kvm-import-script:location}/${template-kvm-import-script:filename}
+output = $${directory:bin}/$${slap-parameter:namebase}-importer
+mode = 0755
+backup-disk-path = $${directory:backup}/virtual.qcow2
+disk-path = $${directory:srv}/virtual.qcow2
+# Resilient stack wants a "wrapper" parameter
+wrapper = $${:output}
+
+backup-disk-path = $${directory:backup}/virtual.qcow2

software/kvm/instance-kvm-resilient.cfg.jinja2

+# vim: set ft=cfg:
+
+{% import 'parts' as parts %}
+{% import 'replicated' as replicated with context %}
+
+[buildout]
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
+parts +=
+  {{ parts.replicate("kvm", "2") }}
+  publish-connection-informations
+
+{{ replicated.replicate("kvm", "2", "kvm-export", "kvm-import") }}
+
+# Bubble down the parameters of the requested instance to the user
+[request-kvm]
+# Note: += doesn't work.
+return =
+# Resilient related parameters
+  url ssh-public-key ssh-url notification-id ip
+# KVM related parameters
+  backend-url url
+
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+backend-url = ${request-kvm:connection-backend-url}
+url = ${request-kvm:connection-url}
+

software/kvm/instance-kvm.cfg.in

@@ -6,10 +6,13 @@
 [buildout]
 parts =
   certificate-authority
-  publish-kvm-connection-information
+  publish-connection-information
   kvm-promise
   websockify-sighandler
   novnc-promise
+#  kvm-monitor
+  cron
+#  cron-entry-monitor
   frontend-promise
 
 eggs-directory = ${buildout:eggs-directory}
@@ -22,12 +25,16 @@ etc = $${buildout:directory}/etc
 bin = $${buildout:directory}/bin
 srv = $${buildout:directory}/srv
 var = $${buildout:directory}/var
+log = $${:var}/log
 scripts = $${:etc}/run
 services = $${:etc}/service
 promises = $${:etc}/promise
 novnc-conf = $${:etc}/novnc
 run = $${:var}/run
 ca-dir = $${:srv}/ssl
+cron-entries = $${:etc}/cron.d
+crontabs = $${:etc}/crontabs
+cronstamps = $${:etc}/cronstamps
 
 [create-mac]
 recipe = slapos.cookbook:generate.mac
@@ -44,6 +51,7 @@ bytes = 8
 recipe = slapos.cookbook:kvm
 vnc-ip = $${slap-network-information:local-ipv4}
 vnc-port = 5901
+default-disk-image = ${debian-amd64-netinst.iso:location}/${debian-amd64-netinst.iso:filename}
 nbd-host = $${slap-parameter:nbd-host}
 nbd-port = $${slap-parameter:nbd-port}
 nbd2-host = $${slap-parameter:nbd2-host}
@@ -122,11 +130,43 @@ hostname = $${novnc-instance:ip}
 port = $${novnc-instance:port}
 
 
-[kvm-monitor]
-recipe = slapos.cookbook:generic.slapmonitor
-db-path = $${directory:srv}/slapmonitor_database
+#[kvm-monitor]
+#recipe = slapos.cookbook:wrapper
+#wrapper-path = $${directory:services}/kvm_monitor
+#command-line = ${buildout:bin-directory}/kvm.monitor.test
+#             $${buildout:directory}/buildout-switch-softwaretype.cfg
+#             $${buildout:directory}/report.xml
+#             -s slap-parameter
+#             -opts disk-size ram-size cpu-count
 
 
+
+#----------------
+#--
+#-- Deploy cron.
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${directory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${directory:bin}/cron_simplelogger
+log = $${directory:log}/crond.log
+
+
+#[cron-entry-monitor]
+#<= cron
+#recipe = slapos.cookbook:cron.d
+#name = kvm_monitor
+#frequency = 0 0 * * *
+#command = $${kvm-monitor:wrapper-path}
+
 [request-slave-frontend]
 recipe = slapos.cookbook:requestoptional
 software-url = $${slap-parameter:frontend-software-url}
@@ -146,17 +186,16 @@ sla = instance_guid
 sla-instance_guid = $${slap-parameter:frontend-instance-guid}
 
 
-[publish-kvm-connection-information]
+[publish-connection-information]
 recipe = slapos.cookbook:publish
-backend-url = https://[$${novnc-instance:ip}]:$${novnc-instance:port}/vnc_auto.html?host=[$${novnc-instance:ip}]&port=$${novnc-instance:port}&encrypt=1
-password = $${kvm-instance:passwd}
-url = $${request-slave-frontend:connection-url}/vnc_auto.html?host=$${request-slave-frontend:connection-domainname}&port=$${request-slave-frontend:connection-port}&encrypt=1&path=$${request-slave-frontend:connection-resource}
+backend-url = https://[$${novnc-instance:ip}]:$${novnc-instance:port}/vnc_auto.html?host=[$${novnc-instance:ip}]&port=$${novnc-instance:port}&encrypt=1&password=$${kvm-instance:passwd}
+url = $${request-slave-frontend:connection-url}/vnc_auto.html?host=$${request-slave-frontend:connection-domainname}&port=$${request-slave-frontend:connection-port}&encrypt=1&path=$${request-slave-frontend:connection-resource}&password=$${kvm-instance:passwd}
 
 
 [frontend-promise]
 recipe = slapos.cookbook:check_url_available
 path = $${directory:promises}/frontend_promise
-url = $${publish-kvm-connection-information:url}
+url = $${publish-connection-information:url}
 dash_path = ${dash:location}/bin/dash
 curl_path = ${curl:location}/bin/curl
 
@@ -164,9 +203,9 @@ curl_path = ${curl:location}/bin/curl
 # Default values if not specified
 frontend-software-type = frontend
 frontend-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg
-
+frontend-instance-guid =
 nbd-port = 1024
-nbd-host = debian.nbd.vifib.net
+nbd-host =
 nbd2-port = 1024
 nbd2-host =
 

software/kvm/instance.cfg.in

@@ -4,7 +4,6 @@ parts =
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
-offline = true
 
 [switch-softwaretype]
 recipe = slapos.cookbook:softwaretype
@@ -13,11 +12,33 @@ kvm = ${template-kvm:output}
 nbd = ${template-nbd:output}
 frontend = ${template-frontend:output}
 
-[slap-connection]
-# part to migrate to new - separated words
-computer-id = $${slap_connection:computer_id}
-partition-id = $${slap_connection:partition_id}
-server-url = $${slap_connection:server_url}
-software-release-url = $${slap_connection:software_release_url}
-key-file = $${slap_connection:key_file}
-cert-file = $${slap_connection:cert_file}
+kvm-resilient = $${dynamic-template-kvm-resilient:rendered}
+kvm-import = ${template-kvm-import:output}
+kvm-export = ${template-kvm-export:output}
+
+frozen = ${instance-frozen:output}
+pull-backup = ${template-pull-backup:output}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[dynamic-template-kvm-resilient]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
+md5sum = 1b7a2fcc884649b4d08f238e828899c1
+rendered = $${buildout:directory}/template-kvm-resilient.cfg
+context = key buildout buildout:bin-directory
+          key develop_eggs_directory buildout:develop-eggs-directory
+          key eggs_directory buildout:eggs-directory
+          key slapparameter_dict slap-configuration:configuration
+template-parts-destination = ${template-parts:destination}
+template-replicated-destination = ${template-replicated:destination}
+import-list = file parts :template-parts-destination
+              file replicated :template-replicated-destination
+mode = 0644
+

software/kvm/software.cfg

@@ -47,98 +47,96 @@ signature-certificate-list =
   x2IMeSwJ82BpdEI5niXxB+iT0HxhmR+XaMI=
   -----END CERTIFICATE-----
 
-
 [versions]
-numpy = 1.6.2
-Jinja2 = 2.6
-Werkzeug = 0.8.3
-apache-libcloud = 0.12.1
+Werkzeug = 0.9.3
+apache-libcloud = 0.13.0
 async = 0.6.1
 buildout-versions = 1.7
 gitdb = 0.5.4
-hexagonit.recipe.cmmi = 1.6
-lxml = 3.1.0
+itsdangerous = 0.22
+lxml = 3.2.3
 meld3 = 0.6.10
 plone.recipe.command = 1.1
 pycrypto = 2.6
-slapos.cookbook = 0.73.1
+rdiff-backup = 1.0.5
+slapos.cookbook = 0.78.3
+slapos.recipe.cmmi = 0.2
+slapos.recipe.download = 1.0.dev-r4053
 slapos.recipe.template = 2.4.2
-slapos.toolbox = 0.33.1
+slapos.toolbox = 0.35.0
 smmap = 0.8.2
-websockify = 0.3.0
+websockify = 0.5.1
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
 # slapos.core==0.35.1
-# slapos.toolbox==0.33.1
-Flask = 0.9
+# slapos.toolbox==0.35.0
+Flask = 0.10.1
 
 # Required by:
-# slapos.toolbox==0.33.1
+# slapos.toolbox==0.35.0
 GitPython = 0.3.2.RC1
 
 # Required by:
-# slapos.toolbox==0.33.1
+# slapos.toolbox==0.35.0
 atomize = 0.1.1
 
 # Required by:
-# slapos.toolbox==0.33.1
+# slapos.toolbox==0.35.0
 feedparser = 5.1.3
 
 # Required by:
-# hexagonit.recipe.cmmi==1.6
-hexagonit.recipe.download = 1.6nxd002
+# slapos.cookbook==0.78.3
+inotifyx = 0.2.0-1
 
 # Required by:
-# slapos.cookbook==0.73.1
-inotifyx = 0.2.0
+# slapos.cookbook==0.78.3
+lock-file = 2.0
 
 # Required by:
-# slapos.cookbook==0.73.1
+# slapos.cookbook==0.78.3
 netaddr = 0.7.10
 
 # Required by:
 # slapos.core==0.35.1
-netifaces = 0.8
+netifaces = 0.8-1
 
 # Required by:
-# slapos.toolbox==0.33.1
-paramiko = 1.10.0
+# websockify==0.5.1
+numpy = 1.7.1
 
 # Required by:
-# slapos.toolbox==0.33.1
-psutil = 0.6.1
+# slapos.toolbox==0.35.0
+paramiko = 1.11.0
 
 # Required by:
-# slapos.core==0.35.1
-pyflakes = 0.6.1
+# slapos.toolbox==0.35.0
+psutil = 1.0.1
 
 # Required by:
-# slapos.cookbook==0.73.1
-pytz = 2012j
+# slapos.core==0.35.1
+pyflakes = 0.7.3
 
 # Required by:
-# slapos.cookbook==0.73.1
-# slapos.core==0.35.1
-# slapos.toolbox==0.33.1
-setuptools = 0.6c12dev-r88846
+# slapos.cookbook==0.78.3
+pytz = 2013b
 
 # Required by:
-# slapos.cookbook==0.73.1
-# slapos.toolbox==0.33.1
+# slapos.cookbook==0.78.3
+# slapos.toolbox==0.35.0
 slapos.core = 0.35.1
 
 # Required by:
 # slapos.core==0.35.1
-supervisor = 3.0b1
+supervisor = 3.0b2
 
 # Required by:
 # slapos.core==0.35.1
 unittest2 = 0.5.1
 
 # Required by:
-# slapos.cookbook==0.73.1
-# slapos.toolbox==0.33.1
+# slapos.cookbook==0.78.3
+# slapos.toolbox==0.35.0
 xml-marshaller = 0.9.7
 
 # Required by:

software/kvm/template/kvm-export.sh.in

+#!/bin/bash
+# Create a backup of the disk image of the virtual machine
+QEMU_IMG=${kvm-instance:qemu-img-path}
+SNAPSHOT_NAME=$(date +%s)
+DISK_PATH=${kvm-instance:disk-path}
+BACKUP_PATH=${:backup-disk-path}
+
+if [ ! -f $DISK_PATH ]; then
+  echo "Nothing to backup, disk image doesn't exist yet."
+  exit 0;
+fi
+
+$QEMU_IMG snapshot -c $SNAPSHOT_NAME $DISK_PATH
+if [ -f $BACKUP_PATH ]; then
+  rm $BACKUP_PATH
+fi
+$QEMU_IMG convert -f qcow2 -O qcow2 -s $SNAPSHOT_NAME $DISK_PATH $BACKUP_PATH
+$QEMU_IMG snapshot -d $SNAPSHOT_NAME $DISK_PATH

software/kvm/template/kvm-import.sh.in

+#!/bin/bash
+DISK_PATH=${:disk-path}
+BACKUP_PATH=${:backup-disk-path}
+
+# TODO: Use rdiff
+rm $DISK_PATH
+cp $BACKUP_PATH $DISK_PATH

software/lxc/instance-lxc.cfg

@@ -68,9 +68,8 @@ bridge = !!BRIDGE_NAME!!
 interface = lxc$${slap-network-information:network-interface}
 
 [passwd]
-recipe = slapos.cookbook:pwgen
-file = $${buildout:directory}/.password
-pwgen-binary = ${pwgen:location}/bin/pwgen
+recipe = slapos.cookbook:generate.password
+storage-path = $${buildout:directory}/.password
 
 [shellinabox]
 recipe = slapos.cookbook:shellinabox
@@ -79,7 +78,7 @@ port = 8080
 shell = ${lxc:location}/bin/lxc-console -n $${uuid:uuid}
 wrapper = $${rootdirectory:bin}/shellinaboxd_raw
 shellinabox-binary = ${shellinabox:location}/bin/shellinaboxd
-password = $${passwd:password}
+password = $${passwd:passwd}
 directory = $${buildout:directory}/
 login-shell = $${rootdirectory:bin}/login
 certificate-directory = $${directory:shellinabox}

software/lxc/software.cfg

@@ -10,7 +10,6 @@ extends =
   ../../component/xz-utils/buildout.cfg
   ../../component/tar/buildout.cfg
   ../../component/shellinabox/buildout.cfg
-  ../../component/pwgen/buildout.cfg
   ../../component/bash/buildout.cfg
   ../../component/coreutils/buildout.cfg
 
@@ -23,7 +22,6 @@ parts =
   slapos-toolbox
   lxc
   shellinabox
-  pwgen
 
 [template]
 recipe = slapos.recipe.template

software/neoppod/README.txt

@@ -37,11 +37,6 @@ Defautls to 12.
 Number of replicates.
 Defaults to 0 (no resilience).
 
-'verbose' (int, optional)
-~~~~~~~~~~~~~~~~~~~~~~~~~
-Non-zero to increase verbosity.
-Defaults to 0.
-
 'mysql-storage-count' (int, optional)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Number of MySQL-based storage nodes to deploy. One master node is deployed

software/neoppod/cluster.cfg.in

@@ -19,10 +19,9 @@ partition-id = ${slap-connection:partition-id}
 config-cluster = {{ slapparameter_dict['cluster'] }}
 config-partitions = {{ slapparameter_dict.get('partitions', 12) }}
 config-replicas = {{ slapparameter_dict.get('replicas', 0) }}
-config-verbose = {{ dumps(slapparameter_dict.get('verbose', 0)) }}
 
 [request-config]
-base = cluster verbose
+base = cluster
 master = partitions replicas
 node = masters