Merge remote-tracking branch 'slapos/lapp'

Conflicts:
	component/apache-perl/buildout.cfg
	component/postgresql/buildout.cfg
	software/mioga/instance.cfg
	software/mioga/software.cfg

CHANGES.txt

 Changes
 =======
 
+0.68.1 (2012-10-03)
+-------------------
+
+  * slaprunner: fix "logfile" parameter to "log_file"
+
+0.68 (2012-10-02)
+-----------------
+
+  * request.py: Remove useless calls to master, fix "update" method. [Cedric
+    de Saint Martin]
+  * Add webrunner test recipe. [Alain Takoudjou]
+  * Add logfile for slaprunner. [Cedric de Saint Martin]
+  * Fix check_url_available promise (syntax + checks + IPv6 support). [Cedric
+    de Saint Martin]
+
+0.67 (2012-09-26)
+-----------------
+
+  * Add check_page_content promise generator. [Cedric Le Ninivin]
+  * Fix check_url_available recipe. [Cedric de Saint Martin]
+  * Set up timezone database in mariab's mysql table so that we can use
+    timezone conversion function. [Kazuhiko Shiozaki]
+  * Add many resiliency-based recipes [Timothée Lacroix]
+  * Fix and unify request and requestoptional recipes [Cedric de Saint Martin]
+  * Fix Dropbear. [Antoine Catton]
+
 0.66 (2012-09-10)
 -----------------
 

component/apache-php/buildout.cfg

@@ -55,10 +55,50 @@ configure-options =
   --enable-bz2
   --enable-ftp
 
+
+# Changing TMPDIR is required for PEAR installation.
+# It will create a pear/temp directory under the SR instead of a shared /tmp/pear/temp.
+# XXX we could mkdir tmp there
+
 environment =
   PKG_CONFIG_PATH=${libxml2:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig
   PATH=${pkgconfig:location}/bin:${bzip2:location}/bin:${libxml2:location}/bin:%(PATH)s
   LDFLAGS =-L${bzip2:location}/lib -Wl,-rpath -Wl,${bzip2:location}/lib -L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${mariadb:location}/lib -Wl,-rpath -Wl,${mariadb:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${libmcrypt:location}/lib -Wl,-rpath -Wl,${libmcrypt:location}/libblkid
+  TMPDIR=${buildout:parts-directory}/${:_buildout_section_name_}
+
+
+
+
+[apache-php-postgres]
+<=apache-php
+configure-options =
+  --with-apxs2=${apache:location}/bin/apxs
+  --with-libxml-dir=${libxml2:location}
+  --with-zlib-dir=${zlib:location}
+  --with-bz2-dir=${bzip2:location}
+  --with-mcrypt=${libmcrypt:location}
+  --with-gd
+  --with-jpeg-dir=${libjpeg:location}
+  --with-png-dir=${libpng:location}
+  --enable-gd-native-ttf
+  --with-ttf
+  --with-freetype-dir=${freetype:location}
+  --with-curl=${curl:location}
+  --with-zip-dir=${zip:location}
+  --with-imap=${cclient:location}
+  --with-iconv-dir=${libiconv:location}
+  --with-gettext=${gettext:location}
+  --with-ldap=${openldap:location}
+  --with-imap-ssl
+  --with-openssl=${openssl:location}
+  --enable-libxml
+  --enable-mbstring
+  --enable-session
+  --enable-exif
+  --enable-zip
+  --enable-bz2
+  --enable-ftp
+  --with-pgsql=${postgresql:location}
 
 
 [libmcrypt]

component/gcc/buildout.cfg

@@ -170,4 +170,4 @@ make-targets = install -j1
 # until gcc will be simplified by using more robust build recipe (like
 # slapos.recipe.build) each time any of parts which reuses this one gets updated
 # the hack-revision have to be increased
-hack-revision = 1
+hack-revision = 2

component/ghostscript/buildout.cfg

@@ -29,5 +29,5 @@ environment =
 
 [ghostscript-9]
 <= ghostscript-common
-url = http://downloads.ghostscript.com/public/ghostscript-9.05.tar.bz2
-md5sum = 8bcef1f33ddf8a4d12b2cf8da385c191
+url = http://downloads.ghostscript.com/public/ghostscript-9.06.tar.bz2
+md5sum = 46f9ebe40dc52755287b30704270db11

component/groonga/buildout.cfg

@@ -7,12 +7,12 @@ parts =
 
 [groonga]
 recipe = hexagonit.recipe.cmmi
-url = http://packages.groonga.org/source/groonga/groonga-2.0.5.tar.gz
-md5sum = 6ea6634ceee97f701add89ee39f4c1da
+url = http://packages.groonga.org/source/groonga/groonga-2.0.6.tar.gz
+md5sum = 64467d1ab8d5e0df8348a636972a95a7
 configure-options =
   --disable-static
   --disable-glibtest
   --disable-benchmark
   --disable-document
-  --disable-nginx
+  --disable-groonga-httpd
   --without-mecab

component/haproxy/buildout.cfg

@@ -9,8 +9,8 @@ parts = haproxy
 
 [haproxy]
 recipe = hexagonit.recipe.cmmi
-url = http://haproxy.1wt.eu/download/1.4/src/haproxy-1.4.21.tar.gz
-md5sum = f36d5d13fd4a44ab454eee5d06cb38a2
+url = http://haproxy.1wt.eu/download/1.4/src/haproxy-1.4.22.tar.gz
+md5sum = a0b007c76f6a78524f3b3dd5e704979c
 configure-command = true
 # If the system is running on Linux 2.6, we use "linux26" as the TARGET,
 # otherwise use "generic".

component/java/buildout.cfg

@@ -29,9 +29,9 @@ slapos_promisee =
   directory:javaws
   file:lib/rt.jar
   file:bin/java
-# http://java.com/en/download/manual.jsp?locale=en
-x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=59621 7466b0b86bef21e3e31ae578b2b9f472
-x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=59623 c5cfb8393d257b51e7a70e56b7784ac9
+# http://java.com/en/download/manual_v6.jsp
+x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68284 7eda40e790de1a907d591b62949b6e72
+x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68286 839548714931443ba89719a995ece846
 script =
   import glob
   if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
@@ -47,48 +47,20 @@ script =
   workdir = guessworkdir(extract_dir)
   self.copyTree(glob.glob(os.path.join(workdir, "jre1.6.0_*"))[0], "%(location)s")
 
-[java-sdk-1.6.0]
+[java-re-1.7.0]
 recipe = slapos.recipe.build
 slapos_promisee =
   directory:bin
   directory:lib
   directory:man
   directory:plugin
-  directory:javaws
-  file:jre/lib/rt.jar
-  file:bin/java
-x86 = http://download.oracle.com/otn-pub/java/jdk/6u27-b07/jdk-6u27-linux-i586.bin bdb5f05bd20c6aa9a4729726191bf6fd
-x86-64 = http://download.oracle.com/otn-pub/java/jdk/6u27-b07/jdk-6u27-linux-x64.bin 94f93a3ff03f824a238ecd79ad90433e
-script =
-  if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
-  download_file = self.download(self.options['url'], self.options.get('md5sum'))
-  extract_dir = tempfile.mkdtemp(self.name)
-  os.chdir(extract_dir)
-  (download_dir, filename) = os.path.split(download_file)
-  auto_extract_bin = os.path.join(extract_dir, filename)
-  shutil.move(download_file, auto_extract_bin)
-  os.chmod(auto_extract_bin, 0777)
-  subprocess.call(["patch", auto_extract_bin, "-i", "${jdk-6u27-no-user-interaction-patch:location}/${jdk-6u27-no-user-interaction-patch:filename}"])
-  subprocess.call([auto_extract_bin])
-  self.cleanup_dir_list.append(extract_dir)
-  workdir = guessworkdir(extract_dir)
-  self.copyTree(os.path.join(workdir, "jdk1.6.0_27"), "%(location)s")
-
-[java-sdk-1.7.0]
-recipe = slapos.recipe.build
-slapos_promisee =
-  directory:bin
-  directory:lib
-  directory:man
-  directory:jre
-  file:jre/lib/rt.jar
+  file:lib/rt.jar
   file:bin/java
-  file:bin/javac
-x86 = http://download.oracle.com/otn-pub/java/jdk/7/jdk-7-linux-i586.tar.gz f97244a104f03731e5ff69f0dd5a9927
-x86-64 = http://download.oracle.com/otn-pub/java/jdk/7/jdk-7-linux-x64.tar.gz b3c1ef5faea7b180469c129a49762b64
+# http://java.com/en/download/manual.jsp?locale=en
+x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68234 ea99bedd9db33e9e2970f4b70abd1e4b
+x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68236 5aa9bd26cdf1fa6afd2b15826b4ba139
 script =
   if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
   extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
   workdir = guessworkdir(extract_dir)
   self.copyTree(workdir, "%(location)s")
-

component/mariadb/buildout.cfg

@@ -59,9 +59,11 @@ environment =
   LDFLAGS=-L${libaio:location}/lib
 
 [mroonga-mariadb]
+# mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
+# http://mroonga.github.com/
 recipe = hexagonit.recipe.cmmi
-url = https://github.com/downloads/mroonga/mroonga/mroonga-2.05.tar.gz
-md5sum = 6aee0e6ff948bba7c02854326afc11df
+url = https://github.com/downloads/mroonga/mroonga/mroonga-2.06.tar.gz
+md5sum = 89ce640a57ed8f2161b8358ff4c29c66
 configure-options =
   --with-mysql-source=${mariadb:location}__compile__/mariadb-${mariadb:version}
   --with-mysql-config=${mariadb:location}/bin/mysql_config

component/percona-toolkit/buildout.cfg

@@ -10,7 +10,7 @@ parts =
 recipe = hexagonit.recipe.cmmi
 depends =
   ${perl:version}
-url = http://www.percona.com/redir/downloads/percona-toolkit/2.0.4/percona-toolkit-2.0.4.tar.gz
-md5sum = df7dffcccb48d50f143849629228d4b4
+url = http://www.percona.com/redir/downloads/percona-toolkit/2.1.3/percona-toolkit-2.1.3.tar.gz
+md5sum = 6f8c4a0a0e43e467766cbdabaec9ef93
 configure-command =
   ${perl:location}/bin/perl Makefile.PL

component/poppler/buildout.cfg

@@ -13,8 +13,8 @@ extends =
 
 [poppler]
 recipe = hexagonit.recipe.cmmi
-md5sum = 45dd2c16c8c4d1a39e830e45745c4e25
-url = http://poppler.freedesktop.org/poppler-0.20.2.tar.gz
+md5sum = 28c40266f374e1960a7bcead17d39f96
+url = http://poppler.freedesktop.org/poppler-0.20.3.tar.gz
 depends =
   ${libtiff:version}
 configure-options =

component/postgresql/buildout.cfg

@@ -8,8 +8,8 @@ parts = postgresql
  
 [postgresql]
 recipe = hexagonit.recipe.cmmi
-url = http://ftp.postgresql.org/pub/source/v9.1.3/postgresql-9.1.3.tar.bz2
-md5sum = 641e1915f7ebfdc9f138e4c55b6aec0e
+url = http://ftp.postgresql.org/pub/source/v9.1.6/postgresql-9.1.6.tar.bz2
+md5sum = 000755f66c0de58bbd4cd2b89b45b8e2
 configure-options = --with-openssl
 environment =
   CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${openssl:location}/include -I${ncurses:location}/lib

component/slapos/buildout.cfg

+# This file is used to install SlapOS Node itself.
+
 [buildout]
 extends =
   ../../stack/shacache-client.cfg
@@ -112,6 +114,7 @@ scripts =
   slapgrid-supervisord = slapos.grid.svcbackend:supervisord
   slapproxy = slapos.proxy:main
   slapos = slapos.entry:main
+  slapos-watchdog = slapos.grid.watchdog:main
 
 [versions]
 zc.buildout = 1.6.0-dev-SlapOS-007
@@ -121,17 +124,17 @@ Werkzeug = 0.8.3
 buildout-versions = 1.7
 collective.recipe.template = 1.9
 hexagonit.recipe.cmmi = 1.6
-lxml = 3.0alpha2
+lxml = 2.3.6
 meld3 = 0.6.9
 netaddr = 0.7.10
-slapos.core = 0.28.7
+slapos.core = 0.31.1
 slapos.libnetworkcache = 0.13.2
 xml-marshaller = 0.9.7
 z3c.recipe.scripts = 1.0.1
 zc.recipe.egg = 1.3.2
 
 # Required by:
-# slapos.core==0.28.7
+# slapos.core==0.31.1
 Flask = 0.9
 
 # Required by:
@@ -139,21 +142,21 @@ Flask = 0.9
 hexagonit.recipe.download = 1.5.1
 
 # Required by:
-# slapos.core==0.28.7
+# slapos.core==0.31.1
 netifaces = 0.8
 
 # Required by:
-# slapos.core==0.28.7
+# slapos.core==0.31.1
 # slapos.libnetworkcache==0.13.2
 # supervisor==3.0b1
-# zc.buildout==1.6.0-dev-SlapOS-006
+# zc.buildout==1.6.0-dev-SlapOS-007
 # zope.interface==4.0.1
 setuptools = 0.6c12dev-r88846
 
 # Required by:
-# slapos.core==0.28.7
+# slapos.core==0.31.1
 supervisor = 3.0b1
 
 # Required by:
-# slapos.core==0.28.7
+# slapos.core==0.31.1
 zope.interface = 4.0.1

component/sphinx/README

+This component is not used anymore - but might be revived within 4 months.
+If not used within 6 months, feel free to delete this.

component/sphinx/buildout.cfg

@@ -20,8 +20,8 @@ filename = sphinx-1.10-beta-snowball.patch
 recipe = hexagonit.recipe.cmmi
 depends =
   ${mariadb:version}
-url = http://sphinxsearch.com/files/sphinx-2.0.2-beta.tar.gz
-md5sum = fafe0f1a71d0ded32404c067eba7d0b3
+url = http://sphinxsearch.com/files/sphinx-2.0.5-release.tar.gz
+md5sum = e71fdb5b0c2911247d48fb30550b9584
 configure-options =
   --with-mysql
   --with-mysql-includes=${mariadb:location}/include/mysql

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '0.67-dev'
+version = '0.68.2-dev'
 name = 'slapos.cookbook'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"
@@ -36,10 +36,12 @@ setup(name=name,
         'xml_marshaller', # need to communication with slapgrid
         'zc.buildout', # plays with buildout
         'zc.recipe.egg', # for scripts generation
+        'pytz', # for timezone database
         ],
       zip_safe=True,
       entry_points={
         'zc.buildout': [
+          'addresiliency = slapos.recipe.addresiliency:Recipe',
           'agent = slapos.recipe.agent:Recipe',
           'apache.frontend = slapos.recipe.apache_frontend:Recipe',
 	  'apacheperl = slapos.recipe.apacheperl:Recipe', 
@@ -60,6 +62,7 @@ setup(name=name,
           'dropbear.add_authorized_key = slapos.recipe.dropbear:AddAuthorizedKey',
           'dropbear.client = slapos.recipe.dropbear:Client',
           'dropbear = slapos.recipe.dropbear:Recipe',
+          'dumpmdb = slapos.recipe.dumpmdb:Recipe',
           'duplicity = slapos.recipe.duplicity:Recipe',
           'egg_test = slapos.recipe.erp5_test:EggTestRecipe',
           'equeue = slapos.recipe.equeue:Recipe',
@@ -83,6 +86,7 @@ setup(name=name,
           'gitinit = slapos.recipe.gitinit:Recipe',
           'haproxy = slapos.recipe.haproxy:Recipe',
           'helloworld = slapos.recipe.helloworld:Recipe',
+          'importmdb = slapos.recipe.importmdb:Recipe',
           'java = slapos.recipe.java:Recipe',
           'kumofs = slapos.recipe.kumofs:Recipe',
           'kvm.frontend = slapos.recipe.kvm_frontend:Recipe',
@@ -108,12 +112,14 @@ setup(name=name,
           'novnc = slapos.recipe.novnc:Recipe',
           'onetimeupload = slapos.recipe.onetimeupload:Recipe',
           'pbs = slapos.recipe.pbs:Recipe',
+          'postgres = slapos.recipe.postgres:Recipe',
+          'maarch.configuration = slapos.recipe.maarch.configuration:Recipe',
           'proactive = slapos.recipe.proactive:Recipe',
           'publish = slapos.recipe.publish:Recipe',
           'publishurl = slapos.recipe.publishurl:Recipe',
           'pwgen = slapos.recipe.pwgen:Recipe',
           'pwgen.stable = slapos.recipe.pwgen:StablePasswordGeneratorRecipe',
-          'requestoptional = slapos.recipe.requestoptional:Recipe',
+          'requestoptional = slapos.recipe.request:RequestOptional',
           'request = slapos.recipe.request:Recipe',
           'seleniumrunner = slapos.recipe.seleniumrunner:Recipe',
           'sheepdogtestbed = slapos.recipe.sheepdogtestbed:SheepDogTestBed',
@@ -127,6 +133,7 @@ setup(name=name,
           'slapmonitor = slapos.recipe.slapmonitor:Recipe',
           'slapreport = slapos.recipe.slapreport:Recipe',
           'slaprunner = slapos.recipe.slaprunner:Recipe',
+          'slaprunner.test = slapos.recipe.slaprunner:Test',
           'softwaretype = slapos.recipe.softwaretype:Recipe',
           'sphinx= slapos.recipe.sphinx:Recipe',
           'sshkeys_authority.request = slapos.recipe.sshkeys_authority:Request',

slapos/recipe/requestoptional.py → slapos/recipe/addresiliency/__init__.py

@@ -24,84 +24,58 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-import logging
+from slapos.recipe.librecipe import GenericSlapRecipe
 
-from slapos import slap as slapmodule
+import sys
+import os
 
-class Recipe(object):
-  """ Request a SlapOS instance. Won't fail if instance is not ready.
-  """
-
-  def __init__(self, buildout, name, options):
-    self.logger = logging.getLogger(name)
-
-    slap = slapmodule.slap()
-
-    self.software_release_url = options['software-url']
-    self.name = options['name']
-
-    slap.initializeConnection(options['server-url'],
-                              options.get('key-file'),
-                              options.get('cert-file'),
-                             )
-    computer_partition = slap.registerComputerPartition(
-      options['computer-id'], options['partition-id'])
-    self.request = computer_partition.request
-
-    self.isSlave = False
-    if 'slave' in options:
-      self.isSlave = options['slave'].lower() in ['y', 'yes', 'true', '1']
 
-    self.return_parameters = []
-    if 'return' in options:
-      self.return_parameters = [str(parameter).strip()
-                               for parameter in options['return'].split()]
-    else:
-      self.logger.debug("No parameter to return to main instance."
-                          "Be careful about that...")
-
-    software_type = 'RootSoftwareInstance'
-    if 'software-type' in options:
-      software_type = options['software-type']
-
-    filter_kw = {}
-    if 'sla' in options:
-      for sla_parameter in options['sla'].split():
-        filter_kw[sla_parameter] = options['sla-%s' % sla_parameter]
-
-    partition_parameter_kw = {}
-    if 'config' in options:
-      for config_parameter in options['config'].split():
-        partition_parameter_kw[config_parameter] = \
-            options['config-%s' % config_parameter]
-
-    self.instance = self.request(options['software-url'], software_type,
-      self.name, partition_parameter_kw=partition_parameter_kw,
-      filter_kw=filter_kw, shared=self.isSlave)
+class Recipe(GenericSlapRecipe):
+  """ This class provides the installation of the resilience
+      script on the partition.
+  """
 
-    self.failed = None
-    for param in self.return_parameters:
-      try:
-        options['connection-%s' % param] = str(
-            self.instance.getConnectionParameter(param))
-      except slapmodule.NotFoundError:
-        options['connection-%s' % param] = ''
-        if self.failed is None:
-          self.failed = param
+  def _install(self):
+    path_list = []
+    self_id = int(self.parameter_dict['number'])
+    ip = self.parameter_dict['ip-list'].split(' ')
+    print 'Creating bully script with  ips : %s\n' % ip
+    slap_connection = self.buildout['slap-connection']
 
-  def install(self):
-    if self.failed is not None:
-      # Check instance status to know if instance has been deployed
-      try:
-        status = self.instance.getState()
-      except slapmodule.NotFoundError:
-        status = 'not ready yet'
-      except AttributeError:
-        status = 'unknown'
-      error_message = 'Connection parameter %s not found. '\
-          'Requested instance is currently %s. If this error persists, '\
-          'check status of this instance.' % (self.failed, status)
-      self.logger.error(error_message)
-    return []
+    path_conf = os.path.join(self.options['script'], 'conf.in')
+    path_bully = os.path.join(self.options['script'], self.parameter_dict['script'])
+    path_bully_new = os.path.join(self.options['script'], 'new.py')
+    path_run = os.path.join(self.options['run'], self.parameter_dict['wrapper'])
+    print 'paths: %s\n%s\n' % (path_run, path_bully)
+    bully_conf = dict(self_id=self_id,
+                      ip_list=ip,
+                      executable=sys.executable,
+                      syspath=sys.path,
+                      server_url=slap_connection['server-url'],
+                      key_file=slap_connection.get('key-file'),
+                      cert_file=slap_connection.get('cert-file'),
+                      computer_id=slap_connection['computer-id'],
+                      partition_id=slap_connection['partition-id'],
+                      software=slap_connection['software-release-url'],
+                      namebase=self.parameter_dict['namebase'],
+                      confpath=path_conf)
+    try:
+      conf = self.createFile(path_conf,
+                             self.substituteTemplate(
+                             self.getTemplateFilename('conf.in.in'),
+                             bully_conf))
+      path_list.append(conf)
+      script = self.createExecutable(path_bully,
+                                     self.substituteTemplate(
+                                     self.getTemplateFilename('bully.py.in'),
+                                     bully_conf))
+      path_list.append(script)
 
-  update = install
+      wrapper = self.createPythonScript(
+          path_run,
+          'slapos.recipe.librecipe.execute.execute',
+          [path_bully])
+      path_list.append(wrapper)
+    except IOError:
+      pass
+    return path_list

slapos/recipe/addresiliency/template/bully.py.in

+#!%(executable)s
+
+import select
+import socket
+import threading
+import time
+import sys
+
+sys.path[:] = %(syspath)s
+
+from slapos import slap as slapmodule
+
+port = 50000
+size = 1024
+wait = True
+
+
+def loadConnectionInfos():
+  connectionInfos = {}
+  file = open('%(confpath)s', 'r')
+  params = file.read().split('\n')
+  file.close()
+  ip_list = [x.strip("' ") for x in params[0].strip('[],').split(',')]
+  connectionInfos['self_id'] = int(params[1])
+
+  connectionInfos['server_list'] = \
+      [(i, ip_list[i]) for i in range(len(ip_list))]
+
+  connectionInfos['self_ip'] = ip_list[connectionInfos['self_id']]
+
+  return connectionInfos
+
+
+def rename_broken_and_stop():
+  try:
+    slap = slapmodule.slap()
+    slap.initializeConnection('%(server_url)s',
+                              '%(key_file)s',
+                              '%(cert_file)s')
+    computer_partition = slap.registerComputerPartition('%(computer_id)s',
+                                                        '%(partition_id)s')
+    broken = computer_partition.request('%(software)s', 'frozen', '%(namebase)s0')
+
+    broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
+    broken.stopped()
+    computer_partition.rename('%(namebase)s0')
+    print 'renaming done\n'
+  except slapos.slap.slap.ServerError:
+    print 'Internal server error\n'
+
+def election():
+  global wait
+  connection = loadConnectionInfos()
+  message = "%%s, %%s" %% (connection['self_id'], "Election")
+  victory = True
+
+  for (remote_id, addr) in connection['server_list']:
+    if remote_id > connection['self_id']:
+      try:
+        s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+        s.connect((addr, port + remote_id))
+        s.send(message)
+        reply = s.recv(size)
+        if reply == "%%s, %%s" %% (remote_id, "Alive"):
+          victory = False
+      except (socket.error, socket.herror, socket.gaierror, socket.timeout):
+        pass
+      finally:
+        s.close()
+
+  if victory:
+    wait = True
+    for (remote_id, addr) in connection['server_list']:
+      if remote_id < connection['self_id']:
+        try:
+          s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+          s.connect((addr, port + remote_id))
+          s.send("%%s, %%s" %% (connection['self_id'], "Victory"))
+        except (socket.error, socket.herror, socket.gaierror, socket.timeout):
+          pass
+        finally:
+          s.close()
+    rename_broken_and_stop()
+
+
+def failure_detect():
+  global wait
+  connection = loadConnectionInfos()
+  while True:
+    time.sleep(30)
+    if wait:
+      print 'waiting 30 minutes\n'
+      time.sleep(30 * 60)
+      wait = False
+    if not connection['server_list'][0]:
+      continue
+    (remote_id, addr) = connection['server_list'][0]
+    try:
+      s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+      s.connect((addr, port + remote_id))
+      s.close()
+    except (socket.error, socket.herror, socket.gaierror, socket.timeout):
+      s.close()
+      election()
+
+failure_detect_thread = threading.Thread(target=failure_detect)
+failure_detect_thread.start()
+
+connection = loadConnectionInfos()
+s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+s.bind((connection['self_ip'], port + connection['self_id']))
+s.listen(5)
+#election()
+while True:
+  force_election = False
+  client, _ = s.accept()
+  client_message = client.recv(1024)
+  if client_message:
+    client_id, message = client_message.split(', ')
+    client_id = eval(client_id)
+    if message == "Victory":
+      wait = True
+      print "%%s wins" %% client_id
+    elif message == "Election":
+      print "%%s starts an election" %% client_id
+      if client_id < connection['self_id']:
+        client.send("%%s, %%s" %% (connection['self_id'], "Alive"))
+        force_election = True
+  client.close()
+  if force_election:
+    election()

slapos/recipe/addresiliency/template/bully_new.py.in

+#!%(executable)s
+
+import socket
+import time
+import sys
+import thread
+import time
+import os
+
+sys.path[:] = %(syspath)s
+
+from slapos import slap as slapmodule
+
+port = 50000
+size = 1024
+
+
+def rename_broken_and_stop():
+  try:
+    slap = slapmodule.slap()
+    slap.initializeConnection('%(server_url)s',
+                              '%(key_file)s',
+                              '%(cert_file)s')
+    computer_partition = slap.registerComputerPartition('%(computer_id)s',
+                                                        '%(partition_id)s')
+    broken = computer_partition.request('%(software)s', 'frozen', '%(namebase)s0')
+
+    broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
+    broken.stopped()
+    computer_partition.rename('%(namebase)s0')
+    print 'renaming done\n'
+  except slapos.slap.slap.ServerError:
+    print 'Internal server error\n'
+
+
+## Leader is always number 0
+
+class ResilientInstance(object):
+
+    def __init__(self, comm):
+        self.comm = comm
+        self.id = 0
+        self.state = 'normal'
+        self.halter = 0
+        self.nbComp = nbComp
+        self.inElection = False
+        self.alive = True
+        self.lastPing = time.clock()
+
+        self.mainCanal = self.comm.canal(['ping', 'halt',
+                                          'victory'])
+
+        self.okCanal = self.comm.canal(['ok'])
+        self.loadConnectionInfos()
+
+
+    def loadConnectionInfos(self):
+        file = open('%(confpath)s', 'r')
+        params = file.read().split('\n')
+        file.close()
+        self.nbComp = len([x.strip("' ") for x in params[0].strip('[],').split(',')])
+        new_id = int(params[1])
+        if self.id != new_id:
+            self.halter = new_id
+            self.id = new_id 
+
+    ## Needs to be changed to use the master
+    def aliveManagement(self):
+        while self.alive:
+            time.sleep(30*60)
+            if self.id == 0:
+                continue
+            self.comm.send('ping', 0)
+            message, sender = self.okCanal.get()
+            if message:
+                continue
+            self.election()
+
+    def listen(self):
+        while self.alive:
+            self.comm.recv()
+
+    def main(self):
+        while self.alive:
+            message, sender = self.mainCanal.get()
+            if message == 'ping':
+                    self.comm.send('ok', sender)
+
+            elif message == 'halt':
+                self.state = 'waitingConfirm'
+                self.halter = sender
+                self.comm.send('ok', sender)
+
+            elif message == 'victory':
+                if int(sender) == int(self.halter) and self.state == 'waitingConfirm':
+                    print '%s thinks %s is the leader\n' % (self.id, sender)
+                    self.comm.send('ok', sender)
+                self.state = 'normal'
+
+    def election(self):
+        self.inElection = True
+        self.loadConnectionInfos()
+        #Check if I'm the highest instance alive
+        for higher in range(self.id + 1, self.nbComp):
+            self.comm.send('ping', higher)
+            message, sender = self.okCanal.get()
+            if message:
+                #print '%s is alive (%s)\n' % (higher, self.id)
+                self.inElection = False
+                return False
+            continue
+
+        if not self.alive:
+            return False
+
+        #I should be the new coordinator, halt those below me
+        print 'Should be ME : %s \n' % self.id
+        self.state = 'election'
+        self.halter = self.id
+        ups = []
+        for lower in range(self.id):
+            self.comm.send('halt', lower)
+            message, sender = self.okCanal.get()
+            if message:
+                ups.append(lower)
+
+        #Broadcast Victory
+        self.state = 'reorganization'
+        for up in ups:
+            self.comm.send('victory', up)
+            message, sender = self.okCanal.get()
+            if message:
+                continue
+            print 'Something is wrong... let\'s start over\n'
+            return self.election()
+        self.state = 'normal'
+        self.active = True
+        print '%s Is THE LEADER \n' % self.id
+
+        rename_broken_and_stop()
+
+        self.inElection = False
+
+        return True
+
+
+class FilteredCanal(object):
+
+    def __init__(self, accept, timeout):
+        self.accept = accept
+        self.list = []
+        self.lock = thread.allocate_lock()
+        self.timeout = timeout
+
+    def append(self, message, sender):
+        if message in self.accept:
+            self.lock.acquire()
+            self.list.append([message, sender])
+            self.lock.release()
+
+    def get(self):
+        start = time.clock()
+        while (time.clock() - start < self.timeout):
+            self.lock.acquire()
+            if self.list:
+                self.lock.release()
+                val = self.list[0]
+                self.list = self.list[1:]
+                return val
+            self.lock.release()
+        return [None, None]
+
+
+class Wrapper(object):
+
+    def __init__(self, timeout=20):
+        self.read_pipes = [os.fdopen(x) for x in read_pipes]
+        self.write_pipes = write_pipes
+        self.canals = []
+        self.ips = []
+        self.id = 0
+        self.timeout = timeout
+        self.getConnectionInfos()
+        self.socket = None
+
+    def getConnectionInfos(self):
+        file = open('%(confpath)s', 'r')
+        params = file.read().split('\n')
+        file.close()
+        self.ips = [x.strip("' ") for x in params[0].strip('[],').split(',')]
+        self.id = int(params[1])
+
+    def start(self):
+        self.getConnectionInfos()
+        self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+        self.socket.bind((self.ips[self.id], port + self.id))
+        s.listen(5)
+
+    def send(self, message, number):
+        self.getConnectionInfos()
+        try:
+            s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+            s.connect((self.ips[number], port + number))
+            s.send(message + (' %s\n' % self.id))
+        except (socket.error, socket.herror, socket.gaierror, socket.timeout):
+          pass
+        finally:
+            s.close()
+
+    def canal(self, accept):
+        created = FilteredCanal(accept, self.timeout)
+        self.canals.append(created)
+        return created
+
+    def recv(self):
+        client, _ = s.accept()
+        client_message = client.recv(1024)
+        if client_message:
+            message, sender = client_message.split()
+            for canal in self.canals:
+                canal.append(message, sender)
+
+
+wrapper = createWrapper(20)
+
+computer = ResilientInstance(wrapper)
+
+#idle waiting for connection infos
+while computer.nbComp < 2 :
+    computer.loadConnectionInfos()
+    time.sleep(30)
+
+print 'Starting\n'
+
+computer.comm.start()
+thread.start_new_thread(computer.listen, ())
+thread.start_new_thread(computer.main, ())
+thread.start_new_thread(computer.aliveManagement, ())
+
+while True:
+    continue
+
+

slapos/recipe/addresiliency/template/conf.in.in

+%(ip_list)s
+%(self_id)s

slapos/recipe/apachephp/template/php.ini.in

@@ -15,4 +15,4 @@ date.timezone = Europe/Paris
 file_uploads = On
 upload_max_filesize = 8M
 post_max_size = 8M
-magic_quotes_gpc=0ff
+magic_quotes_gpc=Off

slapos/recipe/apacheproxy/template/apache.in

@@ -3,7 +3,6 @@
 
 # Basic server configuration
 PidFile "%(pid_file)s"
-LockFile "%(lock_file)s"
 Listen %(ip)s:%(port)s
 ServerAdmin someone@email
 DefaultType text/plain
@@ -22,13 +21,15 @@ CustomLog "%(access_log)s" common
 <Directory />
     Options FollowSymLinks
     AllowOverride None
-    Order deny,allow
-    Deny from all
+    Require all denied
 </Directory>
 
 ProxyPass / %(backend_url)s
 
 # List of modules
+LoadModule unixd_module modules/mod_unixd.so
+LoadModule access_compat_module modules/mod_access_compat.so
+LoadModule authz_core_module modules/mod_authz_core.so
 LoadModule authz_host_module modules/mod_authz_host.so
 LoadModule log_config_module modules/mod_log_config.so
 LoadModule setenvif_module modules/mod_setenvif.so

slapos/recipe/check_url_available/template/check_url.in

@@ -9,11 +9,22 @@ if [ -z $URL ]; then
   exit 3
 fi
 
-CODE=$(%(curl_path)s -k -sL $URL -w %%{http_code} -o /dev/null)
+CODE=$(%(curl_path)s -g -k -sL $URL -w %%{http_code} -o /dev/null)
+
+if [ $? -eq 3 ]; then
+  echo "URL malformed: $URL." >&2
+  exit 1
+fi
+
+if [ $? -eq 7 ]; then
+  echo "Failed to connect to host: $URL." >&2
+  exit 1
+fi
 
 if [ ! $CODE ]; then
   echo "$URL is not available (server not reachable)." >&2
   exit 1
+fi
 
 if [ $CODE -eq 000 ]; then
   echo "$URL is not available (server not reachable)." >&2

slapos/recipe/dropbear.py

@@ -167,6 +167,7 @@ class AddAuthorizedKey(GenericBaseRecipe):
     path_list.append(ssh)
 
     authorized_keys = AuthorizedKeysFile(os.path.join(ssh, 'authorized_keys'))
-    authorized_keys.append(self.options['key'])
+    for key in self.options['key'].split(' '):
+      authorized_keys.append(key)
 
     return path_list

slapos/recipe/generic_mysql/__init__.py

@@ -154,26 +154,38 @@ class Recipe(GenericBaseRecipe):
     # TODO: move to a separate recipe (ack'ed by Cedric)
     # percona toolkit (formerly known as maatkit) installation
     for pt_script_name in (
+        'pt-align',
         'pt-archiver',
+        'pt-collect',
         'pt-config-diff',
         'pt-deadlock-logger',
+        'pt-diskstats',
         'pt-duplicate-key-checker',
         'pt-fifo-split',
         'pt-find',
+        'pt-fingerprint',
         'pt-fk-error-logger',
         'pt-heartbeat',
         'pt-index-usage',
+        'pt-ioprofile',
         'pt-kill',
         'pt-log-player',
+        'pt-mext',
+        'pt-mysql-summary',
         'pt-online-schema-change',
+        'pt-pmp',
         'pt-query-advisor',
         'pt-query-digest',
         'pt-show-grants',
+        'pt-sift',
         'pt-slave-delay',
         'pt-slave-find',
         'pt-slave-restart',
+        'pt-stalk',
+        'pt-summary',
         'pt-table-checksum',
         'pt-table-sync',
+        'pt-table-usage',
         'pt-tcp-model',
         'pt-trend',
         'pt-upgrade',

slapos/recipe/generic_mysql/mysql.py

@@ -2,6 +2,7 @@ import os
 import subprocess
 import time
 import sys
+import pytz
 
 
 def runMysql(args):
@@ -41,32 +42,50 @@ def updateMysql(args):
   sleep = 30
   is_succeed = False
   while True:
-    if not is_succeed:
-      mysql_upgrade_list = [conf['mysql_upgrade_binary'], '--no-defaults', '--user=root', '--socket=%s' % conf['socket']]
-      mysql_upgrade = subprocess.Popen(mysql_upgrade_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-      result = mysql_upgrade.communicate()[0]
-      if mysql_upgrade.returncode is None:
-        mysql_upgrade.kill()
-      if mysql_upgrade.returncode != 0 and not 'is already upgraded' in result:
-        print "Command %r failed with result:\n%s" % (mysql_upgrade_list, result)
-        print 'Sleeping for %ss and retrying' % sleep
+    mysql_upgrade_list = [conf['mysql_upgrade_binary'], '--no-defaults', '--user=root', '--socket=%s' % conf['socket']]
+    mysql_upgrade = subprocess.Popen(mysql_upgrade_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    result = mysql_upgrade.communicate()[0]
+    if mysql_upgrade.returncode is None:
+      mysql_upgrade.kill()
+    if mysql_upgrade.returncode != 0 and not 'is already upgraded' in result:
+      print "Command %r failed with result:\n%s" % (mysql_upgrade_list, result)
+    else:
+      if mysql_upgrade.returncode == 0:
+        print "MySQL database upgraded with result:\n%s" % result
       else:
-        if mysql_upgrade.returncode == 0:
-          print "MySQL database upgraded with result:\n%s" % result
-        else:
-          print "No need to upgrade MySQL database"
-        mysql_list = [conf['mysql_binary'].strip(), '--no-defaults', '-B', '--user=root', '--socket=%s' % conf['socket']]
-        mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE,
-            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        result = mysql.communicate(conf['mysql_script'])[0]
-        if mysql.returncode is None:
-          mysql.kill()
+        print "No need to upgrade MySQL database"
+      mysql_list = [conf['mysql_binary'].strip(), '--no-defaults', '-B', '--user=root', '--socket=%s' % conf['socket']]
+      mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE,
+          stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+      result = mysql.communicate(conf['mysql_script'])[0]
+      if mysql.returncode is None:
+        mysql.kill()
+      if mysql.returncode != 0:
+        print 'Command %r failed with:\n%s' % (mysql_list, result)
+      else:
+        # import timezone database
+        mysql_tzinfo_to_sql_binary = os.path.join(
+          os.path.dirname(conf['mysql_binary'].strip()), 'mysql_tzinfo_to_sql')
+        zoneinfo_directory = '%s/zoneinfo' % os.path.dirname(pytz.__file__)
+        mysql_tzinfo_to_sql_list = [mysql_tzinfo_to_sql_binary, zoneinfo_directory]
+        mysql_tzinfo_to_sql = subprocess.Popen(mysql_tzinfo_to_sql_list, stdin=subprocess.PIPE,
+                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        timezone_sql = mysql_tzinfo_to_sql.communicate()[0]
         if mysql.returncode != 0:
-          print 'Command %r failed with:\n%s' % (mysql_list, result)
-          print 'Sleeping for %ss and retrying' % sleep
+          print 'Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result)
         else:
+          mysql = subprocess.Popen(mysql_list + ['mysql',], stdin=subprocess.PIPE,
+                                   stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+          result = mysql.communicate(timezone_sql)[0]
+          if mysql.returncode is None:
+            mysql.kill()
+          if mysql.returncode != 0:
+            print 'Command %r failed with:\n%s' % (mysql_list, result)
           is_succeed = True
-          print 'SlapOS initialisation script succesfully applied on database.'
+    if is_succeed:
+      print 'SlapOS initialisation script succesfully applied on database.'
+      break
+    print 'Sleeping for %ss and retrying' % sleep
     sys.stdout.flush()
     sys.stderr.flush()
     time.sleep(sleep)

slapos/recipe/generic_mysql/template/my.cnf.in

@@ -24,7 +24,7 @@ long_query_time = 1
 max_allowed_packet = 128M
 query_cache_size = 32M
 
-plugin-load = ha_mroonga.so;ha_sphinx.so;handlersocket.so
+plugin-load = ha_mroonga.so;handlersocket.so
 
 # By default only 100 connections are allowed, when using zeo
 # we may have much more connections

slapos/recipe/haproxy/__init__.py

@@ -73,7 +73,11 @@ class Recipe(GenericBaseRecipe):
     server_snippet = ""
     i = 0
     name = self.options['name']
-    for address in self.options['backend-list'].split():
+    backend_list = self.options['backend-list']
+    if isinstance(backend_list, str):
+      # BBB
+      backend_list = backend_list.split()
+    for address in backend_list:
       i += 1
       server_snippet += self.substituteTemplate(
           snippet_filename, dict(

slapos/recipe/lamp/template/my.cnf.in

@@ -21,7 +21,7 @@ long_query_time = 5
 max_allowed_packet = 128M
 query_cache_size = 32M
 
-plugin-load = ha_mroonga.so;ha_sphinx.so
+plugin-load = ha_mroonga.so
 
 # The following are important to configure and depend a lot on to the size of
 # your database and the available resources.

slapos/recipe/maarch/configuration.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import ConfigParser
+import errno
+import lxml.etree
+import md5
+import os
+
+import lxml
+
+
+# TODO: remove the hack below, used to reach psycopg2
+# XXX: When run inside webrunner, Postgres refuses connection.
+# TODO: make the recipe work inside webrunner
+
+
+
+def temporary_hack():
+    # XXX TODO provide psycopg to sys.path by other means
+    import sys
+    develop_eggs = '/'.join(lxml.__file__.split('/')[:lxml.__file__.split('/').index('develop-eggs')+1])
+    for egg_folder in os.listdir(develop_eggs):
+        if egg_folder.startswith('psycopg2'):
+            sys.path.append(os.path.join(develop_eggs, egg_folder))
+temporary_hack()
+
+import psycopg2
+
+from slapos.recipe.librecipe import GenericBaseRecipe
+
+
+def xpath_set(xml, settings):
+    for path, value in settings.iteritems():
+        xml.xpath(path)[0].text = value
+
+
+class Recipe(GenericBaseRecipe):
+    """\
+    This recipe configures a maarch instance to be ready to run,
+    without going through the initial wizard:
+
+     - creation of two xml files from the provided defaults
+     - php.ini as required by Maarch
+     - database setup.
+
+     The superuser password will be the same as the Postgres one.
+    """
+
+    def install(self):
+        self.update_phpini(php_ini_path=os.path.join(self.options['php_ini_dir'], 'php.ini'))
+
+        self.load_initial_db()
+
+        ret = []
+
+        apps_config_xml = self.create_apps_config_xml()
+        if apps_config_xml:
+            ret.append(apps_config_xml)
+
+        core_config_xml = self.create_core_config_xml()
+        if core_config_xml:
+            ret.append(core_config_xml)
+
+        # confirm that everything is done, the app will run without further setup
+        lck_path = self.installed_lock()
+        ret.append(lck_path)
+
+        return ret
+
+
+
+    def create_apps_config_xml(self):
+        options = self.options
+
+        folder = os.path.join(options['htdocs'], 'apps/maarch_entreprise/xml')
+        config_xml_default = os.path.join(folder, 'config.xml.default')
+        config_xml = os.path.join(folder, 'config.xml')
+
+        # do not overwrite the config.xml file (it can be customized inside the application)
+        if os.path.exists(config_xml):
+            return
+
+        content = open(config_xml_default, 'rb').read()
+        xml = lxml.etree.fromstring(content)
+
+        xpath_set(xml, {
+            'CONFIG/databaseserver': options['db_host'],
+            'CONFIG/databaseserverport': options['db_port'],
+            'CONFIG/databasename': options['db_dbname'],
+            'CONFIG/databaseuser': options['db_username'],
+            'CONFIG/databasepassword': options['db_password'],
+            'CONFIG/lang': options['language'],
+            })
+
+        with os.fdopen(os.open(config_xml, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600), 'w') as fout:
+            fout.write(lxml.etree.tostring(xml, xml_declaration=True, encoding='utf-8').encode('utf-8'))
+
+        return config_xml
+
+
+    def create_core_config_xml(self):
+        options = self.options
+
+        folder = os.path.join(options['htdocs'], 'core/xml')
+        config_xml_default = os.path.join(folder, 'config.xml.default')
+        config_xml = os.path.join(folder, 'config.xml')
+
+        # do not overwrite the config.xml file (it can be customized inside the application)
+        if os.path.exists(config_xml):
+            return
+
+        content = open(config_xml_default, 'rb').read()
+        xml = lxml.etree.fromstring(content)
+
+        xpath_set(xml, {
+            'CONFIG/defaultlanguage': options['language'],
+            })
+
+        with os.fdopen(os.open(config_xml, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600), 'w') as fout:
+            fout.write(lxml.etree.tostring(xml, xml_declaration=True, encoding='utf-8').encode('utf-8'))
+
+        return config_xml
+
+
+    def update_phpini(self, php_ini_path):
+        php_ini = ConfigParser.RawConfigParser()
+        php_ini.read(php_ini_path)
+
+        php_ini.set('PHP', 'error_reporting', 'E_ALL & ~E_DEPRECATED & ~E_NOTICE')
+        php_ini.set('PHP', 'display_errors', 'On')
+        php_ini.set('PHP', 'short_open_tag', 'On')
+        php_ini.set('PHP', 'magic_quotes_gpc', 'Off')
+
+        with os.fdopen(os.open(php_ini_path, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600), 'w') as fout:
+            php_ini.write(fout)
+
+
+    def load_initial_db(self):
+        """
+        This method:
+
+         - creates the initial schema
+         - patches the schema for ipv6
+         - loads initial data
+         - sets initial superadmin password
+         - configures and creates docservers directories
+        """
+
+        options = self.options
+
+        conn = psycopg2.connect(host = options['db_host'],
+                                port = int(options['db_port']),
+                                database = options['db_dbname'],
+                                user = options['db_username'],
+                                password = options['db_password'])
+
+        cur = conn.cursor()
+
+        # skip everything if the tables have already been created
+        cur.execute("SELECT 1 FROM information_schema.tables WHERE table_schema='public' AND table_name='docservers';")
+        if cur.rowcount == 1:
+            conn.close()
+            return
+
+        htdocs = options['htdocs']
+
+        # load the schema
+        with open(os.path.join(htdocs, 'structure.sql')) as fin:
+            cur.execute(fin.read())
+
+        # patch the schema to store long addresses (ipv6)
+        cur.execute('ALTER TABLE HISTORY ALTER COLUMN remote_ip TYPE CHAR(255);')
+
+        with open(os.path.join(htdocs, 'data_mini.sql')) as fin:
+            cur.execute(fin.read())
+
+        # initial admin password
+        enc_password = md5.md5(options['db_password']).hexdigest()
+        cur.execute("UPDATE users SET password=%s WHERE user_id='superadmin';", (enc_password, ))
+
+        # directories described in http://wiki.maarch.org/Maarch_Entreprise/fr/Man/Admin/Stockage
+        for docserver_id, foldername in [
+                ('OFFLINE_1', 'offline'),
+                ('FASTHD_AI', 'ai'),
+                ('OAIS_MAIN_1', 'OAIS_main'),
+                ('OAIS_SAFE_1', 'OAIS_safe'),
+                ('FASTHD_MAN', 'manual'),
+                ('TEMPLATES', 'templates'),
+                ]:
+            full_path = os.path.join(self.options['root_docservers'], foldername)
+            cur.execute('UPDATE docservers SET path_template=%s WHERE docserver_id=%s', (full_path, docserver_id))
+            try:
+                os.makedirs(full_path)
+            except OSError as exc:
+                if exc.errno == errno.EEXIST:
+                    pass
+                else:
+                    raise
+
+        conn.commit()
+        cur.close()
+        conn.close()
+
+
+    def installed_lock(self):
+        """\
+        Create an empty file to mean the setup is completed
+        """
+        htdocs = self.options['htdocs']
+        lck_path = os.path.join(htdocs, 'installed.lck')
+
+        with open(lck_path, 'w'):
+            pass
+
+        return lck_path
+
+

slapos/recipe/mysql/__init__.py

@@ -31,6 +31,7 @@ import subprocess
 from slapos.recipe.librecipe import GenericBaseRecipe
 from slapos.recipe.librecipe import filehash
 
+
 class Recipe(GenericBaseRecipe):
 
   def _options(self, options):

slapos/recipe/mysql/template/my.cnf.in

@@ -19,7 +19,7 @@ long_query_time = 5
 max_allowed_packet = 128M
 query_cache_size = 32M
 
-plugin-load = ha_mroonga.so;ha_sphinx.so
+plugin-load = ha_mroonga.so
 
 # The following are important to configure and depend a lot on to the size of
 # your database and the available resources.

slapos/recipe/pbs.py

@@ -213,6 +213,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
         for slave in slaves:
           path_list.extend(self.add_slave(slave, known_hosts))
 
+
     else:
       command = [self.options['rdiffbackup-binary']]
       self.logger.info("Server mode")

slapos/recipe/postgres/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import md5
+import os
+import subprocess
+import textwrap
+from zc.buildout import UserError
+
+from slapos.recipe.librecipe import GenericBaseRecipe
+
+
+
+class Recipe(GenericBaseRecipe):
+    """\
+    This recipe creates:
+
+        - a Postgres cluster
+        - configuration to allow connections from IPV6 only (or unix socket)
+        - a superuser with provided name and generated password
+        - a database with provided name
+        - a foreground start script in the services directory
+
+    then adds the connection URL to the options.
+    The URL can be used as-is (ie. in sqlalchemy) or by the _urlparse.py recipe.
+    """
+
+    def fetch_host(self, options):
+        """
+        Returns a string represtation of ipv6_host.
+        May receive a regular string, a set or a string serialized by buildout.
+        """
+        ipv6_host = options['ipv6_host']
+
+        if isinstance(ipv6_host, set):
+            return ipv6_host.pop()
+        else:
+            return ipv6_host
+
+
+    def _options(self, options):
+        options['password'] = self.generatePassword()
+        options['url'] = 'postgresql://%(user)s:%(password)s@[%(host)s]:%(port)s/%(dbname)s' % dict(options, host=self.fetch_host(options))
+
+
+    def install(self):
+        pgdata = self.options['pgdata-directory']
+
+        if not os.path.exists(pgdata):
+            self.createCluster()
+            self.createConfig()
+            self.createDatabase()
+            self.createSuperuser()
+            self.createRunScript()
+
+        return [
+                # XXX should we really return something here?
+                # os.path.join(pgdata, 'postgresql.conf')
+                ]
+
+
+    def createCluster(self):
+        initdb_binary = os.path.join(self.options['bin'], 'initdb')
+
+        pgdata = self.options['pgdata-directory']
+
+        try:
+            subprocess.check_call([initdb_binary,
+                                   '-D', pgdata,
+                                   '-A', 'ident',
+                                   '-E', 'UTF8',
+                                   ])
+        except subprocess.CalledProcessError:
+            raise UserError('Could not create cluster directory in %s' % pgdata)
+
+
+    def createConfig(self):
+        pgdata = self.options['pgdata-directory']
+
+        with open(os.path.join(pgdata, 'postgresql.conf'), 'wb') as cfg:
+            cfg.write(textwrap.dedent("""\
+                    listen_addresses = '%s'
+                    logging_collector = on
+                    log_rotation_size = 50MB
+                    max_connections = 100
+                    datestyle = 'iso, mdy'
+
+                    lc_messages = 'en_US.UTF-8'
+                    lc_monetary = 'en_US.UTF-8'
+                    lc_numeric = 'en_US.UTF-8'
+                    lc_time = 'en_US.UTF-8'
+                    default_text_search_config = 'pg_catalog.english'
+
+                    unix_socket_directory = '%s'
+                    unix_socket_permissions = 0700
+                    """ % (
+                        self.fetch_host(self.options),
+                        pgdata,
+                        )))
+
+
+        with open(os.path.join(pgdata, 'pg_hba.conf'), 'wb') as cfg:
+            # see http://www.postgresql.org/docs/9.1/static/auth-pg-hba-conf.html
+
+            cfg.write(textwrap.dedent("""\
+                    # TYPE  DATABASE        USER            ADDRESS                 METHOD
+
+                    # "local" is for Unix domain socket connections only (check unix_socket_permissions!)
+                    local   all             all                                     ident
+                    host    all             all             127.0.0.1/32            md5
+                    host    all             all             ::1/128                 md5
+                    host    all             all             %s/128                  md5
+                    """ % self.fetch_host(self.options)))
+
+
+    def createDatabase(self):
+        self.runPostgresCommand(cmd='CREATE DATABASE "%s"' % self.options['dbname'])
+
+
+    def createSuperuser(self):
+        """
+        Creates a Postgres superuser - other than "slapuser#" for use by the application.
+        """
+
+        # http://postgresql.1045698.n5.nabble.com/Algorithm-for-generating-md5-encrypted-password-not-found-in-documentation-td4919082.html
+
+        user = self.options['user']
+        password = self.options['password']
+
+        # encrypt the password to avoid storing in the logs
+        enc_password = 'md5' + md5.md5(password+user).hexdigest()
+
+        self.runPostgresCommand(cmd="""CREATE USER "%s" ENCRYPTED PASSWORD '%s' SUPERUSER""" % (user, enc_password))
+
+
+    def runPostgresCommand(self, cmd):
+        """
+        Executes a command in single-user mode, with no daemon running.
+
+        Multiple commands can be executed by providing newlines,
+        preceeded by backslash, between them.
+        See http://www.postgresql.org/docs/9.1/static/app-postgres.html
+        """
+
+        pgdata = self.options['pgdata-directory']
+        postgres_binary = os.path.join(self.options['bin'], 'postgres')
+
+        try:
+            p = subprocess.Popen([postgres_binary,
+                                  '--single',
+                                  '-D', pgdata,
+                                  'postgres',
+                                  ], stdin=subprocess.PIPE)
+
+            p.communicate(cmd+'\n')
+        except subprocess.CalledProcessError:
+            raise UserError('Could not create database %s' % pgdata)
+
+
+    def createRunScript(self):
+        """
+        Creates a script that runs postgres in the foreground.
+        'exec' is used to allow easy control by supervisor.
+        """
+        content = textwrap.dedent("""\
+                #!/bin/sh
+                exec %(bin)s/postgres \\
+                    -D %(pgdata-directory)s
+                """ % self.options)
+        name = os.path.join(self.options['services'], 'postgres-start')
+        self.createExecutable(name, content=content)
+
+

slapos/recipe/request.py

@@ -27,6 +27,9 @@
 import logging
 
 from slapos import slap as slapmodule
+import slapos.recipe.librecipe.generic as librecipe
+
+DEFAULT_SOFTWARE_TYPE = 'RootSoftwareInstance'
 
 class Recipe(object):
   """
@@ -97,17 +100,15 @@ class Recipe(object):
     request = slap.registerComputerPartition(
       options['computer-id'], options['partition-id']).request
 
-    isSlave = options.get('slave', '').lower() in ['y', 'yes', 'true', '1']
-
     return_parameters = []
     if 'return' in options:
       return_parameters = [str(parameter).strip()
-        for parameter in options['return'].split()]
+                          for parameter in options['return'].split()]
     else:
       self.logger.debug("No parameter to return to main instance."
         "Be careful about that...")
 
-    software_type = options.get('software-type', 'RootInstanceSoftware')
+    software_type = options.get('software-type', DEFAULT_SOFTWARE_TYPE)
 
     filter_kw = {}
     if 'sla' in options:
@@ -120,6 +121,8 @@ class Recipe(object):
         partition_parameter_kw[config_parameter] = \
             options['config-%s' % config_parameter]
 
+    isSlave = options.get('slave', '').lower() in \
+        librecipe.GenericBaseRecipe.TRUE_VALUES
     self.instance = instance = request(software_url, software_type,
       name, partition_parameter_kw=partition_parameter_kw,
       filter_kw=filter_kw, shared=isSlave)
@@ -128,7 +131,7 @@ class Recipe(object):
       try:
         options['connection-%s' % param] = str(
           instance.getConnectionParameter(param))
-      except slapmodule.NotFoundError:
+      except (slapmodule.NotFoundError, slapmodule.ServerError):
         options['connection-%s' % param] = ''
         if self.failed is None:
           self.failed = param
@@ -137,9 +140,12 @@ class Recipe(object):
     if self.failed is not None:
       # Check instance status to know if instance has been deployed
       try:
-        status = self.instance.getState()
-      except slapmodule.NotFoundError:
-        status = 'not ready yet, please try again'
+        if self.instance.getComputerId() is not None:
+          status = self.instance.getState()
+        else:
+          status = 'not ready yet'
+      except (slapmodule.NotFoundError, slapmodule.ServerError):
+        status = 'not ready yet'
       except AttributeError:
         status = 'unknown'
       error_message = 'Connection parameter %s not found. '\
@@ -150,3 +156,28 @@ class Recipe(object):
     return []
 
   update = install
+
+class RequestOptional(Recipe):
+  """
+  Request a SlapOS instance. Won't fail if instance is not ready.
+  Same as slapos.cookbook:request, but won't raise in case of problem.
+  """
+  def install(self):
+    if self.failed is not None:
+      # Check instance status to know if instance has been deployed
+      try:
+        if self.instance.getComputerId() is not None:
+          status = self.instance.getState()
+        else:
+          status = 'not ready yet'
+      except (slapmodule.NotFoundError, slapmodule.ServerError):
+        status = 'not ready yet'
+      except AttributeError:
+        status = 'unknown'
+      error_message = 'Connection parameter %s not found. '\
+          'Requested instance is currently %s. If this error persists, '\
+          'check status of this instance.' % (self.failed, status)
+      self.logger.warning(error_message)
+    return []
+
+  update = install

slapos/recipe/slaprunner/__init__.py

@@ -39,6 +39,7 @@ class Recipe(GenericBaseRecipe):
     self.instance_directory = options['instance-directory'].strip()
     self.partition_amount = options['partition-amount'].strip()
     self.cloud9_url = options['cloud9-url'].strip()
+    self.log_file = os.path.join(options['log_dir'].strip(), 'slaprunner.log')
     # Set slaprunner access URL
     options['access-url'] = 'http://[%s]:%s' % (self.ipv6, self.runner_port)
 
@@ -79,10 +80,12 @@ class Recipe(GenericBaseRecipe):
     path_list.append(config_file)
 
     environment = dict(
-        PATH=os.path.dirname(self.options['git-binary']) + ':' + os.environ['PATH'],
+        PATH=os.path.dirname(
+            self.options['git-binary']) + ':' + os.environ['PATH'],
         GIT_SSH=self.options['ssh_client']
     )
-    launch_args = [self.options['slaprunner'].strip(), config_file]
+    launch_args = [self.options['slaprunner'].strip(), config_file,
+                   '--log_file', self.log_file]
     if self.optionIsTrue('debug', default=False):
       launch_args.append('--debug')
 
@@ -93,3 +96,69 @@ class Recipe(GenericBaseRecipe):
     path_list.append(wrapper)
 
     return path_list
+
+class Test(GenericBaseRecipe):
+  def _options(self, options):
+    self.ipv4 = options['ipv4'].strip()
+    self.ipv6 = options['ipv6'].strip()
+    self.proxy_port = options['proxy_port'].strip()
+    self.runner_port = options['runner_port'].strip()
+    self.workdir = options['working-directory'].strip()
+    self.software_directory = options['software-directory'].strip()
+    self.instance_directory = options['instance-directory'].strip()
+    self.partition_amount = options['partition-amount'].strip()
+    self.cloud9_url = options['cloud9-url'].strip()
+    # Set slaprunner access URL
+    options['access-url'] = 'http://[%s]:%s' % (self.ipv6, self.runner_port)
+
+  def install(self):
+    path_list = []
+
+    configuration = dict(
+        software_root=self.software_directory,
+        instance_root=self.instance_directory,
+        master_url='http://%s:%s/' % (self.ipv4, self.proxy_port),
+        computer_id='slaprunner',
+        partition_amount=self.partition_amount,
+        slapgrid_sr=self.options['slapgrid_sr'],
+        slapgrid_cp=self.options['slapgrid_cp'],
+        slapproxy=self.options['slapproxy'],
+        supervisor=self.options['supervisor'],
+        supervisord_config=os.path.join(self.instance_directory, 'etc',
+          'supervisord.conf'),
+        runner_workdir=self.workdir,
+        etc_dir=self.options['etc_dir'],
+        runner_host=self.ipv6,
+        runner_port=self.runner_port,
+        ipv4_address=self.ipv4,
+        ipv6_address=self.ipv6,
+        proxy_host=self.ipv4,
+        proxy_port=self.proxy_port,
+        proxy_database=os.path.join(self.workdir, 'proxy.db'),
+        git=self.options['git-binary'],
+        ssh_client=self.options['ssh_client'],
+        public_key=self.options['public_key'],
+        private_key=self.options['private_key'],
+        cloud9_url=self.cloud9_url
+    )
+
+    config_file = self.createFile(self.options['slapos.cfg'],
+        self.substituteTemplate(self.getTemplateFilename('slapos.cfg.in'),
+        configuration))
+    path_list.append(config_file)
+
+    environment = dict(
+        PATH=os.path.dirname(
+            self.options['git-binary']) + ':' + os.environ['PATH'],
+        GIT_SSH=self.options['ssh_client'],
+        CONFIG_FILE_PATH=config_file
+    )
+    launch_args = [self.options['slaprunnertest'].strip()]
+
+    wrapper = self.createPythonScript(self.options['wrapper'],
+        'slapos.recipe.librecipe.execute.executee',
+        (launch_args, environment)
+    )
+    path_list.append(wrapper)
+
+    return path_list

slapos/recipe/slaprunner/execute.py

-import os
-
-
-def execute(args):
-  """Portable execution with process replacement"""
-  if args.get("path", None):
-    os.environ['PATH'] = args["path"]
-  os.execv(args["launch_args"][0], args["launch_args"])

slapos/recipe/softwaretype.py

@@ -87,6 +87,8 @@ class Recipe:
       computer_partition_id)
     self.parameter_dict = self.computer_partition.getInstanceParameterDict()
     software_type = self.parameter_dict['slap_software_type']
+    self.logger.info('Deploying instance with software type %s' % \
+        software_type)
 
     if software_type not in self.options:
       if 'default' in self.options:

slapos/recipe/tidstorage/__init__.py

@@ -29,13 +29,21 @@ from slapos.recipe.librecipe import GenericBaseRecipe
 
 class Recipe(GenericBaseRecipe):
   def install(self):
-    path_list = []
-    a = path_list.append
-    configuration_file = self.createFile(self.options['configuration-path'], self.substituteTemplate(self.getTemplateFilename('tidstorage.py.in'), self.options))
-    a(configuration_file)
-    tidstorage_wrapper = self.createPythonScript(self.options['tidstorage-wrapper'], 'slapos.recipe.librecipe.execute.execute', [self.options['tidstoraged-binary'], '--nofork', '--config', configuration_file])
-    a(tidstorage_wrapper)
-    repozo_wrapper = self.createPythonScript(self.options['repozo-wrapper'], 'slapos.recipe.librecipe.execute.execute', [self.options['tidstorage-repozo-binary'], '--config', configuration_file, '--repozo', self.options['repozo-binary'], '-z'])
-    a(repozo_wrapper)
+    configuration_file = self.createFile(
+      self.options['configuration-path'],
+      self.substituteTemplate(
+        self.getTemplateFilename('tidstorage.py.in'), self.options))
 
-    return path_list
+    tidstorage_wrapper = self.createPythonScript(
+      self.options['tidstorage-wrapper'],
+      'slapos.recipe.librecipe.execute.execute',
+      [self.options['tidstoraged-binary'],  '--nofork', '--config',
+        configuration_file])
+
+    repozo_wrapper = self.createPythonScript(
+      self.options['repozo-wrapper'],
+      'slapos.recipe.librecipe.execute.execute',
+      [self.options['tidstorage-repozo-binary'], '--config',
+        configuration_file, '--repozo', self.options['repozo-binary'], '-z'])
+
+    return [configuration_file, tidstorage_wrapper, repozo_wrapper]

slapos/recipe/tidstorage/template/tidstorage.py.in

 known_tid_storage_identifier_dict = %(known-tid-storage-identifier-dict)s
-base_url = '%(base-url)s'
 address = '%(ip)s'
 port = %(port)s
 

slapos/recipe/xwiki/template/my.cnf.in

@@ -21,7 +21,7 @@ long_query_time = 5
 max_allowed_packet = 128M
 query_cache_size = 32M
 
-plugin-load = ha_mroonga.so;ha_sphinx.so
+plugin-load = ha_mroonga.so
 
 # The following are important to configure and depend a lot on to the size of
 # your database and the available resources.

software/maarch/software.cfg

+[buildout]
+
+develop =
+  /srv/slapgrid/slappart4/srv//runner/project/slapos
+
+extends =
+  ../../stack/lapp/buildout.cfg
+
+parts =
+  slapos-cookbook
+  apache-php-postgres
+  pear-modules
+  eggs
+  template
+  template-apache-php
+
+
+[application]
+url = http://downloads.sourceforge.net/project/maarch/Maarch%20Entreprise/MaarchEntreprise-1.3.zip?r=http%3A%2F%2Fwww.maarch.org%2Ftelecharger&ts=1347961624&use_mirror=ignum
+md5sum = 5c2c859dee9d0dde3ba959474fd5fc86
+
+
+[application-template]
+# XXX we don't need this
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/config.php.in
+#md5sum = 
+download-only = True
+filename = template.in
+mode = 0644
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+
+
+[application-configuration]
+# XXX we don't need this
+location = config.php
+
+
+[pear-modules]
+recipe = cp.recipe.cmd
+pear = ${apache-php-postgres:location}/bin/pear
+install_cmd =
+    ${:pear} channel-info pear.maarch.org >/dev/null || ${:pear} channel-discover pear.maarch.org
+    ${:pear} info maarch/CLITools-0.3.1 >/dev/null || ${:pear} install maarch/CLITools-0.3.1
+    ${:pear} info MIME_Type >/dev/null || ${:pear} install MIME_Type
+

software/postgres/instance.cfg.in

+[buildout]
+parts =
+  symlinks
+  publish
+  postgres-instance
+
+
+# Define egg directories to be the one from Software Release
+# (/opt/slapgrid/...)
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+
+[instance-parameters]
+# Fetches parameters defined in SlapOS Master for this instance
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+
+[directories]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+etc = $${buildout:directory}/etc
+services = $${directories:etc}/run/
+promises = $${directories:etc}/promise/
+var = $${buildout:directory}/var
+
+
+[symlinks]
+recipe = cns.recipe.symlink
+symlink_target = $${directories:bin}
+symlink_base = ${postgresql:location}/bin
+
+
+[postgres-instance]
+# create cluster, configuration files and a database
+recipe = slapos.cookbook:postgres
+
+# Options
+ipv6_host = $${instance-parameters:ipv6}
+user = user
+port = 5432
+dbname = db
+# pgdata_directory is created by initdb, and should not exist beforehand.
+pgdata-directory = $${directories:var}/data
+services = $${directories:services}
+bin = $${directories:bin}
+
+
+[publish]
+recipe = slapos.cookbook:publishurl
+url = $${postgres-instance:url}
+
+
+[slap-connection]
+# part to migrate to new - separated words
+computer-id = $${slap_connection:computer_id}
+partition-id = $${slap_connection:partition_id}
+server-url = $${slap_connection:server_url}
+software-release-url = $${slap_connection:software_release_url}
+key-file = $${slap_connection:key_file}
+cert-file = $${slap_connection:cert_file}
+

software/postgres/software.cfg

+[buildout]
+
+extends = 
+  ../../stack/slapos.cfg
+  ../../component/postgresql/buildout.cfg
+
+parts =
+  eggs
+  slapos-cookbook
+  instance-template
+  postgresql
+
+[instance-template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg.in
+output = ${buildout:directory}/template.cfg
+#md5sum = 
+mode = 0644
+
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+    cns.recipe.symlink
+

software/slaprunner/instance-runner.cfg

@@ -2,6 +2,7 @@
 parts =
   cloud9
   slaprunner
+  test-runner
   sshkeys-dropbear
   dropbear-server-add-authorized-key
   sshkeys-authority
@@ -11,7 +12,7 @@ parts =
   cloud9-promise
   dropbear-promise
 
-  
+
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
@@ -38,13 +39,18 @@ services = $${rootdirectory:etc}/run/
 run = $${rootdirectory:var}/run/
 backup = $${rootdirectory:srv}/backup/
 promises = $${rootdirectory:etc}/promise/
+test = $${rootdirectory:etc}/test/
 
 [runnerdirectory]
 recipe = slapos.cookbook:mkdirectory
 home = $${rootdirectory:srv}/runner/
+test = $${rootdirectory:srv}/test/
 project = $${:home}/project
 software-root = $${:home}/software
 instance-root = $${:home}/instance
+project-test = $${:test}/project
+software-test = $${:test}/software
+instance-test = $${:test}/instance
 
 #Create password recovery code for slaprunner
 [recovery-code]
@@ -85,9 +91,10 @@ ipv4 = $${slap-network-information:local-ipv4}
 ipv6 = $${slap-network-information:global-ipv6}
 proxy_port = 50000
 runner_port = 50000
-partition-amount = 7
+partition-amount = $${slap-parameter:instance-amount}
 cloud9-url = $${cloud9:access-url}
 wrapper = $${basedirectory:services}/slaprunner
+debug = False
 
 # Deploy dropbear (minimalist ssh server)
 [sshkeys-directory]
@@ -95,6 +102,18 @@ recipe = slapos.cookbook:mkdirectory
 requests = $${directory:sshkeys}/requests/
 keys = $${directory:sshkeys}/keys/
 
+[test-runner]
+<= slaprunner
+recipe = slapos.cookbook:slaprunner.test
+slaprunnertest = ${buildout:directory}/bin/slaprunnertest
+slapos.cfg = $${rootdirectory:etc}/slapos-test.cfg
+working-directory = $${runnerdirectory:test}
+project-directory = $${runnerdirectory:project-test}
+software-directory = $${runnerdirectory:software-test}
+instance-directory = $${runnerdirectory:instance-test}
+wrapper = $${rootdirectory:bin}/runTestSuite
+etc_dir = $${basedirectory:test}
+
 [sshkeys-authority]
 recipe = slapos.cookbook:sshkeys_authority
 request-directory = $${sshkeys-directory:requests}
@@ -182,3 +201,5 @@ port = $${dropbear-server:port}
 [slap-parameter]
 # Default value if no ssh key is specified
 authorized-key =
+# Default value of instances number in slaprunner
+instance-amount = 10

software/slaprunner/software.cfg

@@ -25,7 +25,7 @@ md5sum = 5307e4200f044ae57b504ad68444491c
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner.cfg
 output = ${buildout:directory}/template-runner.cfg
-md5sum = b57ebff565595fa92cd7d5adc6a22c7e
+md5sum = bd1c73c149b4adf2fbab80bc29c782c8
 mode = 0644
 
 [eggs]
@@ -54,44 +54,48 @@ signature-certificate-list =
   -----END CERTIFICATE-----
 
 [versions]
+# Forced stable version of lxml - 3.0 alpha keeps appearing/diseappearing on
+# Pypi
+lxml = 2.3.6
+
 Flask-Auth = 0.8
 Jinja2 = 2.6
 Werkzeug = 0.8.3
-apache-libcloud = 0.11.1
+apache-libcloud = 0.11.3
 async = 0.6.1
 buildout-versions = 1.7
 gitdb = 0.5.4
 hexagonit.recipe.cmmi = 1.6
-meld3 = 0.6.8
+meld3 = 0.6.9
 plone.recipe.command = 1.1
 pycrypto = 2.6
-slapos.cookbook = 0.64.2
+slapos.cookbook = 0.68.1
 slapos.libnetworkcache = 0.13.2
-slapos.recipe.build = 0.11
+slapos.recipe.build = 0.11.5
 slapos.recipe.template = 2.4.2
-slapos.toolbox = 0.30
+slapos.toolbox = 0.31
 smmap = 0.8.2
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
-# slapos.core==0.28.5
-# slapos.toolbox==0.30
+# slapos.core==0.31.2
+# slapos.toolbox==0.31
 Flask = 0.9
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31
 GitPython = 0.3.2.RC1
 
 # Required by:
-# slapos.cookbook==0.64.2
+# slapos.cookbook==0.68
 PyXML = 0.8.4
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31
 atomize = 0.1.1
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31
 feedparser = 5.1.2
 
 # Required by:
@@ -99,58 +103,56 @@ feedparser = 5.1.2
 hexagonit.recipe.download = 1.5.1
 
 # Required by:
-# slapos.cookbook==0.64.2
+# slapos.cookbook==0.68
 inotifyx = 0.2.0
 
 # Required by:
-# slapos.core==0.28.5
-# slapos.toolbox==0.30
-# xml-marshaller==0.9.7
-lxml = 3.0alpha2
-
-# Required by:
-# slapos.cookbook==0.64.2
-netaddr = 0.7.9
+# slapos.cookbook==0.68
+netaddr = 0.7.10
 
 # Required by:
-# slapos.core==0.28.5
+# slapos.core==0.31.2
 netifaces = 0.8
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31
 paramiko = 1.7.7.2
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31
 psutil = 0.6.1
 
 # Required by:
-# slapos.cookbook==0.64.2
-# slapos.core==0.28.5
+# slapos.cookbook==0.68
+pytz = 2012f
+
+# Required by:
+# slapos.cookbook==0.68
+# slapos.core==0.31.2
 # slapos.libnetworkcache==0.13.2
-# slapos.toolbox==0.30
-# supervisor==3.0a12
+# slapos.toolbox==0.31
+# supervisor==3.0b1
 # zc.buildout==1.6.0-dev-SlapOS-007
 # zc.recipe.egg==1.3.2
 # zope.interface==4.0.1
 setuptools = 0.6c12dev-r88846
 
 # Required by:
-# slapos.cookbook==0.64.2
-slapos.core = 0.28.5
+# slapos.cookbook==0.68
+slapos.core = 0.31.2
 
 # Required by:
-# slapos.core==0.28.5
-supervisor = 3.0a12
+# slapos.core==0.31.2
+supervisor = 3.0b1
 
 # Required by:
-# slapos.cookbook==0.64.2
+# slapos.cookbook==0.68
 xml-marshaller = 0.9.7
 
 # Required by:
-# slapos.cookbook==0.64.2
+# slapos.cookbook==0.68
 zc.recipe.egg = 1.3.2
 
 # Required by:
-# slapos.core==0.28.5
+# slapos.core==0.31.2
 zope.interface = 4.0.1

stack/erp5/buildout.cfg

@@ -17,7 +17,7 @@ find-links =
 
 extends =
 # Exact version of Zope
-  http://svn.zope.org/repos/main/Zope/tags/2.12.23/versions.cfg
+  http://svn.zope.org/repos/main/Zope/tags/2.12.25/versions.cfg
   ../../stack/slapos.cfg
   ../../component/logrotate/buildout.cfg
   ../../component/dcron/buildout.cfg
@@ -48,7 +48,6 @@ extends =
   ../../component/python-2.7/buildout.cfg
   ../../component/python-ldap-python/buildout.cfg
   ../../component/rdiff-backup/buildout.cfg
-  ../../component/sphinx/buildout.cfg
   ../../component/stunnel/buildout.cfg
   ../../component/subversion/buildout.cfg
   ../../component/tesseract/buildout.cfg
@@ -85,7 +84,6 @@ parts =
   ghostscript
   mariadb
   mroonga-mariadb
-  sphinx
   imagemagick
   libdmtx
   dmtx-utils
@@ -186,7 +184,7 @@ context =
 [template-mariadb]
 < = template-jinja2-base
 filename = instance-mariadb.cfg
-md5sum = 0e8e9bfc8c8a5f1bd71f5c4f346ef68b
+md5sum = 867bd8e3c4f56db2aab2fcad59f2efd8
 extra-context =
     key coreutils_location coreutils:location
     key dcron_location dcron:location
@@ -200,13 +198,6 @@ extra-context =
     key sed_location sed:location
     key xtrabackup_location xtrabackup:location
 
-[template-sphinx]
-< = template-jinja2-base
-filename = instance-sphinx.cfg
-md5sum = e3975afd5220f251d6b6e7024a2ca08b
-extra-context =
-    key sphinx_location sphinx:location
-
 [template-zope]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-zope.cfg.in
@@ -227,7 +218,7 @@ extra-context =
 [template-tidstorage]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-tidstorage.cfg.in
-md5sum = 520e39f2612a045b42c0ad914bfdc5a9
+md5sum = 1f3e7cfc719d3c2eee4860678f6a67df
 mode = 640
 
 [template-cloudooo]
@@ -239,7 +230,7 @@ mode = 640
 [template-zope-conf]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/zope.conf.in
-md5sum = 599e004c18fd3461aa846ae75bd112f2
+md5sum = bc2154161a1d5baddc4ed4dfaaf94fbe
 mode = 640
 
 [template]
@@ -247,7 +238,7 @@ mode = 640
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 filename = template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 26d26c357053af48602b5110120bd085
+md5sum = 700b8729a4d9a8423ab2a99de486cc31
 extra-context =
     key apache_location apache:location
     key aspell_location aspell:location
@@ -292,7 +283,6 @@ extra-context =
     key template_kumofs template-kumofs:rendered
     key template_mariadb template-mariadb:rendered
     key template_memcached template-memcached:rendered
-    key template_sphinx template-sphinx:rendered
     key template_tidstorage template-tidstorage:target
     key template_varnish template-varnish:target
     key template_zope template-zope:target
@@ -320,7 +310,7 @@ mode = 640
 [template-erp5-cluster]
 < = template-jinja2-base
 filename = instance-erp5-cluster.cfg
-md5sum = 3051f3fea39d7409c417dcac26027b56
+md5sum = b4a9688ca7983fac5dd573e66c6e4263
 extra-context =
     key local_bt5_repository local-bt5-repository:list
 
@@ -473,9 +463,10 @@ eggs =
   huBarcode
   qrcode
 
-# Zope 2.12 with patched acquisition
+# Zope
   ZODB3
   Zope2
+# Zope acquisition patch
   Acquisition
 
 # Other Zope 2 packages
@@ -574,7 +565,7 @@ pysvn = 1.7.4nxd006
 
 # CMF 2.3 requries Zope 2.13.
 Products.CMFCalendar = 2.2.2
-Products.CMFCore = 2.2.5
+Products.CMFCore = 2.2.6
 Products.CMFDefault = 2.2.2
 Products.CMFTopic = 2.2.1
 Products.CMFUid = 2.2.1
@@ -583,7 +574,7 @@ Products.CMFUid = 2.2.1
 slapos.cookbook =
 
 # Pinned versions
-Flask = 0.8
+Flask = 0.9
 GitPython = 0.3.2.RC1
 MySQL-python = 1.2.3
 PIL = 1.1.7
@@ -591,68 +582,63 @@ Paste = 1.7.5.1
 PasteDeploy = 1.5.0
 PasteScript = 1.7.5
 Products.CMFActionIcons = 2.1.3
-Products.CMFCalendar = 2.2.2
-Products.CMFCore = 2.2.5
-Products.CMFDefault = 2.2.2
-Products.CMFTopic = 2.2.1
-Products.CMFUid = 2.2.1
 Products.DCWorkflowGraph = 0.4.1
 Products.ExternalEditor = 1.1.0
-Products.GenericSetup = 1.7.1
+Products.GenericSetup = 1.7.2
 Products.MimetypesRegistry = 2.0.3
-Products.PluggableAuthService = 1.7.7
-Products.PluginRegistry = 1.3b1
-Products.TIDStorage = 5.4.7.dev-r45842
+Products.PluggableAuthService = 1.8.0
+Products.PluginRegistry = 1.3
+Products.TIDStorage = 5.4.8
 Products.Zelenium = 1.0.3
 StructuredText = 2.11.1
 WSGIUtils = 0.7
 Werkzeug = 0.8.3
-apache-libcloud = 0.9.1
-argparse = 1.1
+apache-libcloud = 0.11.1
+argparse = 1.2.1
 async = 0.6.1
 atomize = 0.1.1
 buildout-versions = 1.7
 cElementTree = 1.0.5-20051216
-chardet = 1.0.1
-coverage = 3.5.1
+chardet = 1.1
+coverage = 3.5.2
 csp-eventlet = 0.7.0
 elementtree = 1.2.7-20070827-preview
 erp5.recipe.cmmiforcei686 = 0.1.3
-erp5.util = 0.4.6
+erp5.util = 0.4.7
 erp5diff = 0.8.1.5
-eventlet = 0.9.16
-feedparser = 5.1.1
+eventlet = 0.9.17
+feedparser = 5.1.2
 five.localsitemanager = 2.0.5
 fpconst = 0.7.2
 gitdb = 0.5.4
-greenlet = 0.3.4
-hexagonit.recipe.cmmi = 1.5.0
-http-parser = 0.7.5
+greenlet = 0.4.0
+hexagonit.recipe.cmmi = 1.6
+http-parser = 0.7.8
 huBarcode = 0.63
 inotifyx = 0.2.0
-ipdb = 0.6.1
-ipython = 0.12
+ipdb = 0.7
+ipython = 0.13
 meld3 = 0.6.8
-netaddr = 0.7.6
+netaddr = 0.7.9
 netifaces = 0.8
 ordereddict = 1.1
-paramiko = 1.7.7.1
+paramiko = 1.7.7.2
 plone.recipe.command = 1.1
 ply = 3.4
 polib = 1.0.0
-psutil = 0.4.1
-python-ldap = 2.4.9
-python-magic = 0.4.2
+psutil = 0.6.1
+python-ldap = 2.4.10
+python-magic = 0.4.3
 python-memcached = 1.47
-restkit = 4.1.2
+restkit = 4.2.0
 rtjp-eventlet = 0.3.2
 setuptools = 0.6c12dev-r88846
-slapos.core = 0.24
-slapos.recipe.build = 0.7
-slapos.recipe.template = 2.4
-slapos.toolbox = 0.23
+slapos.core = 0.28
+slapos.recipe.build = 0.10
+slapos.recipe.template = 2.4.2
+slapos.toolbox = 0.29
 smmap = 0.8.2
-socketpool = 0.3.0
+socketpool = 0.5.2
 supervisor = 3.0a12
 threadframe = 0.2
 timerserver = 2.0.2
@@ -661,3 +647,6 @@ uuid = 1.30
 validictory = 0.8.3
 xml-marshaller = 0.9.7
 xupdate-processor = 0.4
+mr.developer = 1.21
+qrcode = 2.4.1
+requests = 0.13.2

stack/erp5/instance-erp5-cluster.cfg.in

@@ -24,13 +24,6 @@ cert-file = ${slap-connection:cert-file}
 computer-id = ${slap-connection:computer-id}
 partition-id = ${slap-connection:partition-id}
 
-[request-sphinx]
-<=request-common
-name = Sphinx Search Engine
-software-type = sphinx
-sla-computer_guid = ${slap-parameter:sphinx-computer-guid}
-return = url-sphinx url-sphinx-sql
-
 [request-mariadb]
 <=request-common
 name = MariaDB DataBase
@@ -67,8 +60,6 @@ config =
   memcached-url
   cloudooo-url
   kumofs-url
-  sphinx-url-sphinx
-  sphinx-url-sphinx-sql
   smtp-url
   bt5
   bt5-repository-url
@@ -77,8 +68,6 @@ config-mysql-url = ${request-mariadb:connection-url}
 config-memcached-url = ${request-memcached:connection-url}
 config-cloudooo-url = ${request-cloudooo:connection-url}
 config-kumofs-url = ${request-kumofs:connection-url}
-config-sphinx-url-sphinx = ${request-sphinx:connection-url-sphinx}
-config-sphinx-url-sphinx-sql = ${request-sphinx:connection-url-sphinx-sql}
 config-bt5 = ${slap-parameter:bt5}
 config-bt5-repository-url = ${slap-parameter:bt5-repository-url}
 config-smtp-url = ${slap-parameter:smtp-url}
@@ -97,7 +86,6 @@ sla-computer_guid = ${slap-parameter:varnish-computer-guid}
 
 [slap-parameter]
 # Default value if no computer_guid is specified for each type
-sphinx-computer-guid = ${slap-connection:computer-id}
 mariadb-computer-guid = ${slap-connection:computer-id}
 cloudooo-computer-guid = ${slap-connection:computer-id}
 memcached-computer-guid = ${slap-connection:computer-id}

stack/erp5/instance-mariadb.cfg.in

@@ -30,7 +30,7 @@ recipe = slapos.cookbook:generic.mysql
 
 # Options
 user = user
-parallel-test-database-amount = 100
+parallel-test-database-amount = ${slap-parameter:test-database-amount}
 port = 45678
 ip = ${slap-network-information:local-ipv4}
 database = erp5
@@ -178,3 +178,6 @@ recipe = slapos.cookbook:check_port_listening
 path = ${basedirectory:promise}/mariadb
 hostname = ${mariadb-instance:ip}
 port = ${mariadb-instance:port}
+
+[slap-parameter]
+test-database-amount = 100

stack/erp5/instance-sphinx.cfg.in

-[buildout]
-parts =
-  publish-sphinx-url
-  sphinxd-instance
-
-eggs-directory = {{ eggs_directory }}
-develop-eggs-directory = {{ develop_eggs_directory }}
-offline = true
-
-[publish-sphinx-url]
-recipe = slapos.cookbook:publish
-url-sphinx = sphinx://${sphinxd-instance:ip}:${sphinxd-instance:sphinx-port}
-url-sphinx-sql = mysql://${sphinxd-instance:ip}:${sphinxd-instance:sql-port}
-
-[sphinxd-instance]
-recipe = slapos.cookbook:sphinx
-data-directory = ${directory:sphinx-data}
-configuration-file = ${rootdirectory:etc}/sphinx.conf
-searchd-log = ${basedirectory:log}/sphinx-searchd.log
-query-log = ${basedirectory:log}/sphinx-query.log
-pid = ${basedirectory:run}/sphinx-searchd.pid
-ip = ${slap-network-information:local-ipv4}
-sphinx-port = 9312
-sql-port = 9306
-wrapper = ${basedirectory:services}/sphinxd
-sphinx-searchd-binary = {{ sphinx_location }}/bin/searchd
-
-[rootdirectory]
-recipe = slapos.cookbook:mkdirectory
-etc = ${buildout:directory}/etc
-var = ${buildout:directory}/var
-srv = ${buildout:directory}/srv
-bin = ${buildout:directory}/bin
-
-[basedirectory]
-recipe = slapos.cookbook:mkdirectory
-log = ${rootdirectory:var}/log
-services = ${rootdirectory:etc}/run
-run = ${rootdirectory:var}/run
-
-[directory]
-recipe = slapos.cookbook:mkdirectory
-sphinx-data = ${rootdirectory:srv}/sphinx

stack/erp5/instance-tidstorage.cfg.in

@@ -485,7 +485,6 @@ recipe = slapos.cookbook:publish
 [tidstorage]
 recipe = slapos.cookbook:tidstorage
 known-tid-storage-identifier-dict = {{ known_tid_storage_identifier_dict }}
-base-url = http://${zope-admin:ip}:${zope-admin:port}/%s/serialize
 configuration-path = ${directory:etc}/tidstorage.py
 ip = {{ ipv4 }}
 port = 6001

stack/erp5/instance.cfg.in

@@ -197,7 +197,6 @@ memcached = {{ template_memcached }}
 cloudooo = ${dynamic-template-cloudooo:rendered}
 zope = ${dynamic-template-zope:rendered}
 mariadb = {{ template_mariadb }}
-sphinx = {{ template_sphinx }}
 tidstorage = ${dynamic-template-tidstorage:rendered}
 varnish = ${dynamic-template-varnish:rendered}
 

stack/erp5/zope.conf.in

@@ -43,10 +43,11 @@ products {{ instance_products }}
 </product-config>
 
 {% endif -%}
-{% if 'timerserver-interval' in parameter_dict -%}
+{% set timerserver_interval = parameter_dict.get('timerserver-interval', '0') | int -%}
+{% if timerserver_interval -%}
 %import timerserver
 <timer-server>
-  interval {{ parameter_dict['timerserver-interval'] }}
+  interval {{ timerserver_interval }}
 </timer-server>
 
 {% endif -%}

stack/lamp/apache/instance-apache-import.cfg

+[buildout]
+extends = ${template-apache-php:output}
+          ${template-pbsready-export:output}
+
+parts =
+  apache-proxy
+  logrotate
+  logrotate-entry-apache
+  cron
+  cron-entry-logrotate
+  sshkeys-authority
+  sshkeys-dropbear
+  dropbear-server
+  dropbear-server-pbs-authorized-key
+
+[apache-proxy]
+recipe = slapos.cookbook:apacheproxy
+url = $${slap-parameter:proxy-url}
+pid-file = $${basedirectory:run}/apache.pid
+lock-file = $${basedirectory:run}/apache.lock
+ip = $${slap-network-information:global-ipv6}
+port = 8080
+error-log = $${directory:httpd-log}/error.log
+access-log = $${directory:httpd-log}/access.log
+httpd-conf = $${rootdirectory:etc}/apache.conf
+wrapper = $${basedirectory:services}/apache
+
+promise = $${basedirectory:promises}/apache
+
+httpd-binary = ${apache:location}/bin/httpd
+
+[sshkeys-directory]
+recipe = slapos.cookbook:mkdirectory
+requests = $${directory:sshkeys}/requests/
+keys = $${directory:sshkeys}/keys/
+
+[sshkeys-authority]
+recipe = slapos.cookbook:sshkeys_authority
+request-directory = $${sshkeys-directory:requests}
+keys-directory = $${sshkeys-directory:keys}
+wrapper = $${basedirectory:services}/sshkeys_authority
+keygen-binary = ${dropbear:location}/bin/dropbearkey
+
+[sshkeys-dropbear]
+<= sshkeys-authority
+recipe = slapos.cookbook:sshkeys_authority.request
+name = dropbear
+type = rsa
+executable = $${dropbear-server:wrapper}
+public-key = $${dropbear-server:rsa-keyfile}.pub
+private-key = $${dropbear-server:rsa-keyfile}
+wrapper = $${basedirectory:services}/sshd
+
+[dropbear-server]
+recipe = slapos.cookbook:dropbear
+host = $${slap-network-information:global-ipv6}
+port = 2222
+home = $${directory:ssh}
+wrapper = $${rootdirectory:bin}/raw_sshd
+shell = $${rdiff-backup-server:wrapper}
+rsa-keyfile = $${directory:ssh}/server_key.rsa
+dropbear-binary = ${dropbear:location}/sbin/dropbear
+
+[dropbear-server-pbs-authorized-key]
+<= dropbear-server
+recipe = slapos.cookbook:dropbear.add_authorized_key
+key = $${slap-parameter:authorized-key}
+
+[rdiff-backup-server]
+recipe = slapos.cookbook:pbs
+client = false
+path = $${directory:htdocs}
+wrapper = $${rootdirectory:bin}/rdiffbackup-server
+rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = ${gzip:location}/bin/gzip
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${rootdirectory:bin}/logrotate
+conf = $${rootdirectory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${rootdirectory:srv}/logrotate.status
+
+[logrotate-entry-apache]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = apache
+log = $${apache-proxy:error-log} $${apache-proxy:access-log}
+frequency = daily
+rotate-num = 30
+sharedscripts = true
+notifempty = true
+create = true
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc/
+var = $${buildout:directory}/var/
+srv = $${buildout:directory}/srv/
+bin = $${buildout:directory}/bin/
+tmp = $${buildout:directory}/tmp/
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log/
+services = $${rootdirectory:etc}/run/
+run = $${rootdirectory:var}/run/
+backup = $${rootdirectory:srv}/backup/
+promises = $${rootdirectory:etc}/promise/
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+htdocs = $${rootdirectory:srv}/www/
+logrotate-entries = $${rootdirectory:etc}/logrotate.d/
+logrotate-backup = $${basedirectory:backup}/logrotate/
+cronstamps = $${rootdirectory:etc}/cronstamps/
+cron-entries = $${rootdirectory:etc}/cron.d/
+crontabs = $${rootdirectory:etc}/crontabs/
+ssh = $${rootdirectory:etc}/ssh/
+sshkeys = $${rootdirectory:srv}/sshkeys
+httpd-log = $${basedirectory:log}/apache/

stack/lamp/instance-apache-php.cfg → stack/lamp/apache/instance-apache-php.cfg

@@ -107,7 +107,7 @@ software-url = $${slap-connection:software-release-url}
 software-type = mariadb
 return = url
 sla = computer_guid
-sla-computer_guid = ${slap-connection:computer-id}
+sla-computer_guid = $${slap-connection:computer-id}
 
 [mariadb-urlparse]
 recipe = slapos.cookbook:urlparse

stack/lamp/buildout.cfg

 [buildout]
+
+ignore-existing = true
+
 parts =
   apache-php
   mariadb
@@ -9,13 +12,16 @@ parts =
   eggs
   instance-recipe-egg
   template
+
   template-apache-php
   template-mariadb
-  template-mariadb-pbsready
-  template-mariadb-pbsready-import
-  template-mariadb-pbsready-export
+
+#Contains the importer and exporter recipes for mariadb
+  template-mariadb-import
+  template-mariadb-export
 
 extends =
+  ../resilient/buildout.cfg
   ../../component/mariadb/buildout.cfg
   ../../component/apache/buildout.cfg
   ../../component/apache-php/buildout.cfg
@@ -59,64 +65,60 @@ strip-top-level-dir = true
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
 output = ${buildout:directory}/template.cfg
-md5sum = 8117f10e814a13c5376af4c01e6546d4
+md5sum = 8b4660ccaccda1fa8b0e73b8ac38be11
 mode = 0644
 
 [template-apache-php]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-apache-php.cfg
+url = ${:_profile_base_location_}/apache/instance-apache-php.cfg
 output = ${buildout:directory}/template-apache-php.cfg
-md5sum = 59cee571ea0fcdf2e6c9b3195e258738
+md5sum = a5dd222b3faa4e1ef2df9b3b9bb47966
 mode = 0644
 
 [template-apache-backup]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-apache-backup.cfg
+url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg
 output = ${buildout:directory}/template-apache-backup.cfg
 md5sum = cfb77ac8785e0d125a785f69a5339014
 mode = 0644
 
-[template-backuped]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-backuped.cfg
-output = ${buildout:directory}/template-backuped.cfg
-md5sum = f43d1c6412ea8dc83b75573dc00daf9e
+[template-resilient-lamp]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance-resilient.cfg
+rendered = ${buildout:directory}/template-resilient.cfg
+
+context = key templateapache template-apache-php:output
+          key dropbear dropbear:location
+          key buildout buildout:bin-directory
+
+import-list = file parts template-parts:destination
+              file replicated template-replicated:destination
+
+md5sum = 03aafcba5c626a4a1bd180d71007be1e
 mode = 0644
+ 
 
 [template-mariadb]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-mariadb.cfg
+url = ${:_profile_base_location_}/mariadb/instance-mariadb.cfg
 output = ${buildout:directory}/template-mariadb.cfg
-md5sum = 767452bc503ff6c1b7af0ebfac590c9f
+md5sum = fa9dc10efbcf61119f4cbab37c741322
 mode = 0644
 
-[template-mariadb-pbsready]
+[template-mariadb-import]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-mariadb-pbsready.cfg
-output = ${buildout:directory}/template-mariadb-pbsready.cfg
-md5sum = d2a580dcd7efdd528be45c5ffadfe760
+url = ${:_profile_base_location_}/mariadb/instance-mariadb-import.cfg
+output = ${buildout:directory}/template-mariadb-import.cfg
+md5sum = fa696733db4bd5b2e3e9fb6e0b09c59b
 mode = 0644
 
-[template-mariadb-pbsready-import]
+[template-mariadb-export]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-mariadb-pbsready-import.cfg
-output = ${buildout:directory}/template-mariadb-pbsready-import.cfg
-md5sum = 4a96ff02da3898fef7077fa8baec81ac
+url = ${:_profile_base_location_}/mariadb/instance-mariadb-export.cfg
+output = ${buildout:directory}/template-mariadb-export.cfg
+md5sum = 4b7dec765265b27c8235419b82ca7b02
 mode = 0644
 
-[template-mariadb-pbsready-export]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-mariadb-pbsready-export.cfg
-output = ${buildout:directory}/template-mariadb-pbsready-export.cfg
-md5sum = 11a9e45e8bc590bc11bfdd304b07a4a5
-mode = 0644
-
-[template-pull-backup]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-pull-backup.cfg
-output = ${buildout:directory}/template-pull-backup.cfg
-md5sum = 9aab30ba5aa23a37d4b507e7c414be00
-mode = 0644
 
 # Dummy parts in case no application configuration file is needed
 [application-template]
@@ -265,43 +267,45 @@ signature-certificate-list =
   -----END CERTIFICATE-----
 
 [versions]
+# 1.2.4b doesn't download
+MySQL-python = 1.2.3
+
 Jinja2 = 2.6
 Werkzeug = 0.8.3
-apache-libcloud = 0.11.1
+apache-libcloud = 0.11.3
 async = 0.6.1
 buildout-versions = 1.7
 gitdb = 0.5.4
 hexagonit.recipe.cmmi = 1.6
-meld3 = 0.6.8
+meld3 = 0.6.9
 pycrypto = 2.6
 rdiff-backup = 1.0.5
-slapos.cookbook = 0.66
-slapos.libnetworkcache = 0.13.2
-slapos.recipe.build = 0.11.2
+slapos.cookbook = 0.68
+slapos.recipe.build = 0.11.5
 slapos.recipe.download = 1.0.dev-r4053
 slapos.recipe.template = 2.4.2
-slapos.toolbox = 0.30
+slapos.toolbox = 0.31
 smmap = 0.8.2
 
 # Required by:
-# slapos.core==0.28.5
-# slapos.toolbox==0.30
+# slapos.core==0.31.2
+# slapos.toolbox==0.31
 Flask = 0.9
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31
 GitPython = 0.3.2.RC1
 
 # Required by:
-# slapos.cookbook==0.66
+# slapos.cookbook==0.68
 PyXML = 0.8.4
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31
 atomize = 0.1.1
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31
 feedparser = 5.1.2
 
 # Required by:
@@ -309,58 +313,61 @@ feedparser = 5.1.2
 hexagonit.recipe.download = 1.5.1
 
 # Required by:
-# slapos.cookbook==0.66
+# slapos.cookbook==0.68
 inotifyx = 0.2.0
 
 # Required by:
-# slapos.cookbook==0.66
-# slapos.core==0.28.5
+# slapos.cookbook==0.68
+# slapos.core==0.31.2
 # xml-marshaller==0.9.7
 lxml = 3.0alpha2
 
 # Required by:
-# slapos.cookbook==0.66
+# slapos.cookbook==0.68
 netaddr = 0.7.10
 
 # Required by:
-# slapos.core==0.28.5
+# slapos.core==0.31.2
 netifaces = 0.8
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31
 paramiko = 1.7.7.2
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31
 psutil = 0.6.1
 
 # Required by:
-# slapos.cookbook==0.66
-# slapos.core==0.28.5
-# slapos.toolbox==0.30
-# zc.buildout==1.6.0-dev-SlapOS-006
+# slapos.cookbook==0.68
+pytz = 2012f
+
+# Required by:
+# slapos.cookbook==0.68
+# slapos.core==0.31.2
+# slapos.toolbox==0.31
+# zc.buildout==1.6.0-dev-SlapOS-010
 # zc.recipe.egg==1.3.2
 setuptools = 0.6c12dev-r88846
 
 # Required by:
-# slapos.cookbook==0.66
-# slapos.toolbox==0.30
-slapos.core = 0.28.5
+# slapos.cookbook==0.68
+# slapos.toolbox==0.31
+slapos.core = 0.31.2
 
 # Required by:
-# slapos.core==0.28.5
-supervisor = 3.0a12
+# slapos.core==0.31.2
+supervisor = 3.0b1
 
 # Required by:
-# slapos.cookbook==0.66
-# slapos.toolbox==0.30
+# slapos.cookbook==0.68
+# slapos.toolbox==0.31
 xml-marshaller = 0.9.7
 
 # Required by:
-# slapos.cookbook==0.66
+# slapos.cookbook==0.68
 zc.recipe.egg = 1.3.2
 
 # Required by:
-# slapos.core==0.28.5
+# slapos.core==0.31.2
 zope.interface = 4.0.1
-

stack/lamp/instance-backuped.cfg

-[buildout]
-
-extends =
-  ${template-apache-php:output}
-
-parts =
-  request-mariadb
-  request-mariadb-pseudo-replicating
-  request-apache-backup
-  request-pull-backup-server
-  request-pull-backup-server-mariadb
-  request-pull-backup-server-mariadb-backup
-  request-pull-backup-server-apache
-  request-pull-backup-server-apache-backup
-  url
-  apache-php
-  stunnel
-  certificate-authority
-  ca-stunnel
-  logrotate
-  logrotate-entry-apache
-  logrotate-entry-stunnel
-  cron
-  cron-entry-logrotate
-  dropbear-server
-  sshkeys-authority
-  dropbear-server-pbs-authorized-key
-
-[sshkeys-directory]
-recipe = slapos.cookbook:mkdirectory
-requests = $${directory:sshkeys}/requests/
-keys = $${directory:sshkeys}/keys/
-
-[sshkeys-authority]
-recipe = slapos.cookbook:sshkeys_authority
-request-directory = $${sshkeys-directory:requests}
-keys-directory = $${sshkeys-directory:keys}
-wrapper = $${basedirectory:services}/sshkeys_authority
-keygen-binary = ${dropbear:location}/bin/dropbearkey
-
-[sshkeys-dropbear]
-<= sshkeys-authority
-recipe = slapos.cookbook:sshkeys_authority.request
-name = dropbear
-type = rsa
-executable = $${dropbear-server:wrapper}
-public-key = $${dropbear-server:rsa-keyfile}.pub
-private-key = $${dropbear-server:rsa-keyfile}
-wrapper = $${basedirectory:services}/sshd
-
-[dropbear-server]
-recipe = slapos.cookbook:dropbear
-host = $${slap-network-information:global-ipv6}
-port = 2222
-home = $${directory:ssh}
-wrapper = $${rootdirectory:bin}/raw_sshd
-shell = $${rdiff-backup-server:wrapper}
-rsa-keyfile = $${directory:ssh}/server_key.rsa
-dropbear-binary = ${dropbear:location}/sbin/dropbear
-
-[dropbear-server-pbs-authorized-key]
-<= dropbear-server
-recipe = slapos.cookbook:dropbear.add_authorized_key
-key = $${request-pull-backup-server:connection-ssh-key}
-
-[rdiff-backup-server]
-recipe = slapos.cookbook:pbs
-client = false
-path = $${directory:htdocs}
-wrapper = $${rootdirectory:bin}/rdiffbackup-server
-rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
-
-[request-pull-backup-server]
-<= slap-connection
-recipe = slapos.cookbook:request
-name = PBS (Pull Backup Server)
-software-url = $${slap-connection:software-release-url}
-software-type = pull-backup
-return = ssh-key notification-url feeds-url
-slave = false
-
-[request-mariadb]
-software-type = mariadb-pbsready-export
-config = authorized-key notify
-config-authorized-key = $${request-pull-backup-server:connection-ssh-key}
-config-notify = $${request-pull-backup-server:connection-notification-url}
-return = url ssh-public-key ssh-url notification-id
-
-[request-mariadb-pseudo-replicating]
-<= slap-connection
-recipe = slapos.cookbook:request
-name = MariaDB Backup
-software-url = $${slap-connection:software-release-url}
-software-type = mariadb-pbsready-import
-return = url ssh-public-key ssh-url notification-url
-config = authorized-key on-notification
-config-authorized-key = $${request-pull-backup-server:connection-ssh-key}
-pbs-notification-id = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb-push
-config-on-notification = $${request-pull-backup-server:connection-feeds-url}$${:pbs-notification-id}
-
-[request-apache-backup]
-<= slap-connection
-recipe = slapos.cookbook:request
-name = Apache Backup
-software-url = $${slap-connection:software-release-url}
-software-type = apache-backup
-return = url ssh-url ssh-public-key
-config = authorized-key proxy-url
-config-authorized-key = $${request-pull-backup-server:connection-ssh-key}
-config-proxy-url = $${url:url}
-
-[request-pull-backup-server-apache]
-<= request-pull-backup-server
-   slap-connection
-recipe = slapos.cookbook:request
-name = PBS pulling from Apache
-software-url = $${slap-connection:software-release-url}
-config = url name type server-key notify notification-id frequency
-config-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
-config-name = $${slap-connection:computer-id}-$${slap-connection:partition-id}-apache
-config-type = pull
-config-server-key = $${sshkeys-dropbear:public-key-value}
-config-notify = $${request-pull-backup-server:connection-notification-url}
-config-notification-id = $${slap-connection:computer-id}-$${slap-connection:partition-id}-apache-pull
-config-frequency = 30 * * * *
-slave = true
-
-[request-pull-backup-server-apache-backup]
-<= request-pull-backup-server
-   slap-connection
-recipe = slapos.cookbook:request
-name = PBS pushing to $${request-apache-backup:name}
-software-url = $${slap-connection:software-release-url}
-config = url name type server-key on-notification
-config-url = $${request-apache-backup:connection-ssh-url}
-config-name = $${request-pull-backup-server-apache:config-name}
-config-type = push
-config-server-key = $${request-apache-backup:connection-ssh-public-key}
-config-on-notification = $${request-pull-backup-server:connection-feeds-url}$${request-pull-backup-server-apache:config-notification-id}
-slave = true
-
-[request-pull-backup-server-mariadb]
-<= request-pull-backup-server
-   slap-connection
-name = PBS pulling from $${request-mariadb:name}
-config = url name type server-key on-notification notify notification-id title
-config-url = $${request-mariadb:connection-ssh-url}
-config-name = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb
-config-type = pull
-config-server-key = $${request-mariadb:connection-ssh-public-key}
-config-on-notification = $${request-mariadb:connection-notification-id}
-config-notify = $${request-pull-backup-server:connection-notification-url}
-config-notification-id = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb-pull
-config-title = Pulling from MariaDB
-slave = true
-
-[request-pull-backup-server-mariadb-backup]
-<= request-pull-backup-server
-   slap-connection
-name = PBS pushing on $${request-mariadb-pseudo-replicating:name}
-config = url name type server-key on-notification notify notification-id title
-config-url = $${request-mariadb-pseudo-replicating:connection-ssh-url}
-config-name = $${request-pull-backup-server-mariadb:config-name}
-config-type = push
-config-server-key = $${request-mariadb-pseudo-replicating:connection-ssh-public-key}
-config-on-notification = $${request-pull-backup-server:connection-feeds-url}$${request-pull-backup-server-mariadb:config-notification-id}
-config-notify = $${request-mariadb-pseudo-replicating:connection-notification-url}
-config-notification-id = $${request-mariadb-pseudo-replicating:pbs-notification-id}
-config-title = Pushing to MariaDB backup
-slave = true
-
-[directory]
-ssh = $${rootdirectory:etc}/ssh/
-sshkeys = $${rootdirectory:srv}/sshkeys

stack/lamp/instance-mariadb-pbsready-export.cfg

-[buildout]
-extends = ${template-mariadb-pbsready:output}
-
-parts += cron-entry-mariadb-backup
-
-[urls]
-notification-id = http://[$${notifier:host}]:$${notifier:port}/get/$${notifier-mydumper:name}
-
-[mydumper]
-recipe = slapos.cookbook:mydumper
-wrapper = $${rootdirectory:bin}/raw_mydumper
-backup-directory = $${directory:mariadb-backup}
-socket = $${mariadb:socket}
-user = root
-mydumper-binary = ${mydumper:location}/bin/mydumper
-database = $${mariadb:database}
-import = false
-
-[notifier-mydumper]
-<= notifier
-recipe = slapos.cookbook:notifier.notify
-name = mydumper
-title = Dumping MariaDB Database
-executable = $${mydumper:wrapper}
-wrapper = $${rootdirectory:bin}/mydumper
-notify = $${slap-parameter:notify}
-
-[cron-entry-mariadb-backup]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = backup
-frequency = 0 * * * *
-command = $${notifier-mydumper:wrapper}

stack/lamp/instance-resilient.cfg

+{% import 'parts' as parts %}
+{% import 'replicated' as replicated %}
+
+[buildout]
+
+extends =
+   {{templateapache}}
+
+parts =
+  {{ parts.replicate("mariadb","3") }}
+  request-apache-backup-1
+  request-apache-backup-2
+
+  request-pull-backup-server-apache-1
+  request-pull-backup-server-apache-backup-1
+
+  request-pull-backup-server-apache-2
+  request-pull-backup-server-apache-backup-2
+
+  publish-connection-informations
+  apache-php
+  stunnel
+  certificate-authority
+  ca-stunnel
+  logrotate
+  logrotate-entry-apache
+  logrotate-entry-stunnel
+  cron
+  cron-entry-logrotate
+  dropbear-server
+  sshkeys-authority
+  dropbear-server-pbs-authorized-key
+
+  request-pull-backup-server
+
+{{ replicated.replicate("mariadb", "3", "mariadb-export", "mariadb-import") }}
+
+
+[request-pull-backup-server]
+<= request-pbs-common
+name = PBS (Pull Backup Server)
+return = ssh-key notification-url feeds-url
+slave = false
+
+[sshkeys-directory]
+recipe = slapos.cookbook:mkdirectory
+requests = ${directory:sshkeys}/requests/
+keys = ${directory:sshkeys}/keys/
+
+[sshkeys-authority]
+recipe = slapos.cookbook:sshkeys_authority
+request-directory = ${sshkeys-directory:requests}
+keys-directory = ${sshkeys-directory:keys}
+wrapper = ${basedirectory:services}/sshkeys_authority
+keygen-binary = {{dropbear}}/bin/dropbearkey
+
+[sshkeys-dropbear]
+<= sshkeys-authority
+recipe = slapos.cookbook:sshkeys_authority.request
+name = dropbear
+type = rsa
+executable = ${dropbear-server:wrapper}
+public-key = ${dropbear-server:rsa-keyfile}.pub
+private-key = ${dropbear-server:rsa-keyfile}
+wrapper = ${basedirectory:services}/sshd
+
+[dropbear-server]
+recipe = slapos.cookbook:dropbear
+host = ${slap-network-information:global-ipv6}
+port = 2222
+home = ${directory:ssh}
+wrapper = ${rootdirectory:bin}/raw_sshd
+shell = ${rdiff-backup-server:wrapper}
+rsa-keyfile = ${directory:ssh}/server_key.rsa
+dropbear-binary = {{dropbear}}/sbin/dropbear
+
+[dropbear-server-pbs-authorized-key]
+<= dropbear-server
+recipe = slapos.cookbook:dropbear.add_authorized_key
+key = ${request-pull-backup-server:connection-ssh-key}
+
+[rdiff-backup-server]
+<= apache-php
+recipe = slapos.cookbook:pbs
+client = false
+path = ${apache-php:htdocs}
+wrapper = ${rootdirectory:bin}/rdiffbackup-server
+rdiffbackup-binary = {{buildout}}/rdiff-backup
+
+[request-apache-backup-1]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Apache Backup 1
+software-url = ${slap-connection:software-release-url}
+software-type = apache-backup
+return = url ssh-url ssh-public-key
+config = authorized-key proxy-url
+config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
+config-proxy-url = ${publish-connection-informations:url}
+
+[request-apache-backup-2]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Apache Backup 2
+software-url = ${slap-connection:software-release-url}
+software-type = apache-backup
+return = url ssh-url ssh-public-key
+config = authorized-key proxy-url
+config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
+config-proxy-url = ${publish-connection-informations:url}
+
+[request-pull-backup-server-apache-1]
+<= request-pbs-common
+name = PBS pulling from Apache 1
+config = url name type server-key notify notification-id frequency
+config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
+config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
+config-type = pull
+config-server-key = ${sshkeys-dropbear:public-key-value}
+config-notify = ${request-pull-backup-server:connection-notification-url}
+config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
+config-frequency = 30 * * * *
+slave = true
+
+[request-pull-backup-server-apache-2]
+<= request-pbs-common
+name = PBS pulling from Apache 2
+config = url name type server-key notify notification-id frequency
+config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
+config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
+config-type = pull
+config-server-key = ${sshkeys-dropbear:public-key-value}
+config-notify = ${request-pull-backup-server:connection-notification-url}
+config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
+config-frequency = 30 * * * *
+slave = true
+
+
+[request-pull-backup-server-apache-backup-1]
+<= request-pbs-common
+name = PBS pushing to ${request-apache-backup-1:name}
+config = url name type server-key on-notification
+config-url = ${request-apache-backup-1:connection-ssh-url}
+config-name = ${request-pull-backup-server-apache-1:config-name}
+config-type = push
+config-server-key = ${request-apache-backup-1:connection-ssh-public-key}
+config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-1:config-notification-id}
+slave = true
+
+[request-pull-backup-server-apache-backup-2]
+<= request-pbs-common
+name = PBS pushing to ${request-apache-backup-2:name}
+config = url name type server-key on-notification
+config-url = ${request-apache-backup-2:connection-ssh-url}
+config-name = ${request-pull-backup-server-apache-2:config-name}
+config-type = push
+config-server-key = ${request-apache-backup-2:connection-ssh-public-key}
+config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-2:config-notification-id}
+slave = true
+
+
+[directory]
+ssh = ${rootdirectory:etc}/ssh/
+sshkeys = ${rootdirectory:srv}/sshkeys

stack/lamp/instance.cfg

 [buildout]
+
+extends = 
+  ${template-switchsoftware:output}
+
 parts =
   switch_softwaretype
 
@@ -9,9 +13,11 @@ offline = true
 [switch_softwaretype]
 recipe = slapos.cookbook:softwaretype
 default = ${template-apache-php:output}
-backuped = ${template-backuped:output}
+resilient = ${template-resilient-lamp:rendered}
 mariadb = ${template-mariadb:output}
-mariadb-pbsready-import = ${template-mariadb-pbsready-import:output}
-mariadb-pbsready-export = ${template-mariadb-pbsready-export:output}
+mariadb-import = ${template-mariadb-import:output}
+mariadb-export = ${template-mariadb-export:output}
 pull-backup = ${template-pull-backup:output}
 apache-backup = ${template-apache-backup:output}
+frozen = ${template-frozen:output}
+

stack/lamp/mariadb/instance-mariadb-export.cfg

+[buildout]
+extends = ${template-mariadb:output}
+          ${template-pbsready-export:output}
+
+parts += mariadb
+
+[exporter]
+recipe = slapos.cookbook:mydumper
+wrapper = $${rootdirectory:bin}/$${slap-parameter:namebase}-exporter
+backup-directory = $${directory:backup}
+socket = $${mariadb:socket}
+user = root
+mydumper-binary = ${mydumper:location}/bin/mydumper
+database = $${mariadb:database}
+import = false
\ No newline at end of file

stack/lamp/instance-mariadb-pbsready-import.cfg → stack/lamp/mariadb/instance-mariadb-import.cfg

 [buildout]
+extends = ${template-mariadb:output}
+          ${template-pbsready-import:output}
 
-extends = ${template-mariadb-pbsready:output}
+parts += mariadb
 
-parts += mariadb-import-on-notification
-
-[urls]
-notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
-
-[mydumper-import]
+[importer]
 recipe = slapos.cookbook:mydumper
 wrapper = $${rootdirectory:bin}/myloader
-backup-directory = $${directory:mariadb-backup}
+backup-directory = $${directory:backup}
 socket = $${mariadb:socket}
 user = root
 myloader-binary = ${mydumper:location}/bin/myloader
 database = $${mariadb:database}
 import = true
-
-[mariadb-import-on-notification]
-<= notifier
-recipe = slapos.cookbook:notifier.callback
-on-notification-id = $${slap-parameter:on-notification}
-callback = $${mydumper-import:wrapper}

stack/lamp/instance-mariadb.cfg → stack/lamp/mariadb/instance-mariadb.cfg

@@ -23,6 +23,7 @@ offline = true
 [urls]
 recipe = slapos.cookbook:publish
 url = mysqls://$${mariadb:user}:$${mariadb:password}@[$${stunnel:remote-host}]:$${stunnel:remote-port}/$${mariadb:database}
+ip = $${slap-network-information:global-ipv6}
 
 [mariadb]
 recipe = slapos.cookbook:mysql
@@ -52,6 +53,7 @@ mysql-install-binary = ${mariadb:location}/scripts/mysql_install_db
 mysql-upgrade-binary = ${mariadb:location}/bin/mysql_upgrade
 mysqld-binary = ${mariadb:location}/bin/mysqld
 
+
 [slapmonitor]
 recipe = slapos.cookbook:slapmonitor
 pid-file = $${basedirectory:run}/mariadb.pid
@@ -195,6 +197,7 @@ recipe = slapos.cookbook:mkdirectory
 log = $${rootdirectory:var}/log/
 services = $${rootdirectory:etc}/run/
 run = $${rootdirectory:var}/run/
+script = $${rootdirectory:etc}/script/
 backup = $${rootdirectory:srv}/backup/
 promises = $${rootdirectory:etc}/promise/
 

stack/lapp/README.txt

+
+LAPP stack
+==========
+
+This fork of the LAMP stack provides:
+
+ - a Postgres instance, with an empty database and a 'postgres' superuser.
+   Log rotation is handled by Postgres itself.
+
+ - symlinks to all the postgres binaries, usable through unix socket
+   with no further authentication, or through ipv6
+
+ - a psycopg2 (postgres driver) egg
+
+ - configuration for a maarch instance (this part should be brought outside the stack)
+

stack/lapp/apache/instance-apache-php.cfg.in

+[buildout]
+
+parts =
+  certificate-authority
+  ca-stunnel
+  logrotate
+  logrotate-entry-apache
+  logrotate-entry-stunnel
+  cron
+  cron-entry-logrotate
+  promise
+  slapmonitor
+  slapreport
+  frontend-promise
+  content-promise
+  publish-connection-informations
+  maarch-instance
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+
+# Creation of all needed directories
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc/
+var = $${buildout:directory}/var/
+srv = $${buildout:directory}/srv/
+bin = $${buildout:directory}/bin/
+tmp = $${buildout:directory}/tmp/
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log/
+services = $${rootdirectory:etc}/run/
+run = $${rootdirectory:var}/run/
+backup = $${rootdirectory:srv}/backup/
+promises = $${rootdirectory:etc}/promise/
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+ca-dir = $${rootdirectory:srv}/ssl/
+httpd-log = $${basedirectory:log}/apache/
+php-ini-dir = $${rootdirectory:etc}/php/
+tmp-php = $${rootdirectory:tmp}/php/
+logrotate-entries = $${rootdirectory:etc}/logrotate.d/
+logrotate-backup = $${basedirectory:backup}/logrotate/
+stunnel-conf = $${rootdirectory:etc}/stunnel/
+cronstamps = $${rootdirectory:etc}/cronstamps/
+cron-entries = $${rootdirectory:etc}/cron.d/
+crontabs = $${rootdirectory:etc}/crontabs/
+
+[cadirectory]
+recipe = slapos.cookbook:mkdirectory
+requests = $${directory:ca-dir}/requests/
+private = $${directory:ca-dir}/private/
+certs = $${directory:ca-dir}/certs/
+newcerts = $${directory:ca-dir}/newcerts/
+crl = $${directory:ca-dir}/crl/
+
+
+# Deploy stunnel
+[stunnel]
+recipe = slapos.cookbook:stunnel
+client = true
+stunnel-binary = ${stunnel:location}/bin/stunnel
+remote-host = $${postgres-urlparse:host}
+remote-port = $${postgres-urlparse:port}
+local-host = $${slap-network-information:local-ipv4}
+local-port = 33060
+log-file = $${basedirectory:log}/stunnel.log
+config-file = $${directory:stunnel-conf}/stunnel.conf
+key-file = $${directory:stunnel-conf}/stunnel.key
+cert-file = $${directory:stunnel-conf}/stunnel.crt
+pid-file = $${basedirectory:run}/stunnel.pid
+wrapper = $${rootdirectory:bin}/raw_stunnel
+post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate
+
+
+# Certificate stuffs
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = ${openssl:location}/bin/openssl
+ca-dir = $${directory:ca-dir}
+requests-directory = $${cadirectory:requests}
+wrapper = $${basedirectory:services}/ca
+ca-private = $${cadirectory:private}
+ca-certs = $${cadirectory:certs}
+ca-newcerts = $${cadirectory:newcerts}
+ca-crl = $${cadirectory:crl}
+
+[ca-stunnel]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+executable = $${stunnel:wrapper}
+wrapper = $${basedirectory:services}/stunnel
+key-file = $${stunnel:key-file}
+cert-file = $${stunnel:cert-file}
+
+
+# Request Postgres instance and parse its URL
+[request-postgres]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Postgres
+software-url = $${slap-connection:software-release-url}
+software-type = postgres
+return = url
+sla = computer_guid
+sla-computer_guid = $${slap-connection:computer-id}
+
+[postgres-urlparse]
+recipe = slapos.cookbook:urlparse
+url = $${request-postgres:connection-url}
+
+
+# Deploy Apache + PHP application
+[apache-php]
+recipe = slapos.cookbook:apachephp
+source = ${application:location}
+template = ${application-template:location}/${application-template:filename}
+configuration = ${application-configuration:location}
+
+htdocs = $${rootdirectory:srv}/www/
+pid-file = $${basedirectory:run}/apache.pid
+lock-file = $${basedirectory:run}/apache.lock
+ip = $${slap-network-information:global-ipv6}
+port = 8080
+url = http://[$${:ip}]:$${:port}/
+error-log = $${directory:httpd-log}/error.log
+access-log = $${directory:httpd-log}/access.log
+php-ini-dir = $${directory:php-ini-dir}
+tmp-dir = $${directory:tmp-php}
+httpd-conf = $${rootdirectory:etc}/apache.conf
+wrapper = $${basedirectory:services}/apache
+
+httpd-binary = ${apache:location}/bin/httpd
+
+mysql-username = $${postgres-urlparse:username}
+mysql-password = $${postgres-urlparse:password}
+mysql-database = $${postgres-urlparse:path}
+mysql-host = $${stunnel:local-host}
+mysql-port = $${stunnel:local-port}
+
+
+# Deploy logrotate, cron, configure it
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = ${gzip:location}/bin/gzip
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${rootdirectory:bin}/logrotate
+conf = $${rootdirectory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${rootdirectory:srv}/logrotate.status
+
+[logrotate-entry-apache]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = apache
+log = $${apache-php:error-log} $${apache-php:access-log}
+frequency = daily
+rotate-num = 30
+sharedscripts = true
+notifempty = true
+create = true
+
+[logrotate-entry-stunnel]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = stunnel
+log = $${stunnel:log-file}
+frequency = daily
+rotate-num = 30
+notifempty = true
+create = true
+post = $${stunnel:post-rotate-script}
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
+
+
+# Request frontend
+[request-frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Frontend
+# XXX We have hardcoded SR URL here.
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config = url custom_domain
+config-url = http://[$${apache-php:ip}]:$${apache-php:port}/
+return = site_url
+config-custom_domain = $${slap-parameter:domain}
+
+
+# Deploy slapmonitor
+[slapmonitor]
+recipe = slapos.cookbook:slapmonitor
+pid-file = $${basedirectory:run}/apache.pid
+database-path = $${basedirectory:log}/slapmonitor.db
+shell-path = ${dash:location}/bin/dash
+slapmonitor-path = ${buildout:bin-directory}/slapmonitor
+path = $${basedirectory:services}/slapmonitor
+
+[slapreport]
+recipe = slapos.cookbook:slapreport
+pid-file = $${basedirectory:run}/apache.pid
+database-path = $${basedirectory:log}/slapmonitor.db
+consumption-log-path = $${basedirectory:log}/instance_consumption.log
+logbox-ip = 87.98.152.12
+logbox-port = 5122
+logbox-user = admin
+logbox-passwd = passer
+shell-path = ${dash:location}/bin/dash
+slapreport-path = ${buildout:bin-directory}/slapreport
+path = $${basedirectory:services}/slapreport
+
+
+# Publish all instance parameters (url of instance)
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+backend_url = $${apache-php:url}
+url = $${request-frontend:connection-site_url}
+
+
+# Deploy promises scripts
+[promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${basedirectory:promises}/apache
+hostname = $${apache-php:ip}
+port = $${apache-php:port}
+
+[frontend-promise]
+recipe = slapos.cookbook:check_url_available
+path = $${basedirectory:promises}/frontend
+url = $${request-frontend:connection-site_url}
+dash_path = ${dash:location}/bin/dash
+curl_path = ${curl:location}/bin/curl
+
+[content-promise]
+recipe = slapos.cookbook:check_page_content
+path = $${basedirectory:promises}/content
+url = $${request-frontend:connection-site_url}
+dash_path = ${dash:location}/bin/dash
+curl_path = ${curl:location}/bin/curl
+
+[slap-parameter]
+# Default value if no domain is specified
+domain =
+# Default value if no ssh parameter is specified
+logbox-ip =
+logbox-port =
+logbox-user =
+logbox-passwd =
+
+[maarch-instance]
+recipe = slapos.cookbook:maarch.configuration
+htdocs = $${apache-php:htdocs}
+db_host = $${postgres-urlparse:host}
+db_port = $${postgres-urlparse:port}
+db_dbname = $${postgres-urlparse:path}
+db_username = $${postgres-urlparse:username}
+db_password = $${postgres-urlparse:password}
+language = en
+php_ini_dir = $${directory:php-ini-dir}
+root_docservers = $${buildout:directory}/srv/docservers
+

stack/lapp/buildout.cfg

+[buildout]
+parts =
+  apache-php-postgres
+  eggs
+  template
+  psycopg2
+  template-apache-php
+  template-postgres
+
+
+extends =
+  ../../component/apache/buildout.cfg
+  ../../component/apache-php/buildout.cfg
+  ../../component/dash/buildout.cfg
+  ../../component/dcron/buildout.cfg
+  ../../component/gzip/buildout.cfg
+  ../../component/logrotate/buildout.cfg
+  ../../component/lxml-python/buildout.cfg
+  ../../component/postgresql/buildout.cfg
+  ../../component/stunnel/buildout.cfg
+  ../slapos.cfg
+
+
+[application]
+recipe = hexagonit.recipe.download
+strip-top-level-dir = true
+
+
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+  ${psycopg2:egg}
+  slapos.toolbox
+  cns.recipe.symlink
+
+
+
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg
+output = ${buildout:directory}/template.cfg
+#md5sum =
+mode = 0644
+
+
+[template-apache-php]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/apache/instance-apache-php.cfg.in
+output = ${buildout:directory}/template-apache-php.cfg
+#md5sum = 
+mode = 0644
+
+[template-postgres]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/postgres/instance-postgres.cfg.in
+output = ${buildout:directory}/template-postgres.cfg
+#md5sum = 
+mode = 0644
+
+
+# Dummy parts in case no application configuration file is needed
+[application-template]
+filename =
+location =
+
+[application-configuration]
+location =
+
+[psycopg2-env]
+PATH = ${postgresql:location}/bin:%(PATH)s
+
+
+[psycopg2]
+recipe = zc.recipe.egg:custom
+egg = psycopg2
+define = PSYCOPG_EXTENSIONS,PSYCOPG_DISPLAY_SIZE,PSYCOPG_NEW_BOOLEAN,HAVE_PQFREEMEM,HAVE_PQPROTOCOL3
+environment = psycopg2-env
+include-dirs =
+    ${postgresql:location}/include
+library-dirs =
+    ${postgresql:location}/lib
+rpath = 
+    ${postgresql:location}/lib
+
+

stack/lapp/instance.cfg

+[buildout]
+parts =
+  switch_softwaretype
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[switch_softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = ${template-apache-php:output}
+postgres = ${template-postgres:output}

stack/lapp/postgres/instance-postgres.cfg.in

+[buildout]
+parts =
+  symlinks
+  publish
+  postgres-instance
+  postgres-promise
+
+
+# Define egg directories to be the one from Software Release
+# (/opt/slapgrid/...)
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+
+[instance-parameters]
+# Fetches parameters defined in SlapOS Master for this instance
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+etc = $${buildout:directory}/etc
+services = $${rootdirectory:etc}/run/
+promises = $${rootdirectory:etc}/promise/
+var = $${buildout:directory}/var
+
+
+[symlinks]
+recipe = cns.recipe.symlink
+symlink_target = $${rootdirectory:bin}
+symlink_base = ${postgresql:location}/bin
+
+
+[postgres-instance]
+# create cluster, configuration files and a database
+recipe = slapos.cookbook:postgres
+
+# Options
+ipv6_host = $${slap-network-information:global-ipv6}
+user = postgres
+port = 5432
+dbname = db
+# pgdata_directory is created by initdb, and should not exist beforehand.
+pgdata-directory = $${rootdirectory:var}/data
+services = $${rootdirectory:services}
+bin = $${rootdirectory:bin}
+
+
+# Deploy promises scripts
+[postgres-promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${rootdirectory:promises}/postgres
+hostname = $${slap-network-information:global-ipv6}
+port = $${postgres-instance:port}
+
+
+[publish]
+recipe = slapos.cookbook:publishurl
+url = $${postgres-instance:url}
+

stack/resilient/buildout.cfg

+[buildout]
+
+parts =
+#Templates needed to setup automatic backup
+  template-pbsready
+  template-pbsready-import
+  template-pbsready-export
+  template-replicated
+  template-parts
+
+#Frozen is the state used to not destroy a broken instance's content
+  template-frozen
+
+  template-resilient
+  template-switchsoftware
+
+
+[template-pbsready]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-pbsready.cfg
+output = ${buildout:directory}/template-pbsready.cfg
+md5sum = 45e64cfb6afbcfda1f9f85e33c73bd99
+mode = 0644
+
+[template-pbsready-import]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-pbsready-import.cfg
+output = ${buildout:directory}/template-pbsready-import.cfg
+md5sum = 5ba7477f9499a7dbde5f33ca96bd6ba4
+mode = 0644
+
+[template-pbsready-export]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-pbsready-export.cfg
+output = ${buildout:directory}/template-pbsready-export.cfg
+md5sum = 29d36aac2008b173cb9ce5da9e88c0fa
+mode = 0644
+
+[template-pull-backup]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-pull-backup.cfg
+output = ${buildout:directory}/template-pull-backup.cfg
+md5sum = f88cc9192a63c88f83a9e5191075534e
+mode = 0644
+
+[template-replicated]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/template-replicated.cfg
+md5sum = 61780842ccda1600a102456b0da69ac9
+mode = 0644
+destination = ${buildout:directory}/template-replicated.cfg
+
+[template-parts]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/template-parts.cfg
+md5sum = f5fc27235725f05fdbde76a78ebc363e
+mode = 0644
+destination = ${buildout:directory}/template-parts.cfg
+
+[template-frozen]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-frozen.cfg
+output = ${buildout:directory}/template-frozen.cfg
+
+[template-resilient]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/resilient.cfg
+output = ${buildout:directory}/resilient.cfg
+md5sum = 59e74d290d623de2c1e147e48f284fba
+mode = 0644
+
+[template-switchsoftware]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/switchsoftware.cfg
+output = ${buildout:directory}/switchsoftware.cfg
+md5sum = c94a0ed85fce2e72254ae956dce7e40d
+mode = 0644
\ No newline at end of file

stack/resilient/instance-frozen.cfg

+[buildout]
+  parts = 
\ No newline at end of file

stack/resilient/instance-pbsready-export.cfg

+[buildout]
+extends = ${template-pbsready:output}
+
+parts += cron-entry-backup
+
+[urls]
+notification-id = http://[$${notifier:host}]:$${notifier:port}/get/$${notifier-exporter:name}
+
+#notify launches executable, and once it's done, notifies the pull-backup-servers.
+[notifier-exporter]
+<= notifier
+recipe = slapos.cookbook:notifier.notify
+name = exporter
+title = Dumping $${slap-parameter:namebase}
+executable = $${exporter:wrapper}
+wrapper = $${rootdirectory:bin}/exporter
+notify = $${slap-parameter:notify}
+
+#adds the exporter to cron
+[cron-entry-backup]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = backup
+frequency = 0 * * * *
+command = $${notifier-exporter:wrapper}

stack/resilient/instance-pbsready-import.cfg

+[buildout]
+
+extends = ${template-pbsready:output}
+
+parts += import-on-notification
+
+[urls]
+notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
+
+#Launches callback, when a notification is received
+[import-on-notification]
+<= notifier
+recipe = slapos.cookbook:notifier.callback
+on-notification-id = $${slap-parameter:on-notification}
+callback = $${importer:wrapper}

stack/lamp/instance-mariadb-pbsready.cfg → stack/resilient/instance-pbsready.cfg

 [buildout]
 
-extends = ${template-mariadb:output}
-
 parts =
+  resiliency
   urls
-  mariadb
   stunnel
   certificate-authority
   ca-stunnel
   logrotate
-  logrotate-entry-mariadb
   logrotate-entry-stunnel
   logrotate-entry-cron
   logrotate-entry-equeue
@@ -21,6 +18,13 @@ parts =
   dropbear-server-pbs-authorized-key
   notifier
 
+# adds the resiliency script for the bully algorithm
+[resiliency]
+recipe = slapos.cookbook:addresiliency
+script = $${basedirectory:script}
+run = $${basedirectory:services}
+
+# sets up an rdiff-backup server (with a dropbear server for ssh)
 [urls]
 ssh-public-key = $${sshkeys-dropbear:public-key-value}
 ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
@@ -65,10 +69,12 @@ key = $${slap-parameter:authorized-key}
 [rdiff-backup-server]
 recipe = slapos.cookbook:pbs
 client = false
-path = $${directory:mariadb-backup}
+path = $${directory:backup}
 wrapper = $${rootdirectory:bin}/rdiffbackup-server
 rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
 
+
+## Sets up the execution queue for the notifier
 [logrotate-entry-equeue]
 <= logrotate
 recipe = slapos.cookbook:logrotate.d
@@ -85,6 +91,9 @@ database = $${rootdirectory:srv}/equeue.db
 wrapper = $${basedirectory:services}/equeue
 equeue-binary = ${buildout:bin-directory}/equeue
 
+
+## notifier.notify adds the [exporter, notifier] to the execution queue
+## notifier.notify.callback sets up a callback
 [notifier]
 recipe = slapos.cookbook:notifier
 feeds = $${directory:notifier-feeds}
@@ -99,12 +108,15 @@ notifier-binary = ${buildout:bin-directory}/pubsubnotifier
 
 
 [basedirectory]
+script = $${rootdirectory:etc}/script/
+services = $${rootdirectory:etc}/run/
 cache = $${rootdirectory:var}/cache/
 notifier = $${rootdirectory:etc}/notifier/
 
 [directory]
-mariadb-backup = $${basedirectory:backup}/mariadb/
+backup = $${basedirectory:backup}/$${slap-parameter:namebase}
 ssh = $${rootdirectory:etc}/ssh/
 sshkeys = $${rootdirectory:srv}/sshkeys
 notifier-feeds = $${basedirectory:notifier}/feeds/
 notifier-callbacks = $${basedirectory:notifier}/callbacks/
+script = $${basedirectory:script}

stack/lamp/instance-pull-backup.cfg → stack/resilient/instance-pull-backup.cfg

@@ -19,6 +19,8 @@ ssh-key = $${sshkeys-dropbear:public-key-value}
 notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
 feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/
 
+
+## sets up the equeue for the notifier
 [equeue]
 recipe = slapos.cookbook:equeue
 socket = $${basedirectory:run}/equeue.sock
@@ -27,6 +29,7 @@ database = $${rootdirectory:srv}/equeue.db
 wrapper = $${basedirectory:services}/equeue
 equeue-binary = ${buildout:bin-directory}/equeue
 
+
 [notifier]
 recipe = slapos.cookbook:notifier
 feeds = $${directory:notifier-feeds}
@@ -39,6 +42,8 @@ wrapper = $${basedirectory:services}/notifier
 server-binary = ${buildout:bin-directory}/pubsubserver
 notifier-binary = ${buildout:bin-directory}/pubsubnotifier
 
+
+## Dropbear Client to provide ssh 
 [dropbear-client]
 recipe = slapos.cookbook:dropbear.client
 dbclient-binary = ${dropbear:location}/bin/dbclient
@@ -68,6 +73,9 @@ public-key = $${dropbear-client:identity-file}.pub
 private-key = $${dropbear-client:identity-file}
 wrapper = $${rootdirectory:bin}/do_backup
 
+
+## The pull-backup-server contains every backup (incremental).
+## to prevent a corrupt dump from destroying everything.
 [pbs]
 <= notifier
 recipe = slapos.cookbook:pbs
@@ -82,6 +90,7 @@ wrappers-directory = $${directory:pbs-wrappers}
 notifier-url = http://[$${notifier:host}]:$${notifier:port}/
 slave-instance-list = $${slap-parameter:slave_instance_list}
 
+
 [cron-simplelogger]
 recipe = slapos.cookbook:simplelogger
 wrapper = $${rootdirectory:bin}/cron_simplelogger
@@ -170,3 +179,8 @@ notifier = $${rootdirectory:etc}/notifier/
 
 [rootdirectory]
 home = $${buildout:directory}/home/
+
+
+# Default values
+[slap-parameter]
+slave_instance_list = []

stack/resilient/resilient.cfg

+## not used at the moment
+
+[buildout]
+
+parts =
+  request-pull-backup-server
+
+[request-pull-backup-server]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = PBS (Pull Backup Server)
+software-url = $${slap-connection:software-release-url}
+software-type = pull-backup
+return = ssh-key notification-url feeds-url
+slave = false
\ No newline at end of file

stack/resilient/switchsoftware.cfg

+[buildout]
+parts =
+  switch_softwaretype
+
+[switch_softwaretype]
+recipe = slapos.cookbook:softwaretype
+pull-backup = ${template-pull-backup:output}
+#frozen creates a syntax error, meaning it can keep its data.
+#It's dirty as hell, it needs to be replaced.
+frozen = ${template-frozen:output}

stack/resilient/template-parts.cfg

+## Parts Needed for the resiliency
+
+  {% macro replicate(namebase, nbbackup) %}
+
+  request-{{namebase}}
+  request-{{namebase}}-2
+  
+  {% for i in range(1,nbbackup|int) %}
+  request-{{namebase}}-pseudo-replicating-{{i}}
+  request-{{namebase}}-pseudo-replicating-{{i}}-2
+  {% endfor %}
+  
+  {% for i in range(1,nbbackup|int) %}
+  request-pbs-{{namebase}}-{{i}}
+  request-pull-backup-server-{{namebase}}-{{i}}
+  request-pull-backup-server-{{namebase}}-backup-{{i}}
+  {% endfor %}
+  
+
+
+  {% endmacro %}
\ No newline at end of file

stack/resilient/template-replicated.cfg

+{% macro replicate(namebase, nbbackup, typeexport, typeimport, heriteLeader='', heriteBackup='') %}
+
+## Tells the Backupable recipe that we want a backup
+[resilient]
+config-script = bully.py
+config-wrapper = bully
+config-namebase = {{namebase}}
+
+## Every request is double to provide the 3 IPs.
+[request-{{namebase}}]
+<= resilient
+   slap-connection
+   {{heriteLeader}}
+software-type = {{typeexport}}
+name = {{namebase}}0
+return = url ssh-public-key ssh-url notification-id ip
+
+config = number script wrapper authorized-key notify ip-list namebase
+
+config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
+
+config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
+
+config-ip-list =
+config-number = 0
+
+{% for id in range(1,nbbackup|int) %}
+
+[request-{{namebase}}-pseudo-replicating-{{id}}]
+<= slap-connection
+   resilient
+   {{heriteBackup}}
+recipe = slapos.cookbook:request
+name = {{namebase}}{{id}}
+
+software-url = ${slap-connection:software-release-url}
+software-type = {{typeimport}}
+return = url ssh-public-key ssh-url notification-url ip
+
+config = number script wrapper authorized-key on-notification ip-list namebase
+
+config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
+config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
+pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
+
+config-ip-list =
+config-number = {{id}}
+
+{% endfor %}
+
+[iplist]
+config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbbackup|int) %} ${request-{{namebase}}-pseudo-replicating-{{j}}:connection-ip}{% endfor %}
+
+[request-{{namebase}}-2]
+<= resilient
+   slap-connection
+   iplist
+   {{heriteLeader}}
+
+recipe = slapos.cookbook:request
+name = {{namebase}}0
+
+config = number script wrapper authorized-key notify ip-list namebase
+
+software-url = ${slap-connection:software-release-url}
+software-type = {{typeexport}}
+return = url ssh-public-key ssh-url notification-id ip
+
+
+config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
+config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
+
+
+config-number=0
+
+{% for id in range(1,nbbackup|int) %}
+[request-{{namebase}}-pseudo-replicating-{{id}}-2]
+<= slap-connection
+   resilient
+   iplist
+   {{heriteBackup}}
+
+recipe = slapos.cookbook:request
+name = {{namebase}}{{id}}
+
+software-url = ${slap-connection:software-release-url}
+software-type = {{typeimport}}
+return = url ssh-public-key ssh-url notification-url
+
+config = number script wrapper authorized-key on-notification ip-list namebase
+
+config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
+config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
+pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
+
+
+config-number = {{id}}
+{% endfor %}
+
+
+
+## The PBS and their push / pull slaves
+## Adding a PBS provides resiliency
+## Adding a backup server provides availability
+
+## Having 3 backups pulling from the same PBS provides
+##only availability, not resiliency
+
+## WARNING : SLAVES ARE ALLOCATED AT RANDOM, THIS NEEDS TO BE FIXED.
+
+[request-pbs-common]
+<= slap-connection
+recipe = slapos.cookbook:request
+software-url = ${slap-connection:software-release-url}
+software-type = pull-backup
+
+{% for id in range(1,nbbackup|int) %}
+
+[request-pbs-{{namebase}}-{{id}}]
+<= request-pbs-common
+name = PBS ({{namebase}} / {{id}})
+return = ssh-key notification-url feeds-url
+slave = false
+
+[request-pull-backup-server-{{namebase}}-{{id}}]
+<= request-pbs-common
+name = PBS {{id}} pulling from ${request-{{namebase}}:name}
+config = url name type server-key on-notification notify notification-id title
+config-url = ${request-{{namebase}}:connection-ssh-url}
+config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}
+config-type = pull
+config-server-key = ${request-{{namebase}}:connection-ssh-public-key}
+config-on-notification = ${request-{{namebase}}:connection-notification-id}
+config-notify = ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}
+config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-pull
+config-title = Pulling from {{namebase}}
+slave = true
+
+[request-pull-backup-server-{{namebase}}-backup-{{id}}]
+<= request-pbs-common
+name = PBS pushing on ${request-{{namebase}}-pseudo-replicating-{{id}}:name}
+config = url name type server-key on-notification notify notification-id title
+config-url = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-url}
+config-name = ${request-pull-backup-server-{{namebase}}-{{id}}:config-name}
+config-type = push
+config-server-key = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-public-key}
+config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${request-pull-backup-server-{{namebase}}-{{id}}:config-notification-id}
+config-notify = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-notification-url}
+config-notification-id = ${request-{{namebase}}-pseudo-replicating-{{id}}:pbs-notification-id}
+config-title = Pushing to {{namebase}} backup {{id}}
+slave = true
+{% endfor %}
+
+{% endmacro %}
\ No newline at end of file

stack/slapos.cfg

@@ -24,6 +24,7 @@ find-links +=
 
 # Use only quite well working sites.
 allow-hosts +=
+  *.googlecode.com
   *.nexedi.org
   *.python.org
   *.sourceforge.net
@@ -34,7 +35,6 @@ allow-hosts +=
   github.com
   launchpad.net
   peak.telecommunity.com
-  psutil.googlecode.com
   sourceforge.net
   www.dabeaz.com
   www.owlfish.com
@@ -59,7 +59,7 @@ eggs =
 
 [versions]
 # Use SlapOS patched zc.buildout
-zc.buildout = 1.6.0-dev-SlapOS-007
+zc.buildout = 1.6.0-dev-SlapOS-010
 
 [networkcache]
 download-cache-url = http://www.shacache.org/shacache