Merge branch 'master' into kvm

CHANGES.txt

-0.14 (unreleased)
-=================
+Changes
+=======
+
+0.17 (unreleased)
+-----------------
 
  * No changes yet.
 
+0.16 (2011-07-15)
+-----------------
+
+ * Improve Vifib and pure ERP5 instantiation [Rafael Monnerat]
+ * Use configurator for Vifib [Rafael Monnerat]
+
+0.15 (2011-07-13)
+-----------------
+
+ * Encrypt connection by default. [Vivien Alger]
+
+0.14 (2011-07-13)
+-----------------
+
+ * Provide new way to instantiate kvm. [Cedric de Saint Martin, Vivien Alger]
+
 0.13 (2011-07-13)
-=================
+-----------------
 
  * Implement generic execute_wait wrapper, which allows to wait for some files
    to appear before starting service depending on it. [Łukasz Nowak]
 
 0.12 (2011-07-11)
-=================
+-----------------
 
  * Fix slaprunner, phpmyadmin software releases, added
    wordpress software release. [Cedric de Saint Martin]
 
 0.11 (2011-07-07)
-=================
+-----------------
 
  * Enable test suite runner for vifib.
 
 0.10 (2011-07-01)
-=================
+-----------------
 
  * Add PHPMyAdmin software release used in SlapOS tutorials
- [Cedric de Saint Martin]
+   [Cedric de Saint Martin]
  * Add slaprunner software release [Cedric de Saint Martin]
 
 0.9 (2011-06-24)
-================
+----------------
 
  * mysql recipe : Changing slapos.recipe.erp5.execute to
    slapos.recipe.librecipe.execute [Cedric de Saint Martin]
 
 0.8 (2011-06-15)
-================
+----------------
 
  * Add MySQL and MariaDB standalone software release and recipe
    [Cedric de Saint Martin]
  * Fixed slapos.recipe.erp5testnode instantiation [Sebastien Robin]
 
 0.7 (2011-06-14)
-================
+----------------
 
  * Fix slapos.recipe.erp5 package by providing site.zcml in it. [Łukasz Nowak]
  * Improve slapos.recipe.erp5testnode partition instantiation error reporting
    [Sebastien Robin]
 
 0.6 (2011-06-13)
-================
+----------------
 
  * Fixed slapos.recipe.erp5 instantiation. [Łukasz Nowak]
 
 0.5 (2011-06-13)
-================
+----------------
 
  * Implement zabbix agent instantiation. [Łukasz Nowak]
  * Drop dependency on Zope2. [Łukasz Nowak]
  * Share more in slapos.recipe.librecipe module. [Łukasz Nowak]
 
 0.4 (2011-06-09)
-================
+----------------
 
  * Remove reference to slapos.tool.networkcache as it was removed from pypi. [Łukasz Nowak]
  * Add Kumofs standalone software release and recipe [Cedric de Saint Martin]
  * Add Memcached standalone software release and recipe [Cedric de Saint Martin]
 
 0.3 (2011-06-09)
-================
+----------------
 
  * Moved out template and build to separate distributions [Łukasz Nowak]
  * Depend on slapos.core instead of depracated slapos.slap [Romain Courteaud]
@@ -75,11 +94,11 @@
  * Allow to control full environment in erp5 module [Łukasz Nowak]
 
 0.2 (2011-05-30)
-================
+----------------
 
   * Allow to pass zope_environment in erp5 entry point [Łukasz Nowak]
 
 0.1 (2011-05-27)
-================
+----------------
 
   * All slapos.recipe.* became slapos.cookbook:* [Łukasz Nowak]

README.txt

 slapos.cookbook
 ===============
+
+Cookbook of SlapOS recipes.

component/curl/buildout.cfg

@@ -11,14 +11,13 @@ parts =
 
 [curl]
 recipe = hexagonit.recipe.cmmi
-url = http://curl.haxx.se/download/curl-7.21.3.tar.bz2
-md5sum = 5b57fee22090b5c43a6886fdd35af2ce
+url = http://curl.haxx.se/download/curl-7.21.7.tar.bz2
+md5sum = 5f6d50c4d4ee38c57fe37e3cff75adbd
 configure-options =
   --disable-static
   --disable-ldap
   --disable-ldaps
   --disable-rtsp
-  --disable-proxy
   --disable-dict
   --disable-telnet
   --disable-tftp

component/glib/buildout.cfg

@@ -5,8 +5,8 @@ extends =
 
 [glib]
 recipe = hexagonit.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/sources/glib/2.28/glib-2.28.7.tar.bz2
-md5sum = feda1650c8646ad39c7b01d95b03766b
+url = http://ftp.gnome.org/pub/gnome/sources/glib/2.28/glib-2.28.8.tar.bz2
+md5sum = 789e7520f71c6a4bf08bc683ec764d24
 configure-options =
   --disable-static
   --disable-selinux

component/gzip/buildout.cfg

+[buildout]
+parts =
+  gzip
+
+[gzip]
+recipe = hexagonit.recipe.cmmi
+url = ftp://ftp.gnu.org/pub/gnu/gzip/gzip-1.4.tar.gz
+md5sum = e381b8506210c794278f5527cba0e765

component/libpng/buildout.cfg

@@ -3,7 +3,6 @@ extends =
   ../zlib/buildout.cfg
 
 parts =
-  libpng12
   libpng
 
 [libpng-common]
@@ -14,12 +13,7 @@ environment =
   CPPFLAGS =-I${zlib:location}/include
   LDFLAGS =-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
 
-[libpng12]
-<= libpng-common
-url = http://download.sourceforge.net/libpng/libpng-1.2.44.tar.bz2
-md5sum = e3ac7879d62ad166a6f0c7441390d12b
-
 [libpng]
 <= libpng-common
-url = http://download.sourceforge.net/libpng/libpng-1.5.2.tar.bz2
-md5sum = a003b37ed9afb0d9164eb7228421057c
+url = http://download.sourceforge.net/libpng/libpng-1.5.4.tar.bz2
+md5sum = b43afe39237b69859522455b215f9e85

component/libxslt/buildout.cfg

@@ -16,6 +16,8 @@ configure-options =
   --with-libxml-prefix=${libxml2:location}
   --without-crypto
   --without-python
+  --without-debug
+  --without-debugger
 environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
   CPPFLAGS=-I${zlib:location}/include

component/maatkit/buildout.cfg

+[buildout]
+extends =
+  ../perl/buildout.cfg
+  ../perl-DBI/buildout.cfg
+  ../perl-DBD-MySQL/buildout.cfg
+parts =
+  maatkit
+
+[maatkit]
+recipe = hexagonit.recipe.cmmi
+depends =
+  ${perl:version}
+  ${perl-DBI:version}
+  ${perl-DBD-MySQL:version}
+url = http://maatkit.googlecode.com/files/maatkit-7540.tar.gz
+md5sum = 55457f98500b096a6bf549356d3445fe
+configure-command =
+  ${perl:location}/bin/perl Makefile.PL

component/mariadb/buildout.cfg

@@ -12,9 +12,9 @@ parts =
 
 [mariadb]
 recipe = hexagonit.recipe.cmmi
-version = 5.2.6
+version = 5.2.7
 url = http://www.percona.com/downloads/MariaDB/mariadb-${:version}/kvm-tarbake-jaunty-x86/mariadb-${:version}.tar.gz
-md5sum = e562aca71ae16b490196f99aa7e64b55
+md5sum = 06b9b102946a3606b38348c0ebf18367
 # compile directory is required to build mysql plugins.
 keep-compile-dir = true
 # configure: how to avoid searching for my.cnf?

component/perl-DBD-MySQL/DBD-mysql-4.019.rpathsupport.patch

+--- DBD-mysql-4.019.back/Makefile.PL    2011-05-09 03:12:07.000000000 +0200
++++ DBD-mysql-4.019/Makefile.PL 2011-06-22 11:44:06.478371893 +0200
+@@ -358,7 +358,14 @@  
+                      'Data::Dumper' => 0 };
+ }
+
+-ExtUtils::MakeMaker::WriteMakefile(%o);
++
++my %config;
++if (defined($ENV{'OTHERLDFLAGS'})) {
++    $config{dynamic_lib} = { OTHERLDFLAGS => " $ENV{'OTHERLDFLAGS'} " };
++}
++
++
++ExtUtils::MakeMaker::WriteMakefile(%o, %config);
+ exit 0;
+
+
+

component/perl-DBD-MySQL/buildout.cfg

+[buildout]
+extends =
+  ../perl/buildout.cfg
+  ../perl-DBI/buildout.cfg
+  ../mysql-tritonn-5.0/buildout.cfg
+  ../zlib/buildout.cfg
+  ../openssl/buildout.cfg
+parts =
+  perl-DBD-MySQL
+
+[perl-DBD-MySQL-patch]
+recipe = hexagonit.recipe.download
+md5sum = e12e9233f20b0370cfcf5228ea767fbc
+url = ${:_profile_base_location_}/${:filename}
+filename = DBD-mysql-4.019.rpathsupport.patch
+download-only = true
+
+[perl-DBD-MySQL]
+recipe = hexagonit.recipe.cmmi
+version = 4.019
+depends =
+  ${perl:version}
+  ${perl-DBI:version}
+url = http://search.cpan.org/CPAN/authors/id/C/CA/CAPTTOFU/DBD-mysql-4.019.tar.gz
+md5sum = 566d98ab8ffac9626a31f6f6d455558e
+patches =
+  ${perl-DBD-MySQL-patch:location}/${perl-DBD-MySQL-patch:filename}
+patch-options = -p1
+configure-command =
+  ${perl:location}/bin/perl Makefile.PL --mysql_config=${mysql-tritonn-5.0:location}/bin/mysql_config
+environment =
+  OTHERLDFLAGS=-Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${mysql-tritonn-5.0:location}/lib/mysql -Wl,-rpath=${openssl:location}/lib

component/perl-DBI/buildout.cfg

+[buildout]
+extends =
+  ../perl/buildout.cfg
+parts =
+  perl-DBI
+
+[perl-DBI]
+recipe = hexagonit.recipe.cmmi
+version = 1.616
+depends =
+  ${perl:version}
+url = http://search.cpan.org/CPAN/authors/id/T/TI/TIMB/DBI-1.616.tar.gz
+md5sum = 799313e54a693beb635b47918458f7c4
+configure-command =
+  ${perl:location}/bin/perl Makefile.PL

component/perl-XML-SAX/buildout.cfg

@@ -12,7 +12,8 @@ depends =
   ${perl-XML-NamespaceSupport:location}
 url = http://search.cpan.org/CPAN/authors/id/G/GR/GRANTM/XML-SAX-0.96.tar.gz
 md5sum = bdcd4119a62505184e211e9dfaef0ab1
+# say 'y' for 'Do you want XML::SAX to alter ParserDetails.ini? [Y]' question.
 configure-command =
-  ${perl:location}/bin/perl Makefile.PL
+  echo y | ${perl:location}/bin/perl Makefile.PL
 environment =
   PERLLIB=blib/lib

component/perl/buildout.cfg

@@ -14,16 +14,17 @@ filename = ${:_buildout_section_name_}
 
 [perl]
 recipe = hexagonit.recipe.cmmi
-version = 5.14.0
+version = 5.14.1
 url = http://www.cpan.org/src/5.0/perl-${:version}.tar.bz2
-md5sum = e7457deea78330c5f8eebb2fd2a45479
+md5sum = 97cd306a2c22929cc141a09568f43bb0
+siteprefix = ${buildout:parts-directory}/site_${:_buildout_section_name_}
 patch-options = -p1
 patches =
   ${perl-keep-linker-flags-in-ldflags.patch:location}/${perl-keep-linker-flags-in-ldflags.patch:filename}
 configure-command =
   sh Configure -des \
     -Dprefix=${buildout:parts-directory}/${:_buildout_section_name_} \
-    -Dsiteprefix=${buildout:parts-directory}/site_${:_buildout_section_name_} \
+    -Dsiteprefix=${:siteprefix} \
     -Dcflags=-I${gdbm:location}/include \
     -Dldflags="-L${gdbm:location}/lib -Wl,-rpath=${gdbm:location}/lib" \
     -Ui_db \

component/readline/buildout.cfg

@@ -6,8 +6,8 @@ extends =
 
 [readline]
 recipe = hexagonit.recipe.cmmi
-url = http://ftp.gnu.org/gnu/readline/readline-6.1.tar.gz
-md5sum = fc2f7e714fe792db1ce6ddc4c9fb4ef3
+url = http://ftp.gnu.org/gnu/readline/readline-6.2.tar.gz
+md5sum = 67948acb2ca081f23359d0256e9a271c
 configure-options =
   --disable-static
   --with-ncurses=${ncurses:location}

component/slapos/buildout.cfg

 [buildout]
 extends =
   ../../stack/shacache-client.cfg
+  ../m2crypto/buildout.cfg
   ../lxml-python/buildout.cfg
   ../python-2.7/buildout.cfg
 
@@ -29,32 +30,52 @@ allowed-eggs-from-site-packages =
 [lxml-python]
 python = python2.7
 
+[M2Crypto]
+python = python2.7
+
 [slapos]
 recipe = z3c.recipe.scripts
 python = python2.7
 eggs =
+  ${M2Crypto:egg}
   slapos.libnetworkcache
   zc.buildout
   ${lxml-python:egg}
   slapos.core
 
+# control scripts generation in order to avoid reinstalling bin/buildout
+scripts =
+  generate-signature-key = slapos.signature:run
+  slapconsole = slapos.console:run
+  slapos-request = slapos.console:request
+  slapformat = slapos.format:main
+  slapgrid = slapos.grid.slapgrid:run
+  slapgrid-sr = slapos.grid.slapgrid:runSoftwareRelease
+  slapgrid-cp = slapos.grid.slapgrid:runComputerPartition
+  slapgrid-ur = slapos.grid.slapgrid:runUsageReport
+  slapgrid-supervisorctl = slapos.grid.svcbackend:supervisorctl
+  slapgrid-supervisord = slapos.grid.svcbackend:supervisord
+  slapproxy = slapos.proxy:main
+
 [versions]
 zc.buildout = 1.5.3-dev-SlapOS-005
-Jinja2 = 2.5.5
-Werkzeug = 0.6.2
+Jinja2 = 2.6
+M2Crypto = 0.21.1
+Werkzeug = 0.7.1
+buildout-versions = 1.6
 hexagonit.recipe.cmmi = 1.5.0
 lxml = 2.3
 meld3 = 0.6.7
 netaddr = 0.7.5
 setuptools = 0.6c12dev-r88846
-slapos.core = 0.9
-slapos.libnetworkcache = 0.2
+slapos.core = 0.12
+slapos.libnetworkcache = 0.4
 xml-marshaller = 0.9.7
 z3c.recipe.scripts = 1.0.1
 zc.recipe.egg = 1.3.2
 
 # Required by:
-# slapos.core==0.9
+# slapos.core==0.12
 Flask = 0.7.2
 
 # Required by:
@@ -62,13 +83,13 @@ Flask = 0.7.2
 hexagonit.recipe.download = 1.5.0
 
 # Required by:
-# slapos.core==0.9
+# slapos.core==0.12
 netifaces = 0.5
 
 # Required by:
-# slapos.core==0.9
+# slapos.core==0.12
 supervisor = 3.0a10
 
 # Required by:
-# slapos.core==0.9
-zope.interface = 3.6.4
+# slapos.core==0.12
+zope.interface = 3.7.0

component/sqlite3/buildout.cfg

@@ -5,8 +5,8 @@ parts =
 
 [sqlite3]
 recipe = hexagonit.recipe.cmmi
-url = http://www.sqlite.org/sqlite-autoconf-3070603.tar.gz
-md5sum = 7eb41eea5ffa5cbe359a48629084c425
+url = http://www.sqlite.org/sqlite-autoconf-3070701.tar.gz
+md5sum = 554026fe7fac47b1cf61c18d5fe43419
 configure-options =
   --disable-static
   --enable-readline

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '0.14-dev'
+version = '0.17-dev'
 name = 'slapos.cookbook'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"

slapos/recipe/erp5/__init__.py

@@ -81,8 +81,25 @@ class Recipe(BaseSlapRecipe):
 
     key, certificate = self.requestCertificate('Login Based Access')
     apache_conf = dict(
-        apache_login=self.installBackendApache(ip=self.getGlobalIPv6Address(),
-          port=13000, backend=site_access, key=key, certificate=certificate))
+         apache_login=self.installBackendApache(ip=self.getGlobalIPv6Address(),
+         port=13000, backend=site_access, key=key, certificate=certificate))
+
+    connection_dict = dict(site_url=apache_conf['apache_login'])
+
+    if self.parameter_dict.get("domain_name") is not None:
+      connection_dict["backend_url"] = apache_conf['apache_login']
+      connection_dict["domain_ip"] = self.getGlobalIPv6Address()
+
+      # XXX Define a fake domain_name for now.
+      frontend_name = self.parameter_dict.get("domain_name")
+      frontend_key, frontend_certificate = \
+             self.requestCertificate(frontend_name)
+
+      connection_dict["site_url"] = self.installFrontendZopeApache(
+        ip=self.getGlobalIPv6Address(), port=13001, name=frontend_name,
+        frontend_path='/%s' % self.site_id, backend_path='/%s' % self.site_id,
+        backend_url="http://%s" % site_access, key=frontend_key,
+        certificate=frontend_certificate)
 
     default_bt5_list = []
     if self.parameter_dict.get("flavour", "default") == 'configurator':
@@ -97,13 +114,13 @@ class Recipe(BaseSlapRecipe):
     self.installTestSuiteRunner(ca_conf, mysql_conf, conversion_server_conf,
                            memcached_conf, kumo_conf)
     self.linkBinary()
-    self.setConnectionDict(dict(
-      site_url=apache_conf['apache_login'],
+    connection_dict.update(**dict(
       site_user=user,
       site_password=password,
       memcached_url=memcached_conf['memcached_url'],
       kumo_url=kumo_conf['kumo_address']
     ))
+    self.setConnectionDict(connection_dict)
     return self.path_list
 
   def installZopeStandalone(self):
@@ -130,14 +147,19 @@ class Recipe(BaseSlapRecipe):
     thread_amount_per_zope = int(self.options.get(
                                 'cluster_zope_thread_amount', 1))
 
-    activity_node_amount = 2
-    user_node_amount = 2
+    activity_node_amount = int(self.options.get(
+                   "cluster_activity_node_amount", 2))
+
+    user_node_amount = int(self.options.get(
+                   "cluster_user_node_amount", 2))
+
     ip = self.getLocalIPv4Address()
     storage_dict = self._requestZeoFileStorage('Zeo Server 1', 'main')
 
     zeo_conf = self.installZeo(ip)
     tidstorage_config = dict(host=ip, port='6001')
 
+    # XXX How to define good values for this?
     mount_point = '/'
     check_path = '/erp5/account_module'
 
@@ -328,6 +350,12 @@ class Recipe(BaseSlapRecipe):
     # workaround wrong assumptions of ERP5Type.tests.runUnitTest about
     # directory existence
     unit_test = os.path.join(testinstance, 'unit_test')
+    connection_string_list = []
+    for test_database, test_user, test_password in \
+          mysql_conf['mysql_parallel_test_dict'][-4:]:
+      connection_string_list.append(
+          '%s@%s:%s %s %s' % (test_database, mysql_conf['ip'],
+                            mysql_conf['tcp_port'], test_user, test_password))
     if not os.path.isdir(unit_test):
       os.mkdir(unit_test)
     runUnitTest = zc.buildout.easy_install.scripts([
@@ -341,6 +369,7 @@ class Recipe(BaseSlapRecipe):
           '--erp5_sql_connection_string', '%(mysql_test_database)s@%'
           '(ip)s:%(tcp_port)s %(mysql_test_user)s '
           '%(mysql_test_password)s' % mysql_conf,
+          '--extra_sql_connection_string_list',','.join(connection_string_list),
           '--conversion_server_hostname=%(conversion_server_ip)s' % \
                                                          conversion_server_conf,
           '--conversion_server_port=%(conversion_server_port)s' % \
@@ -611,7 +640,7 @@ class Recipe(BaseSlapRecipe):
     return user, password
 
   def installERP5Site(self, user, password, zope_access, mysql_conf,
-          conversion_server_conf=None, memcached_conf=None, kumo_conf=None, 
+          conversion_server_conf=None, memcached_conf=None, kumo_conf=None,
           erp5_site_id='erp5', default_bt5_list=[]):
     """ Create a script controlled by supervisor, which creates a erp5
     site on current available zope and mysql environment"""
@@ -881,13 +910,18 @@ class Recipe(BaseSlapRecipe):
     ident = 'frontend_' + name
     apache_conf = self._getApacheConfigurationDict(ident, ip, port)
     apache_conf['server_name'] = name
+    apache_conf['frontend_path'] = frontend_path
     apache_conf['ssl_snippet'] = pkg_resources.resource_string(__name__,
         'template/apache.ssl-snippet.conf.in') % dict(
         login_certificate=certificate, login_key=key)
 
     rewrite_rule_template = \
         "RewriteRule ^%(path)s($|/.*) %(backend_url)s/VirtualHostBase/https/%(server_name)s:%(port)s%(backend_path)s/VirtualHostRoot/_vh_%(vhname)s$1 [L,P]\n"
-    path = pkg_resources.resource_string(__name__, 'template/apache.zope.conf.path-protected.in') % dict(path='/', access_control_string='none')
+
+    path = pkg_resources.resource_string(__name__,
+           'template/apache.zope.conf.path-protected.in') % \
+              dict(path='/', access_control_string='none')
+
     if access_control_string is None:
       path_template = pkg_resources.resource_string(__name__,
         'template/apache.zope.conf.path.in')
@@ -903,8 +937,7 @@ class Recipe(BaseSlapRecipe):
           backend_path=backend_path,
           port=apache_conf['port'],
           vhname=frontend_path.replace('/', ''),
-          server_name=name
-    )
+          server_name=name)
     rewrite_rule = rewrite_rule_template % d
     apache_conf.update(**dict(
       path_enable=path,
@@ -925,7 +958,7 @@ class Recipe(BaseSlapRecipe):
             )
           ]))
     # Note: IPv6 is assumed always
-    return 'https://[%(ip)s]:%(port)s' % apache_conf
+    return 'https://%(server_name)s:%(port)s%(frontend_path)s' % (apache_conf)
 
   def installBackendApache(self, ip, port, backend, key, certificate,
       suffix='', access_control_string=None):
@@ -953,7 +986,8 @@ class Recipe(BaseSlapRecipe):
 
   def installMysqlServer(self, ip, port, database='erp5', user='user',
       test_database='test_erp5', test_user='test_user', template_filename=None,
-      parallel_test_database_amount=100, mysql_conf=None):
+      parallel_test_database_amount=100, mysql_conf=None, with_backup=True,
+      with_maatkit=True):
     if mysql_conf is None:
       mysql_conf = {}
     backup_directory = self.createBackupDirectory('mysql')
@@ -1027,38 +1061,64 @@ class Recipe(BaseSlapRecipe):
        )]))
     self.path_list.extend([mysql_conf_path])
 
-    # backup configuration
-    backup_directory = self.createBackupDirectory('mysql')
-    full_backup = os.path.join(backup_directory, 'full')
-    incremental_backup = os.path.join(backup_directory, 'incremental')
-    self._createDirectory(full_backup)
-    self._createDirectory(incremental_backup)
-    innobackupex_argument_list = [self.options['perl_binary'],
-        self.options['innobackupex_binary'],
-        '--defaults-file=%s' % mysql_conf_path,
-        '--socket=%s' %mysql_conf['socket'].strip(), '--user=root',
-        '--ibbackup=%s'% self.options['xtrabackup_binary']]
-    environment = dict(PATH='%s' % self.bin_directory)
-    innobackupex_incremental = zc.buildout.easy_install.scripts([(
-      'innobackupex_incremental','slapos.recipe.librecipe.execute', 'executee')],
-      self.ws, sys.executable, self.bin_directory, arguments=[
-        innobackupex_argument_list + ['--incremental'],
-        environment])[0]
-    self.path_list.append(innobackupex_incremental)
-    innobackupex_full = zc.buildout.easy_install.scripts([('innobackupex_full',
-     'slapos.recipe.librecipe.execute', 'executee')], self.ws,
-      sys.executable, self.bin_directory, arguments=[
-        innobackupex_argument_list,
-        environment])[0]
-    self.path_list.append(innobackupex_full)
-    backup_controller = zc.buildout.easy_install.scripts([
-      ('innobackupex_controller', __name__ + '.innobackupex', 'controller')],
-      self.ws, sys.executable, self.bin_directory,
-      arguments=[innobackupex_incremental, innobackupex_full, full_backup,
-        incremental_backup])[0]
-    self.path_list.append(backup_controller)
-    mysql_backup_cron = os.path.join(self.cron_d, 'mysql_backup')
-    open(mysql_backup_cron, 'w').write('0 0 * * * ' + backup_controller)
-    self.path_list.append(mysql_backup_cron)
+    if with_backup:
+      # backup configuration
+      backup_directory = self.createBackupDirectory('mysql')
+      full_backup = os.path.join(backup_directory, 'full')
+      incremental_backup = os.path.join(backup_directory, 'incremental')
+      self._createDirectory(full_backup)
+      self._createDirectory(incremental_backup)
+      innobackupex_argument_list = [self.options['perl_binary'],
+          self.options['innobackupex_binary'],
+          '--defaults-file=%s' % mysql_conf_path,
+          '--socket=%s' %mysql_conf['socket'].strip(), '--user=root',
+          '--ibbackup=%s'% self.options['xtrabackup_binary']]
+      environment = dict(PATH='%s' % self.bin_directory)
+      innobackupex_incremental = zc.buildout.easy_install.scripts([(
+        'innobackupex_incremental','slapos.recipe.librecipe.execute', 'executee')],
+        self.ws, sys.executable, self.bin_directory, arguments=[
+          innobackupex_argument_list + ['--incremental'],
+          environment])[0]
+      self.path_list.append(innobackupex_incremental)
+      innobackupex_full = zc.buildout.easy_install.scripts([('innobackupex_full',
+       'slapos.recipe.librecipe.execute', 'executee')], self.ws,
+        sys.executable, self.bin_directory, arguments=[
+          innobackupex_argument_list,
+          environment])[0]
+      self.path_list.append(innobackupex_full)
+      backup_controller = zc.buildout.easy_install.scripts([
+        ('innobackupex_controller', __name__ + '.innobackupex', 'controller')],
+        self.ws, sys.executable, self.bin_directory,
+        arguments=[innobackupex_incremental, innobackupex_full, full_backup,
+          incremental_backup])[0]
+      self.path_list.append(backup_controller)
+      mysql_backup_cron = os.path.join(self.cron_d, 'mysql_backup')
+      open(mysql_backup_cron, 'w').write('0 0 * * * ' + backup_controller)
+      self.path_list.append(mysql_backup_cron)
+
+    if with_maatkit:
+      # maatkit installation
+      for mk_script_name in (
+          'mk-variable-advisor',
+          'mk-table-usage',
+          'mk-visual-explain',
+          'mk-config-diff',
+          'mk-deadlock-logger',
+          'mk-error-log',
+          'mk-index-usage',
+          'mk-query-advisor',
+          ):
+        mk_argument_list = [self.options['perl_binary'],
+            self.options['%s_binary' % mk_script_name],
+            '--defaults-file=%s' % mysql_conf_path,
+            '--socket=%s' %mysql_conf['socket'].strip(), '--user=root',
+            ]
+        environment = dict(PATH='%s' % self.bin_directory)
+        mk_exe = zc.buildout.easy_install.scripts([(
+          mk_script_name,'slapos.recipe.librecipe.execute', 'executee')],
+          self.ws, sys.executable, self.bin_directory, arguments=[
+            mk_argument_list, environment])[0]
+        self.path_list.append(mk_exe)
+
     # The return could be more explicit database, user ...
     return mysql_conf

slapos/recipe/vifib.py

@@ -31,6 +31,9 @@ import zc.buildout
 import sys
 
 class Recipe(slapos.recipe.erp5.Recipe):
+  
+  default_bt5_list = []
+
   def installKeyAuthorisationApache(self, ip, port, backend, key, certificate,
       ca_conf, key_auth_path='/erp5/portal_slap'):
     ssl_template = """SSLEngine on
@@ -172,6 +175,12 @@ SSLCARevocationPath %(ca_crl)s"""
     self.installTidStorage(tidstorage_config['host'], tidstorage_config['port'],
         known_tid_storage_identifier_dict, 'http://'+login_haproxy)
     self.linkBinary()
+
+    # Connect direct to Zope to create the instance.
+    self.installERP5Site(user, password, service_url_list[-1], mysql_conf,
+             conversion_server_conf, memcached_conf, kumo_conf,
+             self.site_id, self.default_bt5_list)
+
     self.setConnectionDict(dict(
       front_end_url=apache_frontend_login,
       site_url=apache_login,
@@ -227,6 +236,10 @@ SSLCARevocationPath %(ca_crl)s"""
     self.installTestSuiteRunner(ca_conf, mysql_conf, conversion_server_conf,
                            memcached_conf, kumo_conf)
     self.linkBinary()
+    self.installERP5Site(user, password, zope_access, mysql_conf,
+             conversion_server_conf, memcached_conf, kumo_conf,
+             self.site_id, self.default_bt5_list)
+
     self.setConnectionDict(dict(
       development_zope='http://%s:%s/' % (ip, zope_port),
       site_user=user,
@@ -260,6 +273,9 @@ SSLCARevocationPath %(ca_crl)s"""
         [('killpidfromfile', 'slapos.recipe.erp5.killpidfromfile',
           'killpidfromfile')], self.ws, sys.executable, self.bin_directory)[0]
     self.path_list.append(self.killpidfromfile)
+    if self.parameter_dict.get("flavour", "default") == 'configurator':
+      self.default_bt5_list = self.options.get("configurator_bt5_list", '').split()
+
     if self.parameter_dict.get('development', 'false').lower() == 'true':
       return self.installDevelopment()
     if self.parameter_dict.get('production', 'false').lower() == 'true':

software/erp5/instance.cfg

@@ -9,6 +9,7 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
 recipe = ${instance-recipe:egg}:${instance-recipe:module}
 dcrond_binary = ${dcron:location}/sbin/crond
 haproxy_binary = ${haproxy:location}/sbin/haproxy
+gzip_binary = ${gzip:location}/bin/gzip
 httpd_binary = ${apache:location}/bin/httpd
 innobackupex_binary = ${xtrabackup:location}/bin/innobackupex
 kumo_gateway_binary = ${kumo:location}/bin/kumo-gateway
@@ -32,6 +33,14 @@ tidstorage_repozo_binary = ${buildout:bin-directory}/tidstorage_repozo
 tidstoraged_binary = ${buildout:bin-directory}/tidstoraged
 xtrabackup_binary = ${xtrabackup:location}/bin/xtrabackup_51
 zabbix_agent_binary = ${zabbix-agent:location}/sbin/zabbix_agent
+mk-variable-advisor_binary = ${perl:siteprefix}/bin/mk-variable-advisor
+mk-table-usage_binary = ${perl:siteprefix}/bin/mk-table-usage
+mk-visual-explain_binary = ${perl:siteprefix}/bin/mk-visual-explain
+mk-config-diff_binary = ${perl:siteprefix}/bin/mk-config-diff
+mk-deadlock-logger_binary = ${perl:siteprefix}/bin/mk-deadlock-logger
+mk-error-log_binary = ${perl:siteprefix}/bin/mk-error-log
+mk-index-usage_binary = ${perl:siteprefix}/bin/mk-index-usage
+mk-query-advisor_binary = ${perl:siteprefix}/bin/mk-query-advisor
 
 # cloudooo specific configuration
 ooo_binary_path = ${libreoffice-bin:location}/program
@@ -69,4 +78,4 @@ environment =
 
 bt5_repository_list = ${bt5-repository:list}
 
-configurator_bt5_list = erp5_core_proxy_field_legacy erp5_full_text_myisam_catalog erp5_base erp5_workflow erp5_configurator erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung
+configurator_bt5_list = ${instance-recipe:configurator_bt5_list}

software/erp5/software.cfg

@@ -21,11 +21,14 @@ unzip = true
 egg = slapos.cookbook
 module = erp5
 
+# Additional Configuration
+configurator_bt5_list = erp5_core_proxy_field_legacy erp5_full_text_myisam_catalog erp5_base erp5_workflow erp5_configurator erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung
+
 [template]
 # Default template for erp5 instance.
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
-md5sum = 65d9b269e204ba49ac5ff11e891a4b84
+md5sum = 77062d472e88aca0fe632bdee4f9bb21
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
@@ -34,7 +37,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/configuration.json
 md5sum = cbe1d75339c6cb20e1aef818797face1 
-output = ${buildout:directory}/validator.json
+output = ${buildout:directory}/schema.json
 mode = 0644
 
 [versions]
@@ -79,17 +82,18 @@ hexagonit.recipe.download = 1.5.0
 http-parser = 0.6.2
 ipdb = 0.4
 meld3 = 0.6.7
+mr.developer = 1.17
 ordereddict = 1.1
 paramiko = 1.7.7.1
 plone.recipe.command = 1.1
 ply = 3.4
-psutil = 0.2.1
+psutil = 0.3.0
 pycrypto = 2.3
-python-ldap = 2.4.0
+python-ldap = 2.4.1
 python-memcached = 1.45
 restkit = 3.3.0
 rtjp-eventlet = 0.3.2
-slapos.cookbook = 0.9
+slapos.cookbook = 0.16
 slapos.recipe.template = 1.1
 threadframe = 0.2
 timerserver = 2.0.2
@@ -99,8 +103,8 @@ validictory = 0.7.1
 xupdate-processor = 0.4
 
 # Required by:
-# slapos.core==0.8
-Flask = 0.7.1
+# slapos.core==0.12
+Flask = 0.7.2
 
 # Required by:
 # PasteScript==1.7.3
@@ -113,7 +117,7 @@ WSGIUtils = 0.7
 
 # Required by:
 # cloudooo==1.2.3
-# slapos.core==0.8
+# slapos.core==0.12
 argparse = 1.1
 
 # Required by:
@@ -129,11 +133,11 @@ fpconst = 0.7.2
 ipython = 0.10.2
 
 # Required by:
-# slapos.cookbook==0.9
+# slapos.cookbook==0.16
 netaddr = 0.7.5
 
 # Required by:
-# slapos.core==0.8
+# slapos.core==0.12
 netifaces = 0.4
 
 # Required by:
@@ -141,17 +145,38 @@ netifaces = 0.4
 python-magic = 0.4.0.1
 
 # Required by:
+# Products.CMFActionIcons==2.1.3
+# Products.CMFCalendar==2.2.2
+# Products.CMFCore==2.2.4
+# Products.CMFDefault==2.2.2
+# Products.CMFTopic==2.2.1
+# Products.CMFUid==2.2.1
+# Products.DCWorkflow==2.2.3nxd002
+# Products.DCWorkflowGraph==0.4nxd001
+# Products.ExternalEditor==1.1.0
+# Products.GenericSetup==1.6.3
+# Products.MimetypesRegistry==2.0.2
+# Products.PluggableAuthService==1.7.5
+# Products.PluginRegistry==1.3b1
+# Products.TIDStorage==5.4.7.dev-r45842
+# Products.Zelenium==1.0.3
+# Zope2==2.12.19
+# five.localsitemanager==2.0.5
+# mr.developer==1.17
+# python-ldap==2.4.1
 # zc.buildout==1.5.3-dev-SlapOS-005
+# zope.deprecation==3.4.0
+# zope.structuredtext==3.4.0
 setuptools = 0.6c12dev-r88846
 
 # Required by:
-# slapos.cookbook==0.9
-slapos.core = 0.8
+# slapos.cookbook==0.16
+slapos.core = 0.12
 
 # Required by:
-# slapos.core==0.8
+# slapos.core==0.12
 supervisor = 3.0a10
 
 # Required by:
-# slapos.cookbook==0.9
+# slapos.cookbook==0.16
 xml-marshaller = 0.9.7

software/nbd/instance.cfg

+[buildout]
+parts =
+  nbdserverinstance
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory} 
+
+[nbdserverinstance]
+recipe = ${instance-recipe:egg}:${instance-recipe:module}
+qemu_path = ${nbdserver:location}/bin/qemu-nbd
+onetimeupload_path = ${buildout:bin-directory}/onetimeupload

software/nbd/software.cfg

+[buildout]
+
+extends =
+  ../../stack/nbd.cfg
+  ../../stack/shacache-client.cfg
+
+parts +=
+  template
+
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg
+md5sum = 82e948e1c0cb0d5540ef185edeef3ec3
+output = ${buildout:directory}/template.cfg
+mode = 0644

software/vifib/software.cfg

@@ -11,19 +11,16 @@ eggs += slapos.core
 [instance-recipe]
 module = vifib
 
+# Additional Configuration
+configurator_bt5_list = erp5_core_proxy_field_legacy erp5_full_text_myisam_catalog erp5_base erp5_workflow erp5_configurator erp5_configurator_vifib
+
+[erp5_repository_list]
+repository_id_list += vifib/master
+
 [vifib]
-# Recipe zerokspot.recipe.git is disabled, as is not possible to change its
-# environment to use localy delivered git.
-# plone.recipe.command can do same job, but it is controllable which binary
-# will be used
-recipe = plone.recipe.command
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-stop-on-error = true
+<= erp5
 repository = http://git.erp5.org/repos/slapos.core.git
-branch = master
-revision = f95ca3ccda07292895939ef9b48678acb5f524ce
-command = ${git:location}/bin/git clone --quiet -b ${:branch} ${:repository} ${:location} && if [ -n ${:revision} ]; then cd ${:location} && ${git:location}/bin/git reset --quiet --hard ${:revision} ; fi
-update-command = cd ${:location} && ${git:location}/bin/git pull --quiet && if [ -n ${:revision} ]; then cd ${:location} && ${git:location}/bin/git reset --quiet --hard ${:revision} ; fi
+revision = 3f766ab93c5312c75b7315a60238c1c5b3e4170d 
 
 [local-bt5-repository]
 # XXX: workaround for zc.buildout bug, as list += ends up with adding new entry

stack/erp5.cfg

@@ -2,12 +2,14 @@
 extensions =
   slapos.rebootstrap
   slapos.zcbworkarounds
+  buildout-versions
   mr.developer
 
 find-links =
     http://www.nexedi.org/static/packages/source/slapos.buildout/
     http://dist.repoze.org
     http://www.nexedi.org/static/packages/source/
+    http://www.owlfish.com/software/wsgiutils/download.html
 
 # Separate from site eggs
 allowed-eggs-from-site-packages =
@@ -25,6 +27,7 @@ allow-hosts =
   peak.telecommunity.com
   psutil.googlecode.com
   www.dabeaz.com
+  www.owlfish.com
 
 extends =
 # Exact version of Zope
@@ -37,12 +40,14 @@ extends =
   ../component/git/buildout.cfg
   ../component/glib/buildout.cfg
   ../component/graphviz/buildout.cfg
+  ../component/gzip/buildout.cfg
   ../component/haproxy/buildout.cfg
   ../component/hookbox/buildout.cfg
   ../component/imagemagick/buildout.cfg
   ../component/kumo/buildout.cfg
   ../component/libreoffice-bin/buildout.cfg
   ../component/lxml-python/buildout.cfg
+  ../component/maatkit/buildout.cfg
   ../component/mariadb/buildout.cfg
   ../component/memcached/buildout.cfg
   ../component/mysql-python/buildout.cfg
@@ -82,7 +87,7 @@ parts =
   stunnel
   w3m
   poppler
-  libpng12
+  libpng
   ghostscript
   mariadb
   sphinx
@@ -215,7 +220,7 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
 stop-on-error = true
 repository = http://git.erp5.org/repos/erp5.git
 branch = master
-revision = 336a8d63bdcabd92bfe3d9466685e5cd47fad716
+revision = dd36cf064495e04f2ad0eaa9c527a2c3246e9ecb 
 command = ${git:location}/bin/git clone --quiet -b ${:branch} ${:repository} ${:location} && if [ -n ${:revision} ]; then cd ${:location} && ${git:location}/bin/git reset --quiet --hard ${:revision} ; fi
 update-command = cd ${:location} && ${git:location}/bin/git pull --quiet && if [ -n ${:revision} ]; then cd ${:location} && ${git:location}/bin/git reset --quiet --hard ${:revision} ; fi
 

stack/nbd.cfg

+[buildout]
+extends =
+  ../component/python-2.6/buildout.cfg
+  ../component/lxml-python/buildout.cfg
+  ../component/zlib/buildout.cfg
+  ../component/git/buildout.cfg
+
+# Python2.6 needed by Flask and pyopenssl
+extensions =
+  slapos.rebootstrap
+
+parts +=
+  nbdserver
+  pyOpenSSL-python
+  eggs
+
+find-links +=
+  http://www.nexedi.org/static/packages/source/slapos.buildout/
+
+versions = versions
+
+# Use only quite well working sites.
+allow-hosts =
+  *.nexedi.org
+  *.python.org
+  *.sourceforge.net
+  dist.repoze.org
+  effbot.org
+  github.com
+  peak.telecommunity.com
+  psutil.googlecode.com
+  www.dabeaz.com
+
+[rebootstrap]
+section = python2.6
+version = 3
+eggs = slapos.libnetworkcache
+
+[nbdserver]
+recipe = hexagonit.recipe.cmmi
+path = ${nbdserversource:location}/
+configure-options =
+  --disable-system
+  --disable-kvm
+  --extra-cflags="-I${zlib:location}/include"
+  --extra-ldflags="-L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib"
+  --disable-werror
+
+[nbdserversource]
+recipe=plone.recipe.command
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+stop-on-error = true
+command =
+  (${git:location}/bin/git clone --quiet http://git.erp5.org/repos/slapos.kvm.git ${:location} ) || (rm -fr ${:location} ; exit 1)
+update-command =
+ cd ${:location} && ${git:location}/bin/git pull --quiet origin master
+
+[instance-recipe]
+egg = slapos.cookbook
+module = nbdserver
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  slapos.toolbox
+  slapos.cookbook
+  pyOpenSSL
+
+[pyOpenSSL-python]
+recipe = zc.recipe.egg:custom
+egg = pyOpenSSL
+include-dirs =
+  ${openssl:location}/include/
+  ${python2.6:location}/include/
+library-dirs =
+  ${openssl:location}/lib/
+  ${python2.6:location}/lib/
+rpath =
+  ${openssl:location}/lib/
+  ${python2.6:location}/lib/
+
+
+[versions]
+# Use SlapOS patched zc.buildout
+zc.buildout = 1.5.3-dev-SlapOS-005
+
+Jinja2 = 2.5.5
+Werkzeug = 0.6.2
+apache-libcloud = 0.5.2
+hexagonit.recipe.cmmi = 1.5.0
+meld3 = 0.6.7
+plone.recipe.command = 1.1
+pyOpenSSL = 0.12
+pycrypto = 2.3
+slapos.cookbook = 0.14
+slapos.recipe.template = 1.1
+slapos.toolbox = 0.6
+
+# Required by:
+# slapos.core==0.9
+# slapos.toolbox==0.6
+Flask = 0.7.2
+
+# Required by:
+# slapos.cookbook==0.14
+PyXML = 0.8.4
+
+# Required by:
+# slapos.core==0.9
+argparse = 1.1
+
+# Required by:
+# slapos.recipe.template==1.1
+collective.recipe.template = 1.8
+
+# Required by:
+# hexagonit.recipe.cmmi==1.5.0
+hexagonit.recipe.download = 1.5.0
+
+# Required by:
+# slapos.cookbook==0.14
+# slapos.core==0.9
+# slapos.toolbox==0.6
+# xml-marshaller==0.9.7
+lxml = 2.3
+
+# Required by:
+# slapos.cookbook==0.14
+netaddr = 0.7.5
+
+# Required by:
+# slapos.core==0.9
+netifaces = 0.4
+
+# Required by:
+# slapos.toolbox==0.6
+paramiko = 1.7.7.1
+
+# Required by:
+# slapos.toolbox==0.6
+psutil = 0.3.0
+
+# Required by:
+# slapos.cookbook==0.14
+# slapos.core==0.9
+# slapos.toolbox==0.6
+# zc.buildout==1.5.3-dev-SlapOS-005
+# zc.recipe.egg==1.3.2
+setuptools = 0.6c12dev-r88846
+
+# Required by:
+# slapos.cookbook==0.14
+# slapos.toolbox==0.6
+slapos.core = 0.9
+
+# Required by:
+# slapos.core==0.9
+supervisor = 3.0a10
+
+# Required by:
+# slapos.cookbook==0.14
+# slapos.toolbox==0.6
+xml-marshaller = 0.9.7
+
+# Required by:
+# slapos.cookbook==0.14
+zc.recipe.egg = 1.3.2
+
+# Required by:
+# slapos.core==0.9
+zope.interface = 3.6.4