Merge remote-tracking branch 'origin/master' into erp5-cluster

component/coreutils/buildout.cfg

@@ -6,8 +6,8 @@ parts =
 
 [coreutils]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/coreutils/coreutils-8.21.tar.xz
-md5sum = 065ba41828644eca5dd8163446de5d64
+url = http://ftp.gnu.org/gnu/coreutils/coreutils-8.23.tar.xz
+md5sum = abed135279f87ad6762ce57ff6d89c41
 configure-options =
   --prefix=${buildout:parts-directory}/${:_buildout_section_name_} --enable-install-program=tr,basename,uname,cat,cp,ls
 environment =

component/cups/buildout.cfg

@@ -9,8 +9,8 @@ extends =
 # for now we build and install library and header files only.
 [cups]
 recipe = slapos.recipe.cmmi
-url = http://www.cups.org/software/1.7.1/cups-1.7.1-source.tar.bz2
-md5sum = 55277c40fd4b7183dc3671d39c5c42b7
+url = http://www.cups.org/software/1.7.4/cups-1.7.4-source.tar.bz2
+md5sum = 1a2295c2b2d2f422db2e50f40ed2fb99
 configure-options =
   --disable-static
   --disable-dbus

component/gettext/buildout.cfg

@@ -10,8 +10,8 @@ extends =
 
 [gettext]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/pub/gnu/gettext/gettext-0.19.1.tar.xz
-md5sum = b52987f49bc99fa8b410270d47a9d52b
+url = http://ftp.gnu.org/pub/gnu/gettext/gettext-0.19.2.tar.xz
+md5sum = 1e6a827f5fbd98b3d40bd16b803acc44
 
 configure-options =
   --disable-static

component/groonga/buildout.cfg

@@ -11,9 +11,9 @@ extends =
 
 [groonga]
 recipe = slapos.recipe.cmmi
-version = 4.0.3
+version = 4.0.4
 url = http://packages.groonga.org/source/groonga/groonga-${:version}.tar.gz
-md5sum = 6f0cb64c0643fc2dd61df6fc542e17e2
+md5sum = b00411d2efae3ddf03eb1ecb8575177f
 configure-options =
   --disable-static
   --disable-glibtest

component/haproxy/buildout.cfg

@@ -11,8 +11,8 @@ parts = haproxy
 
 [haproxy]
 recipe = slapos.recipe.cmmi
-url = http://www.haproxy.org/download/1.5/src/haproxy-1.5.2.tar.gz
-md5sum = e854fed32ea751d6db7f366cb910225a
+url = http://www.haproxy.org/download/1.5/src/haproxy-1.5.3.tar.gz
+md5sum = e999a547d57445d5a5ab7eb6a06df9a1
 configure-command = true
 # If the system is running on Linux 2.6, we use "linux26" as the TARGET,
 # otherwise use "generic".

component/kumo/buildout.cfg

@@ -8,35 +8,14 @@ extends =
 parts = kumo
 find-links = http://www.nexedi.org/static/packages/source/
 
-[kumo-hooks-download]
-url = ${:_profile_base_location_}/${:filename}
-md5sum = 958a595a02de75624728f8d65e39d800
-recipe = hexagonit.recipe.download
-download-only=true
-filename = kumo-hooks.py
-
-[kumo-ipv6-multiip-patch-download]
-recipe = hexagonit.recipe.download
-url = ${:_profile_base_location_}/${:filename}
-filename = kumofs-0.4.13_ipv6support_multiiplistenfix.patch
-md5sum = 53af9f1f1375940841c589a6cbe11425
-download-only = true
-
-[kumo-gcc-4.9-ftbfs-download]
-recipe = hexagonit.recipe.download
-url = ${:_profile_base_location_}/${:filename}
-filename = kumofs-0.4.13_fix_gcc-4.9_ftbfs.patch
-md5sum = c09e04c620ce11c3fdd4afc3459cd355
-download-only = true
-
 [kumo]
-pre-configure-hook = ${kumo-hooks-download:location}/${kumo-hooks-download:filename}:pre_configure_hook
-recipe = erp5.recipe.cmmiforcei686
+recipe = slapos.recipe.cmmi
 url = https://github.com/downloads/etolabo/kumofs/kumofs-0.4.13.tar.gz
 md5sum = 46148e9536222d0ad2ef36777c55714d
+pre-configure-hook = ${:_profile_base_location_}/kumo-hooks.py#958a595a02de75624728f8d65e39d800:pre_configure_hook
 patches =
-  ${kumo-ipv6-multiip-patch-download:location}/${kumo-ipv6-multiip-patch-download:filename}
-  ${kumo-gcc-4.9-ftbfs-download:location}/${kumo-gcc-4.9-ftbfs-download:filename}
+  ${:_profile_base_location_}/kumofs-0.4.13_ipv6support_multiiplistenfix.patch#53af9f1f1375940841c589a6cbe11425
+  ${:_profile_base_location_}/kumofs-0.4.13_fix_gcc-4.9_ftbfs.patch#c09e04c620ce11c3fdd4afc3459cd355
 patch-options = -p1
 configure-options =
   --enable-tcadb

component/libtasn1/buildout.cfg

@@ -4,8 +4,8 @@ parts =
 
 [libtasn1]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.gnu.org/gnu/libtasn1/libtasn1-3.6.tar.gz
-md5sum = 6ed38e161e11013054f2a2bb4c4da449
+url = ftp://ftp.gnu.org/gnu/libtasn1/libtasn1-4.0.tar.gz
+md5sum = d3d2d9bce3b6668b9827a9df52635be1
 configure-options =
   --disable-static
   --disable-gtk-doc-html

component/mariadb/buildout.cfg

@@ -60,8 +60,8 @@ environment =
 # mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
 # http://mroonga.github.com/
 recipe = slapos.recipe.cmmi
-url = http://packages.groonga.org/source/mroonga/mroonga-4.03.tar.gz
-md5sum = 19ab2721d2d41c234e018a879f392990
+url = http://packages.groonga.org/source/mroonga/mroonga-4.04.tar.gz
+md5sum = a89987dad7b3e1d99014492e6135eec6
 configure-options =
   --with-mysql-source=${mariadb:location}__compile__/mariadb-${mariadb:version}
   --with-mysql-config=${mariadb:location}/bin/mysql_config

component/messagepack/buildout.cfg

@@ -3,7 +3,7 @@ parts = messagepack
 find-links = http://www.nexedi.org/static/packages/source/
 
 [messagepack]
-recipe = erp5.recipe.cmmiforcei686
+recipe = slapos.recipe.cmmi
 url = http://downloads.sourceforge.net/project/msgpack/msgpack/cpp/msgpack-0.5.4.tar.gz
 md5sum = 18d96a3178f7cad73c0ca44f6284ae7d
 configure-options =

component/openssl/buildout.cfg

@@ -16,8 +16,8 @@ parts =
 
 [openssl]
 recipe = slapos.recipe.cmmi
-url = https://www.openssl.org/source/openssl-1.0.1h.tar.gz
-md5sum = 8d6d684a9430d5cc98a62a5d8fbda8cf
+url = https://www.openssl.org/source/openssl-1.0.1i.tar.gz
+md5sum = c8dc151a671b9b92ff3e4c118b174972
 depends =
   ${ca-certificates:version}
 patch-binary = ${patch:location}/bin/patch

setup.py

@@ -127,6 +127,7 @@ setup(name=name,
           'ipv4toipv6 = slapos.recipe.6tunnel:FourToSix',
           'ipv6toipv4 = slapos.recipe.6tunnel:SixToFour',
           'java = slapos.recipe.java:Recipe',
+          'jsondump = slapos.recipe.jsondump:Recipe',
           'kumofs = slapos.recipe.kumofs:Recipe',
           'kvm = slapos.recipe.kvm:Recipe',
           'kvm.frontend = slapos.recipe.kvm_frontend:Recipe',
@@ -180,6 +181,7 @@ setup(name=name,
           'siptester = slapos.recipe.siptester:SipTesterRecipe',
           'slapconfiguration = slapos.recipe.slapconfiguration:Recipe',
           'slapconfiguration.serialised = slapos.recipe.slapconfiguration:Serialised',
+          'slapconfiguration.jsondump = slapos.recipe.slapconfiguration:JsonDump',
           'slapcontainer = slapos.recipe.container:Recipe',
           'slapmonitor = slapos.recipe.slapmonitor:MonitorRecipe',
           'slapmonitor-xml = slapos.recipe.slapmonitor:MonitorXMLRecipe',

slapos/recipe/jsondump.py

+
+from slapos.recipe.librecipe import GenericBaseRecipe
+
+import json
+import os
+
+class Recipe(GenericBaseRecipe):
+
+    def install(self):
+        parameter_dict = {
+            key: value
+            for key, value in self.options.items()
+            if key not in ['json-output', 'recipe']
+        }
+
+        with os.fdopen(os.open(self.options['json-output'], os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as fout:
+            fout.write(json.dumps(parameter_dict, indent=2, sort_keys=True))
+
+        return [self.options['json-output']]
+
+    update = install
+

slapos/recipe/slapconfiguration.py

@@ -24,6 +24,10 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+
+import json
+import os
+
 import slapos.slap
 from slapos.recipe.librecipe import unwrap
 from ConfigParser import RawConfigParser
@@ -87,6 +91,15 @@ class Recipe(object):
   OPTCRE_match = RawConfigParser.OPTCRE.match
 
   def __init__(self, buildout, name, options):
+      parameter_dict = self.fetch_parameter_dict(options)
+
+      match = self.OPTCRE_match
+      for key, value in parameter_dict.iteritems():
+          if match(key) is not None:
+              continue
+          options['configuration.' + key] = value
+
+  def fetch_parameter_dict(self, options):
       slap = slapos.slap.slap()
       slap.initializeConnection(
           options['url'],
@@ -138,12 +151,7 @@ class Recipe(object):
           options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8')
 
       options['tap'] = tap_set
-      parameter_dict = self._expandParameterDict(options, parameter_dict)
-      match = self.OPTCRE_match
-      for key, value in parameter_dict.iteritems():
-          if match(key) is not None:
-              continue
-          options['configuration.' + key] = value
+      return self._expandParameterDict(options, parameter_dict)
 
   def _expandParameterDict(self, options, parameter_dict):
       options['configuration'] = parameter_dict
@@ -158,3 +166,16 @@ class Serialised(Recipe):
           return parameter_dict
       else:
           return {}
+
+class JsonDump(Recipe):
+  def __init__(self, buildout, name, options):
+    parameter_dict = self.fetch_parameter_dict(options)
+    self._json_output = options['json-output']
+    with os.fdopen(os.open(self._json_output, os.O_WRONLY | os.O_CREAT, 0600), 'w') as fout:
+      fout.write(json.dumps(parameter_dict, indent=2, sort_keys=True))
+
+    def install(self):
+        return [self._json_output]
+
+    update = install
+

software/apache-frontend/README.apache_frontend.txt

@@ -25,6 +25,7 @@ These parameters are :
   * "-frontend-type" : the type to deploy frontends with. (default to 2)
   * "-frontend-quantity" : The quantity of frontends to request (default to "default")
   * "-frontend-i-state": The state of frontend i
+  * "-frontend-config-i-foo": Frontend i will be requested with parameter foo
   * "-frontend-software-release-url": Software release to be used for frontends, default to the current software release
   * "-sla-i-foo" : where "i" is the number of the concerned frontend (between 1 and "-frontend-quantity") and "foo" a sla parameter.
 ex:
@@ -38,6 +39,12 @@ will request the third frontend on COMP-1234. All frontends will be of software
 Note: the way slaves are transformed to a parameter avoid modifying more than 3 lines in the frontend logic.
 Important NOTE: The way you ask for slave to a replicate frontend is the same as the one you would use for the software given in "-frontend-quantity". Do not forget to use "replicate" for software type. XXXXX So far it is not possible to do a simple request on a replicate frontend if you do not know the software_guid or other sla-parameter of the master instance. In fact we do not know yet the software type of the "requested" frontends. TO BE IMPLEMENTED
 
+XXX Should be moved to specific JSON File
+Extra-parameter per frontend with default :
+ram-cache-size = 1G
+disk-cache-size = 8G
+
+
 How to deploy a frontend server
 ===============================
 

software/apache-frontend/common.cfg

@@ -67,20 +67,20 @@ mode = 0644
 [template-apache-frontend]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-apache-frontend.cfg
-md5sum = 986988ed3a357f87fc023e14d87dcc5c
+md5sum = 53de57ef78345cedd3c715a105539ca3
 output = ${buildout:directory}/template-apache-frontend.cfg
 mode = 0644
 
 [template-apache-replicate]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
-md5sum = 8ec70e6276daaed240faa5059514929c
+md5sum = da22cc3b2095766c5e14b29afab2b760
 mode = 0644
 
 [template-slave-list]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/apache-custom-slave-list.cfg.in
-md5sum = 1c2a47e7e2aa58b4f350ed29ab1a98fe
+md5sum = bae669cdc917c68186a387903478a53d
 mode = 640
 
 [template-slave-configuration]
@@ -98,7 +98,7 @@ mode = 640
 [template-apache-frontend-configuration]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/apache.conf.in
-md5sum = 72922908c1f4e72c92bb03e072660c7c
+md5sum = ce88924c53f09c9a3ef12ec4d8a8ad16
 mode = 640
 
 [template-apache-cached-configuration]
@@ -160,7 +160,7 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
 [template-trafficserver-records-config]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/templates/trafficserver/${:filename}
-md5sum = 950a19be225a25309a3bda3f61fb5f6a
+md5sum = a892d3e71988a8293e44382cbf10810f
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 filename = records.config.jinja2
 download-only = true

software/apache-frontend/instance-apache-frontend.cfg

@@ -118,8 +118,10 @@ configuration.apache_custom_https = ""
 configuration.apache_custom_http = ""
 configuration.apache-key =
 configuration.apache-certificate =
+configuration.apache-ca-certificate =
 configuration.open-port = 80 443
 configuration.extra_slave_instance_list =
+configuration.disk-cache-size = 8G
 
 [frontend-configuration]
 template-log-access = ${template-log-access:target}
@@ -255,6 +257,7 @@ extra-context =
     key access_control_string apache-configuration:access-control-string
     key login_certificate ca-frontend:cert-file
     key login_key ca-frontend:key-file
+    key login_ca_crt ca-custom-frontend:rendered
     key ca_dir  certificate-authority:ca-dir
     key ca_crl certificate-authority:ca-crl
     key access_log apache-configuration:access-log
@@ -379,6 +382,13 @@ cert-content = $${instance-parameter:configuration.apache-certificate}
 # Put domain name
 name = $${instance-parameter:configuration.domain}
 
+[ca-custom-frontend]
+< = jinja2-template-base
+template = ${template-empty:target}
+rendered = $${cadirectory:certs}/apache_frontend.ca.crt
+extra-context =
+    key content instance-parameter:configuration.apache-ca-certificate
+
 [cron]
 recipe = slapos.cookbook:cron
 dcrond-binary = ${dcron:location}/sbin/crond
@@ -456,7 +466,7 @@ local-ip = $${instance-parameter:ipv4-random}
 input-port = 23432
 hostname = $${slap-parameter:frontend-name}
 remap = map / http://$${instance-parameter:ipv4-random}:$${apache-configuration:cache-through-port}
-disk-cache-config = $${trafficserver-directory:cache-path} 8G volume=$${slap-parameter:frontend-name}
+disk-cache-config = $${trafficserver-directory:cache-path} $${instance-parameter:configuration.disk-cache-size} volume=$${slap-parameter:frontend-name}
 plugin-config = ${trafficserver:location}/libexec/trafficserver/rfc5861.so
 
 [trafficserver-configuration-directory]
@@ -481,7 +491,7 @@ environment = TS_ROOT=$${buildout:directory}
 template = ${template-trafficserver-records-config:location}/${template-trafficserver-records-config:filename}
 rendered = $${trafficserver-directory:configuration}/records.config
 mode = 700
-context =
+extra-context =
     import os_module os
     section ats_directory trafficserver-directory
     section ats_configuration trafficserver-variable

software/apache-frontend/instance-apache-replicate.cfg.in

@@ -23,6 +23,7 @@ context =
 {% set slave_list_name = 'extra_slave_instance_list' -%}
 {% set frontend_list = [] %}
 {% set frontend_section_list = [] %}
+{% set request_dict = {} %}
 {% set namebase = 'apache-frontend' -%}
 # XXX Dirty hack, not possible to define default value before
 {% set sla_computer_apache_1_key = '-sla-1-computer_guid' -%}
@@ -37,28 +38,30 @@ context =
 {%   set request_section_title = 'request-%s' % frontend_name -%}
 {%   set sla_key = "-sla-%s-" % i -%}
 {%   set sla_key_length = sla_key | length %}
-{%   set sla_parameters = [] %}
+{%   set sla_dict = {} %}
+{%   set config_key = "-frontend-config-%s-" % i %}
+{%   set config_key_length = config_key | length %}
+{%   set config_dict = {} %}
 {%   for key in slapparameter_dict.keys() %}
 {%     if key.startswith(sla_key) %}
-{%       do sla_parameters.append(key[sla_key_length:]) %}
+{%       do sla_dict.__setitem__(key[sla_key_length:], slapparameter_dict.pop(key)) %}
+# We check for specific configuration regarding the frontend
+{%     elif key.startswith(config_key) -%}
+{%       do config_dict.__setitem__(key[config_key_length:], slapparameter_dict.pop(key)) %}
 {%     endif -%}
 {%   endfor -%}
 {%   do frontend_list.append(frontend_name) -%}
 {%   do frontend_section_list.append(request_section_title) -%}
 {%   do part_list.append(request_section_title) -%}
-[{{request_section_title}}]
-<= replicate
-name = {{frontend_name}}
+# Filling request dict for slave
 {%   set state_key = "-frontend-%s-state" % i %}
-{%   if slapparameter_dict.has_key(state_key) %}
-state = {{ slapparameter_dict.pop(state_key) }}
-{%   endif%}
-{%   if sla_parameters %}
-sla = {{ ' '.join(sla_parameters) }}
-{%     for parameter in sla_parameters -%}
-sla-{{ parameter }} = {{ slapparameter_dict.pop( sla_key + parameter ) }}
-{%     endfor -%}
-{%   endif -%}
+{%   do request_dict.__setitem__(request_section_title,
+                                  {
+                                  'config': config_dict,
+                                  'name': frontend_name,
+                                  'sla': sla_dict,
+                                  'state': slapparameter_dict.pop(state_key, None)
+                                  }) %}
 {% endfor -%}
 
 {% set authorized_slave_string = slapparameter_dict.pop('-frontend-authorized-slave-string', '') -%}
@@ -83,13 +86,29 @@ software-url = ${slap-connection:software-release-url}
 {% endif %}
 software-type = {{frontend_type}}
 return = private-ipv4 public-ipv4 slave-instance-information-list monitor_url
-config = {{ ' '.join(slapparameter_dict.keys()) + ' ' + slave_list_name }}
-{% for parameter, value in slapparameter_dict.iteritems() -%}
-config-{{parameter}} = {{ value }}
-{% endfor -%}
-config-{{ slave_list_name }} = {{ json_module.dumps(authorized_slave_list) }}
+config = _
 connection-monitor_url =
 
+{% for section, frontend_request in request_dict.iteritems() %}
+[{{section}}]
+<= replicate
+name = {{ frontend_request.get('name') }}
+{%   if frontend_request.get('state') %}
+state = {{ frontend_request.get('state') }}
+{%   endif%}
+{%   set slave_configuration_dict = frontend_request.get('config') %}
+{%   do slave_configuration_dict.update(**slapparameter_dict) %}
+{%   do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list)) %}
+config-_ = {{ json_module.dumps(slave_configuration_dict) }}
+{%   if frontend_request.get('sla') %}
+sla = {{ ' '.join(frontend_request.get('sla').keys()) }}
+{%     for parameter, value in frontend_request.get('sla').iteritems() -%}
+sla-{{ parameter }} = {{ value }}
+{%     endfor -%}
+{%   endif -%}
+{% endfor -%}
+
+
 [publish-information]
 recipe = slapos.cookbook:publish
 domain = {{ slapparameter_dict.get('domain') }}

software/apache-frontend/templates/apache-custom-slave-list.cfg.in

@@ -189,7 +189,7 @@ extra-context =
     raw http_port {{ http_port }}
 {{ '\n' }}
 
-{%     do slave_publish_dict.update(**{'slave-reference':slave_instance.get('slave_reference'), 'public-ipv4':public_ipv4, 'domain':slave_instance.get('custom_domain'), 'url':"http://%s" % slave_instance.get('custom_domain'), 'site_url':"http://%s" % slave_instance.get('custom_domain')}) %}
+{%     do slave_publish_dict.update(**{'slave-reference':slave_instance.get('slave_reference'), 'public-ipv4':public_ipv4, 'domain':slave_instance.get('custom_domain'), 'url':"http://%s" % slave_instance.get('custom_domain'), 'site_url':"http://%s" % slave_instance.get('custom_domain'), 'secure_access': 'https://%s' % slave_instance.get('custom_domain')}) %}
 
 {%   endif -%}
 

software/apache-frontend/templates/apache.conf.in

@@ -97,14 +97,6 @@ BrowserMatch "^Dreamweaver-WebDAV-SCM1" redirect-carefully
    IPReadLimit {{ slapparameter_dict.get('ip-read-limit', '10') }}
 </IfModule>
 
-# Cache directives
-CacheEnable mem /
-CacheDefaultExpire 3600
-MCacheSize 8192
-MCacheMaxObjectCount 1000
-MCacheMaxObjectSize 8192
-MCacheRemovalAlgorithm LRU
-
 # Deflate
 AddOutputFilterByType DEFLATE text/html text/plain text/xml text/css text/javascript application/x-javascript application/javascript
 BrowserMatch ^Mozilla/4 gzip-only-text/html
@@ -114,6 +106,9 @@ BrowserMatch \bMSIE !no-gzip !gzip-only-text/html
 # SSL Configuration
 SSLCertificateFile {{ login_certificate }}
 SSLCertificateKeyFile {{ login_key }}
+{% if slapparameter_dict.get('apache-ca-certificate') %}
+SSLCACertificateFile {{ login_ca_crt }}
+{% endif %}
 SSLRandomSeed startup builtin
 SSLRandomSeed connect builtin
 SSLSessionCache shmcb:/{{ httpd_mod_ssl_cache_directory }}/ssl_scache(512000)

software/apache-frontend/templates/trafficserver/records.config.jinja2

@@ -228,8 +228,8 @@ CONFIG proxy.config.http.cache.http INT 1
    # Enabling this setting allows the proxy to cache empty documents. This currently
    # requires that the response has a Content-Length: header, with a value of "0".
 CONFIG proxy.config.http.cache.allow_empty_doc INT 0
-CONFIG proxy.config.http.cache.ignore_client_no_cache INT 1
-CONFIG proxy.config.http.cache.ims_on_client_no_cache INT 1
+CONFIG proxy.config.http.cache.ignore_client_no_cache INT 0
+CONFIG proxy.config.http.cache.ims_on_client_no_cache INT 0
 CONFIG proxy.config.http.cache.ignore_server_no_cache INT 0
 CONFIG proxy.config.http.cache.ignore_client_cc_max_age INT 0
 CONFIG proxy.config.http.normalize_ae_gzip INT 0
@@ -331,8 +331,7 @@ CONFIG proxy.config.cache.permit.pinning INT 0
    # default the ram cache size to AUTO_SIZE (-1) based on cache size
    #   (approximately 10 MB of RAM cache per GB of disk cache)
    # alternatively, set to a fixed value such as 21474836480 (20GB)
-CONFIG proxy.config.cache.ram_cache.size INT 1G
-CONFIG proxy.config.cache.ram_cache_cutoff INT 4194304
+CONFIG proxy.config.cache.ram_cache.size INT {{ slapparameter_dict.get('ram-cache-size', '1G') }}
    # Replacement algorithm
    #  0 : Clocked Least Frequently Used by Size (CLFUS) w/optional compression
    #  1 : LRU w/o optional compression - trivially simple
@@ -366,9 +365,6 @@ CONFIG proxy.config.cache.target_fragment_size INT 1048576
 CONFIG proxy.config.cache.max_doc_size INT 0
    # enable the cache to read from an object while it is being added to the cache
 CONFIG proxy.config.cache.enable_read_while_writer INT 0
-   # This controls how many objects (average) the disk caches can hold, and
-   # how much memory it'll consume for the directory structure.
-CONFIG proxy.config.cache.min_average_object_size INT 8000
    # How many I/O threads to allocate per disk (spindle). Be aware that RAID
    # disks would show up to TS as a single spindle.
 CONFIG proxy.config.cache.threads_per_disk INT 8

software/erp5testnode/software.cfg

@@ -74,7 +74,7 @@ recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-default.cfg
 output = ${buildout:directory}/template-default.cfg
 mode = 0644
-md5sum = 8c6a106be8e7af2b30506a9a2e24044d
+md5sum = 7beecdffede3c0abf83944b57476f84f 
 
 [networkcache]
 # signature certificates of the following uploaders.
@@ -193,7 +193,7 @@ buildout-versions = 1.7
 cffi = 0.8.2
 cmd2 = 0.6.7
 cryptography = 0.4
-erp5.util = 0.4.39
+erp5.util = 0.4.41
 inotifyx = 0.2.0-1
 itsdangerous = 0.24
 lxml = 3.3.5

software/kumofs/software.cfg

@@ -76,8 +76,6 @@ signature-certificate-list =
   Jinja2 = 2.6
   Werkzeug = 0.8.3
   buildout-versions = 1.7
-  erp5.recipe.cmmiforcei686 = 0.1.3
-  hexagonit.recipe.cmmi = 1.6
   meld3 = 0.6.10
   rdiff-backup = 1.0.5
   slapos.cookbook = 0.71.1

software/test-slave-instance-deployment/instance-default.cfg

+[buildout]
+parts =
+  publish-connection-informations
+  request-slave-instance
+  check-connection-informations-promise
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[directories]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+etc = $${buildout:directory}/etc
+scripts = $${:etc}/run
+services = $${:etc}/service
+promises = $${:etc}/promise
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[request-test-instance]
+# Deploy a dummy instance
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Test Root Instance
+software-url = $${slap-connection:software-release-url}
+software-type = test
+return = foo slave-instance-list
+
+[request-slave-instance]
+# Deploy a dummy instance
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Test Root Instance - Slave
+software-url = $${slap-connection:software-release-url}
+software-type = test
+slave = true
+sla = instance_guid
+sla-instance_guid = $${request-test-instance:instance-guid}
+return = foo
+
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+note = test-instance-guid should be equal to XXX, test-instance-state should be equal to started, test-instance-slave-instance-list should be NOT empty, and root-instance-slave-instance-list SHOULD be empty, slave-instance-parameter should NOT be empty.
+test-instance-guid = $${request-test-instance:instance-guid}
+test-instance-state = $${request-test-instance:instance-state}
+test-instance-slave-instance-list = $${request-test-instance:connection-slave-instance-list}
+root-instance-slave-instance-list = $${slap-configuration:slave-instance-list}
+slave-instance-parameter = $${request-slave-instance:connection-foo}
+
+# Create a file containing connection parameters
+# Then create a promise that will check content of this file
+[dump-connection-informations]
+<= publish-connection-informations
+recipe = slapos.cookbook:jsondump
+json-output = $${buildout:directory}/connection-parameters.json
+
+[check-connection-informations-promise]
+recipe = collective.recipe.template
+input = ${promise-template:output}
+output = $${directories:promises}/checkparameters
+mode = 700

software/test-slave-instance-deployment/instance-test.cfg

+[buildout]
+parts =
+  publish-connection-informations
+  publish-slave-information
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+foo = bar
+slave-instance-list = $${slap-configuration:slave-instance-list}
+
+[publish-slave-information]
+recipe = slapos.cookbook:publish
+# XXX hardcoded
+-slave-reference = slappart0_Test Root Instance - Slave
+foo = I am slave instance, and I am deployed!
\ No newline at end of file

software/test-slave-instance-deployment/instance.cfg

+[buildout]
+parts =
+  switch-softwaretype
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = ${instance-default-profile:output}
+test = ${instance-test-profile:output}
+
+[slap-connection]
+# part to migrate to new - separated words
+computer-id = $${slap_connection:computer_id}
+partition-id = $${slap_connection:partition_id}
+server-url = $${slap_connection:server_url}
+software-release-url = $${slap_connection:software_release_url}
+key-file = $${slap_connection:key_file}
+cert-file = $${slap_connection:cert_file}

software/test-slave-instance-deployment/parameter-check-promise.py.in

+#!${buildout:executable}
+import sys
+import os
+import json
+
+json_file = '$${dump-connection-informations:json-output}'
+connection_parameter_list = json.loads(open(json_file, 'r').read())
+# XXX list all problems before exit
+# XXX: factor in a clean dict loop
+test_instance_guid = connection_parameter_list['test-instance-guid']
+if not test_instance_guid == 'slappart1':
+  print 'Bad test-instance-guid connection parameter: %s' % test_instance_guid
+  sys.exit(1)
+test_instance_state = connection_parameter_list['test-instance-state']
+if not test_instance_state == 'started':
+  print 'Bad test-instance-state connection parameter: %s instead of started' % test_instance_state
+  sys.exit(1)
+test_instance_slave_instance_list = connection_parameter_list['test-instance-slave-instance-list']
+if test_instance_slave_instance_list == []:
+  print 'test-instance-slave-instance-list connection parameter is empty.'
+  sys.exit(1)
+root_instance_slave_instance_list = connection_parameter_list['root-instance-slave-instance-list']
+if root_instance_slave_instance_list != []:
+  print 'root-instance-slave-instance-list connection parameter is not empty.'
+  sys.exit(1)
+slave_instance_parameter = connection_parameter_list['slave-instance-parameter']
+if slave_instance_parameter != 'I am slave instance, and I am deployed!':
+  print 'parameter slave-instance-parameter is not correct.'
+  sys.exit(1)

software/test-slave-instance-deployment/software.cfg

+# This Software Release is used to test
+# the behavior of Slave Instances in slapos and in slaprunner.
+[buildout]
+
+develop =
+	/srv/slapgrid/slappart74/srv//runner/project/slapos
+
+parts =
+  slapos-cookbook
+  instance-profile
+  collective-egg
+
+[instance-profile]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg
+output = ${buildout:directory}/instance.cfg
+mode = 0644
+
+[instance-default-profile]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-default.cfg
+output = ${buildout:directory}/instance-default.cfg
+mode = 0644
+
+[instance-test-profile]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-test.cfg
+output = ${buildout:directory}/instance-test.cfg
+mode = 0644
+
+[promise-template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/parameter-check-promise.py.in
+output = ${buildout:directory}/parameter-check-promise.py
+mode = 0644
+
+[collective-egg]
+recipe = zc.recipe.egg
+eggs = collective.recipe.template
+
+[versions]
+Jinja2 = 2.7.3
+MarkupSafe = 0.23
+Werkzeug = 0.9.6
+buildout-versions = 1.7
+cffi = 0.8.6
+cliff = 1.6.1
+cmd2 = 0.6.7
+cryptography = 0.5.2
+inotifyx = 0.2.0-1
+itsdangerous = 0.24
+lxml = 3.3.5
+meld3 = 1.0.0
+mr.developer = 1.30
+netaddr = 0.7.12
+netifaces = 0.10.4
+pyOpenSSL = 0.14
+pyparsing = 2.0.2
+pytz = 2014.4
+requests = 2.3.0
+six = 1.7.3
+slapos.cookbook = 0.87
+slapos.core = 1.1.2
+slapos.recipe.cmmi = 0.2
+slapos.recipe.template = 2.5
+stevedore = 0.15
+supervisor = 3.1.0
+xml-marshaller = 0.9.7
+collective.recipe.template = 1.11
+
+# Required by:
+# slapos.core==1.1.2
+Flask = 0.10.1
+
+# Required by:
+# cliff==1.6.1
+argparse = 1.2.1
+
+# Required by:
+# slapos.cookbook==0.87
+lock-file = 2.0
+
+# Required by:
+# slapos.core==1.1.2
+psutil = 2.1.1
+
+# Required by:
+# cffi==0.8.6
+pycparser = 2.10
+
+# Required by:
+# slapos.core==1.1.2
+zope.interface = 4.1.1
+
+
+# Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.
+[buildout]
+# Explicitly disable download-cache
+download-cache =
+
+# Generate list of automatically chosen eggs version
+extensions +=
+  buildout-versions
+  mr.developer
+
+# Use shacache and lxml
+extends =
+  ../../component/lxml-python/buildout.cfg
+
+# Separate from site eggs
+allowed-eggs-from-site-packages =
+include-site-packages = false
+exec-sitecustomize = false
+
+# Add location for modified non-official slapos.buildout
+find-links +=
+  http://www.nexedi.org/static/packages/source/slapos.buildout/
+  http://www.nexedi.org/static/packages/source/
+  http://www.nexedi.org/static/packages/source/hexagonit.recipe.download/
+
+# Use only quite well working sites.
+allow-hosts +=
+  *.googlecode.com
+  *.nexedi.org
+  *.python.org
+  *.sourceforge.net
+  alastairs-place.net
+  bitbucket.org
+  dist.repoze.org
+  effbot.org
+  github.com
+  launchpad.net
+  peak.telecommunity.com
+  sourceforge.net
+  www.dabeaz.com
+  www.owlfish.com
+
+# XXX: Workaround of SlapOS limitation
+# Unzippig of eggs is required, as SlapOS do not yet provide nicely working
+# development / fast switching environment for whole software
+unzip = true
+
+# Define where is defined what version of eggs we should use
+versions = versions
+
+# Define networkcache with shacache.org
+networkcache-section = networkcache
+
+# Install slapos.cookbook containing all officials recipes
+# Explicitely define dependencies as well, because of buildout limitation
+# if using "develop"
+[slapos-cookbook]
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+  slapos.cookbook
+  cliff
+  hexagonit.recipe.download
+  inotifyx
+  netaddr
+  netifaces
+  requests
+  slapos.core
+  supervisor
+  xml_marshaller
+  pytz
+
+[versions]
+# Use SlapOS patched zc.buildout
+zc.buildout = 1.6.0-dev-SlapOS-012
+# zc.recipe.egg 2.x is for Buildout 2
+zc.recipe.egg = 1.3.2
+# Use own version of h.r.download to be able to open xz-like archives
+hexagonit.recipe.download = 1.7nxd002
+# Use pinned version of setuptools. Other versions work, but changing
+# version makes buildout recompile everything. Developers' nightmare.
+setuptools = 0.9.8
+# Official egg of prettytable has permission problems in EGG-INFO.
+prettytable = 0.7.3-nxd001
+
+[networkcache]
+download-cache-url = http://www.shacache.org/shacache
+download-dir-url = http://www.shacache.org/shadir
\ No newline at end of file

stack/erp5/buildout.cfg

@@ -683,29 +683,28 @@ StructuredText = 2.11.1
 WSGIUtils = 0.7
 Werkzeug = 0.9.6
 apache-libcloud = 0.15.1
-astroid = 1.1.1
+astroid = 1.2.0
 async = 0.6.1
 buildout-versions = 1.7
 chardet = 2.2.1
 cmd2 = 0.6.7
 csp-eventlet = 0.7.0
 elementtree = 1.2.7-20070827-preview
-erp5.recipe.cmmiforcei686 = 0.1.3
 erp5diff = 0.8.1.5
 eventlet = 0.15.0
 five.formlib = 1.0.4
 five.localsitemanager = 2.0.5
 gitdb = 0.5.4
-greenlet = 0.4.2
+greenlet = 0.4.3
 http-parser = 0.8.3
 httplib2 = 0.9
 inotifyx = 0.2.0-1
 interval = 1.0.0
 ipdb = 0.8
-ipython = 2.1.0
+ipython = 2.2.0
 itsdangerous = 0.24
 lock-file = 2.0
-logilab-common = 0.62.0
+logilab-common = 0.62.1
 meld3 = 1.0.0
 minitage.paste = 1.4.6
 minitage.recipe.egg = 1.107
@@ -716,7 +715,7 @@ polib = 1.0.4
 pprofile = 1.7.2
 pycountry = 1.8
 pyflakes = 0.8.1
-pylint = 1.2.1
+pylint = 1.3.0
 pyparsing = 2.0.2
 python-ldap = 2.4.15
 python-magic = 0.4.6
@@ -738,7 +737,7 @@ threadframe = 0.2
 timerserver = 2.0.2
 urlnorm = 1.1.2
 uuid = 1.30
-validictory = 1.0.0a1
+validictory = 1.0.0a2
 xupdate-processor = 0.4
 xfw = 0.10
 
@@ -769,7 +768,7 @@ ecdsa = 0.11
 
 # Required by:
 # cloudooo==1.2.5-dev
-erp5.util = 0.4.38
+erp5.util = 0.4.41
 
 # Required by:
 # slapos.toolbox==0.39.3
@@ -779,10 +778,6 @@ feedparser = 5.1.3
 # SOAPpy==0.12.0nxd001
 fpconst = 0.7.2
 
-# Required by:
-# erp5.recipe.cmmiforcei686==0.1.3
-hexagonit.recipe.cmmi = 2.0
-
 # Required by:
 # minitage.recipe.egg==1.107
 iniparse = 0.4
@@ -827,7 +822,7 @@ six = 1.7.3
 
 # Required by:
 # slapos.core==1.1.2
-supervisor = 3.0
+supervisor = 3.1.0
 
 # Required by:
 # slapos.toolbox==0.39.3