erp5: review request parameters for SLA & ZODB

All parameters about SLA, i.e. computer-guid & instance-guid, are removed in
favor of a new "sla-dict" parameter, which is easier to implement and much more
versatile.

All changes in the request parameters are incompatible. The old ones are
ignored without warning/error.

For compatibility, the reference of ZEO partition is still "zodb".

Default settings were also fine. Default name of mount-point and FileStorage
file is reverted to 'root' instead of 'main'.

software/erp5/instance-erp5-input-schema.json

@@ -4,6 +4,17 @@
   "description": "Parameters to instantiate ERP5",
   "additionalProperties": false,
   "properties": {
+    "sla-dict": {
+      "description": "Where to request instances. Each key is a query string for criterions (e.g. \"computer_guid=foo\"), and each value is a list of partition references.",
+      "patternProperties": {
+        ".": {
+          "type": "array",
+          "items": { "type": "string" },
+          "uniqueItems": true
+        }
+      },
+      "type": "object"
+    },
     "site-id": {
       "description": "ERP5Site object's id",
       "default": "erp5",
@@ -60,7 +71,7 @@
       "description": "Front-end slave instance request parameters",
       "properties": {
         "software-url": {
-          "description": "Front-end's software type. If this parameter is the empty string, no front-end instance is requested.",
+          "description": "Front-end's software type. If this parameter is empty, no front-end instance is requested. Else, sla-dict must specify 'frontend' which is a special value matching all frontends (e.g. {\"instance_guid=bar\": [\"frontend\"]}).",
           "default": "",
           "type": "string"
         },
@@ -74,10 +85,6 @@
           "default": "RootSoftwareInstance",
           "type": ""
         },
-        "instance-guid": {
-          "description": "GUID of computer instance must be requested on. Mandatory if software-url is a non-empty string.",
-          "type": "string"
-        }
       },
       "type": "object"
     },
@@ -130,20 +137,12 @@
                 "default": 2200
               }]
             },
-            "computer-guid": {
-              "description": "GUID of computer instance must be requested on",
-              "type": "string"
-            }
           },
           "type": "object"
         }
       },
       "type": "object"
     },
-    "kumofs-computer-guid": {
-      "description": "GUID of computer instance must be requested on",
-      "type": "string"
-    },
     "kumofs": {
       "description": "Persistent memcached service",
       "additionalProperties": {
@@ -151,10 +150,6 @@
       },
       "type": "object"
     },
-    "memcached-computer-guid": {
-      "description": "GUID of computer instance must be requested on",
-      "type": "string"
-    },
     "memcached": {
       "description": "Volatile memcached service",
       "additionalProperties": {
@@ -162,10 +157,6 @@
       },
       "type": "object"
     },
-    "cloudooo-computer-guid": {
-      "description": "GUID of computer instance must be requested on",
-      "type": "string"
-    },
     "cloudooo": {
       "description": "Format conversion service",
       "additionalProperties": {
@@ -173,10 +164,6 @@
       },
       "type": "object"
     },
-    "mariadb-computer-guid": {
-      "description": "GUID of computer instance must be requested on",
-      "type": "string"
-    },
     "mariadb": {
       "description": "Relational database service",
       "additionalProperties": {
@@ -184,26 +171,33 @@
       },
       "type": "object"
     },
-    "zodb-computer-guid": {
-      "description": "GUID of computer instance must be requested on",
-      "type": "string"
-    },
-    "zodb-software-type": {
-      "description": "Zope Object DataBase implementation",
-      "enum": ["zeo"],
-      "type": "string"
+    "zodb-zeo": {
+      "description": "Common settings ZEO servers",
+      "properties": {
+        "tcpv4-port": {
+          "allOf": [{
+            "$ref": "#/definitions/tcpv4port"
+          }, {
+            "description": "Start allocating ports at this value, going upward"
+          }]
+        },
+        "backup-periodicity": {
+          "description": "When to backup, specified in the same format as for systemd.time(7) calendar events (years & seconds not supported, DoW & DoM can not be combined). Enter 'never' to disable backups.",
+          "default": "daily",
+          "type": "string",
+        },
+        "tidstorage-timestamp-path": {
+          "description": "Path to backup timestamp file, occurrences of '%(backup)s' being replaced with the path to partition's srv/backup/tidstorage directory",
+          "default": "%(backup)s/repozo_tidstorage_timestamp.log",
+          "type": "string"
+        }
+      },
+      "type": "object"
     },
     "zodb": {
-      "description": "Zope Object DataBase service",
-      "oneOf": [
-        {"$ref": "./instance-zeo-schema.json#properties"}
-      ]
-    },
-    "zodb-extern": {
-      "description": "External ZODB Mountpoints. See https://github.com/zopefoundation/ZODB/blob/3.10/src/ZODB/component.xml for extra options.",
-      "minItems": 0,
+      "description": "Zope Object DataBase mountpoints. See https://github.com/zopefoundation/ZODB/blob/3.10/src/ZODB/component.xml for extra options.",
       "items": {
-        "required": ["storage_dict"],
+        "required": ["type"],
         "properties": {
           "name": {
             "description": "Database name",
@@ -215,13 +209,19 @@
             "default": "/",
             "type": "string"
           },
-          "storage-type": {
+          "type": {
             "description": "Storage type",
-            "default": "NEOStorage",
+            "enum": ["zeo", "neo"],
             "type": "string"
           },
+          "server": {
+            "description": "Instantiate a server. If missing, 'storage-dict' must contain the necessary properties to mount the ZODB.",
+            "oneOf": [
+              {"$ref": "./instance-zeo-schema.json#properties"}
+            ]
+          },
           "storage-dict": {
-            "description": "Storage configuration. For NEOStorage, 'logfile' is automatically set (see http://git.erp5.org/gitweb/neoppod.git/blob/HEAD:/neo/client/component.xml for other settings).",
+            "description": "Storage configuration. For NEO, 'logfile' is automatically set (see http://git.erp5.org/gitweb/neoppod.git/blob/HEAD:/neo/client/component.xml for other settings).",
             "patternProperties": {
               ".": {
                 "type": "string"
@@ -229,10 +229,10 @@
             },
             "type": "object"
           },
-          "patternProperties": {
-            ".": {
-              "type": "string"
-            }
+        },
+        "patternProperties": {
+          ".": {
+            "type": "string"
           }
         },
         "type": "object"

software/erp5/instance-zeo-schema.json

 {
   "$schema": "http://json-schema.org/draft-04/schema#",
   "extends": "./schema-definitions.json#",
-  "required": ["tcpv4-port", "zodb-dict"],
   "properties": {
-    "tcpv4-port": {
-      "allOf": [{
-        "$ref": "#/definitions/tcpv4port"
-      }, {
-        "description": "Start allocating ports at this value, going upward"
-      }]
-    },
     "backup": {
-      "description": "Controls ZODB backup generation and retention (disabled if null)",
-      "default": {},
-      "properties": {
-        "periodicity": {
-          "description": "When to backup, specified in the same format as for systemd.time(7) calendar events (years & seconds not supported, DoW & DoM can not be combined)",
-          "default": "daily",
-          "type": "string",
-        },
-        "zodb-dict": {
-          "description": "Maps an export identifier to the path it must be backed up into",
-          "default": {},
-          "patternProperties": {
-            ".*": {
-              "description": "'%(backup)s' is expanded to partition's ZODB backup path (typically 'srv/backup/zodb'), and %(name)s with the export id",
-              "default": "%(backup)s/%(name)s",
-              "type": "string"
-            }
-          },
-          "type": "object"
-        },
-        "tidstorage-timestamp-path": {
-          "description": "Path to backup timestamp file, occurrences of '%(backup)s' being eplaced with the path to partition's srv/backup/tidstorage directory",
-          "default": "%(backup)s/repozo_tidstorage_timestamp.log",
-          "type": "string"
-        }
-      },
-      "type": ["object", "null"]
+      "description": "'%(backup)s' is expanded to partition's ZODB backup path (typically 'srv/backup/zodb'), and %(name)s with the export id",
+      "default": "%(backup)s/%(name)s",
+      "type": "string"
+    },
+    "family": {
+      "description": "Opaque name used to regroup/separate mountpoints under different ZEO processes (must be valid as a file name and as a ConfigParser section name)",
+      "default": "default",
+      "pattern": "^[^<>:\"/\\|?*\\]\\[ ]*$",
+      "type": "string"
+    },
+    "path": {
+      "description": "FileStorage file path, '%(zodb)s' occurrences are replaced with the path to partition's srv/zodb directory, and %(name)s with the export id",
+      "default": "%(zodb)s/%(name)s.fs",
+      "type": "string"
     },
-    "zodb-dict": {
-      "description": "Maps an export identifier to its settings",
-      "patternProperties": {
-        ".*": {
-          "properties": {
-            "storage-family": {
-              "description": "Opaque name used to regroup/separate mountpoints under different ZEO processes (must be valid as a file name and as a ConfigParser section name)",
-              "default": "default",
-              "pattern": "^[^<>:\"/\\|?*\\]\\[ ]*$",
-              "type": "string"
-            },
-            "mount-point": {
-              "description": "Traversal path export should be mounted at (opaque to this software type)",
-              "default": "/",
-              "type": "string"
-            },
-            "cache-size": {
-              "description": "Client-side cache size in object count (opaque to this software type)",
-              "default": -1,
-              "type": "integer"
-            },
-            "storage-dict": {
-              "default": {},
-              "properties": {
-                "path": {
-                  "description": "FileStorage file path, '%(zodb)s' occurrences are replaced with the path to partition's srv/zodb directory, and %(name)s with the export id",
-                  "default": "%(zodb)s/%(name)s.fs",
-                  "type": "string"
-                },
-                "client": {
-                  "description": "Client-side settings for this mountpoint",
-                  "default": {},
-                  "patternProperties": {
-                    ".*": {
-                      "type": "string"
-                    }
-                  },
-                  "type": "object"
-                }
-              },
-              "type": "object"
-            }
-          },
-          "type": "object"
-        }
-      },
-      "type": "object"
-    }
   }
 }

stack/erp5/buildout.cfg

@@ -186,7 +186,7 @@ md5sum = 48d5e8c3efc5bfd6fc1027b5ebe55e64
 [template-zope-conf]
 < = download-base
 filename = zope.conf.in
-md5sum = 9e68f6a221c16e654efb05632fed5aee
+md5sum = 0bf51218ecbf2bd319214192448a3ef7
 
 [template-runzope-userhosts-preloaded]
 < = download-base
@@ -218,7 +218,7 @@ md5sum = e40e2e39f4941c6372f4357e8589a5cf
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 filename = template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 79bc92c8d6ad75e78b901e96c8804314
+md5sum = 2738587138c9bd10af60c6552ecac360
 extra-context =
     key mariadb_link_binary template-mariadb:link-binary
     key zope_link_binary template-zope:link-binary
@@ -289,17 +289,17 @@ extra-context =
 [template-erp5]
 < = download-base
 filename = instance-erp5.cfg.in
-md5sum = 386f5f15d71636f855f22201d38367e5
+md5sum = 28db329061ae863367a76951b9b1152c
 
 [template-zeo]
 < = download-base
 filename = instance-zeo.cfg.in
-md5sum = 6bf4c5f7f476d420b2429dd0eca8ec0b
+md5sum = e810ce0bf324a33d0d52d3ecda67bc4d
 
 [template-zope]
 < = download-base
 filename = instance-zope.cfg.in
-md5sum = 5ffaadfb2d465cba53bcbc4287df041e
+md5sum = 129b0468957dec95c0c04c3484df8624
 link-binary =
   ${aspell:location}/bin/aspell
   ${coreutils:location}/bin/basename
@@ -687,7 +687,7 @@ python-memcached = 1.53
 qrcode = 5.1
 restkit = 4.2.2
 rtjp-eventlet = 0.3.2
-slapos.recipe.template = 2.5
+slapos.recipe.template = 2.6
 slapos.toolbox = 0.45.1
 smmap = 0.8.3
 socketpool = 0.5.3

stack/erp5/instance-erp5.cfg.in

-{% set frontend_dict = slapparameter_dict.get('frontend', {}) %}
+{% macro assert(x) %}{{ ("",)[not x] }}{% endmacro -%}
+{% set sla_dict = {} -%}
+{% for sla, ref_list in slapparameter_dict.get('sla-dict', {}).iteritems() -%}
+{%   do sla_dict.update(dict.fromkeys(ref_list, sla)) -%}
+{% endfor -%}
+{% set frontend_dict = slapparameter_dict.get('frontend', {}) -%}
 {% set has_frontend = frontend_dict.get('software-url', '') != '' -%}
 {% set site_id = slapparameter_dict.get('site-id', 'erp5') -%}
 {% set inituser_login = slapparameter_dict.get('inituser-login', 'zope') -%}
@@ -13,6 +18,14 @@ computer-id = ${slap-connection:computer-id}
 partition-id = ${slap-connection:partition-id}
 config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
 
+{% macro sla(name, required=False) -%}
+{% if required or name in sla_dict -%}
+{%  for k, (v,) in urlparse.parse_qs(slap_dict.pop(name), strict_parsing=1).iteritems() -%}
+sla-{{ k }} = {{ dumps(v) }}
+{%  endfor -%}
+{% endif -%}
+{% endmacro -%}
+
 {% macro request(name, software_type, config_key, config, ret={'url': True}) -%}
 {% do config.update(slapparameter_dict.get(config_key, {})) -%}
 {% set section = 'request-' ~ name -%}
@@ -20,15 +33,15 @@ config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
 < = request-common
 name = {{ name }}
 software-type = {{ software_type }}
-return = {{ ret.keys() | join(' ') }}
-{% for ret, publish in ret.items() -%}
+return = {{ ' '.join(ret) }}
+{% for ret, publish in ret.iteritems() -%}
 {%   if publish -%}
 {%     do publish_dict.__setitem__(name ~ '-' ~ ret, '${' ~ section ~ ':connection-' ~ ret ~ '}')%}
 {%   endif -%}
 {% endfor -%}
-sla-computer_guid = {{ dumps(slapparameter_dict.get(config_key + '-computer-guid', computer_id)) }}
-{% for option, value in config.items() -%}
-config-{{ option }} = {{ dumps(value) }}
+{{ sla(name) }}
+{% for k, v in config.iteritems() -%}
+config-{{ k }} = {{ dumps(v) }}
 {% endfor -%}
 {% endmacro -%}
 
@@ -36,14 +49,26 @@ config-{{ option }} = {{ dumps(value) }}
 {{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}) }}
 {{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}) }}
 {{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}, {'database-list': True, 'test-database-list': True}) }}
-{# Fail early if an unexpected value is provided -#}
-{% set zodb_type = slapparameter_dict.get('zodb-software-type') -%}
-{% set zodb_extern = slapparameter_dict.get('zodb-extern') -%}
-{% if {'zeo': 1, None: 0}[zodb_type] -%}
-{{   request('zodb', 'zodb-' ~ zodb_type, 'zodb', {'tcpv4-port': 2100, 'zodb-dict': {'root': {}}}, {'zodb-storage-type': False, 'zodb-dict': False, 'tidstorage-ip': False, 'tidstorage-port': False}) }}
-{% else -%}
-{% do zodb_extern[0] %}
-{% endif -%}
+
+{# ZODB -#}
+{% set zodb_dict = {} -%}
+{% set storage_dict = {} -%}
+{% set mountpoints = set() -%}
+{% for zodb in slapparameter_dict.get('zodb') or ({'type': 'zeo', 'server': {}},) -%}
+{%   do mountpoints.add(zodb.setdefault('mount-point', '/')) -%}
+{%   set name = zodb.pop('name', 'root') -%}
+{%   do assert(name not in zodb_dict) -%}
+{%   do zodb_dict.__setitem__(name, zodb) -%}
+{%   if 'server' in zodb -%}
+{%     do storage_dict.setdefault(zodb['type'], {}).__setitem__(name, zodb.pop('server')) -%}
+{%   endif -%}
+{% endfor -%}
+{% do assert(len(mountpoints) == len(zodb_dict)) %}
+{% for server_type, server_dict in storage_dict.iteritems() -%}
+{%   do assert(server_type == 'zeo') -%}
+{# BBB: for compatibility, keep 'zodb' as partition_reference for ZEO -#}
+{{   request('zodb', 'zodb-' ~ server_type, 'zodb-' ~ server_type, {'tcpv4-port': 2100, 'zodb-dict': server_dict}, dict.fromkeys(('storage-dict', 'tidstorage-ip', 'tidstorage-port'))) }}
+{% endfor -%}
 
 [inituser-password]
 {% set inituser_password = slapparameter_dict.get('inituser-password') -%}
@@ -81,14 +106,11 @@ config-mysql-url-list = ${request-mariadb:connection-database-list}
 config-site-id = {{ dumps(site_id) }}
 config-smtp-url = {{ dumps(slapparameter_dict.get('smtp-url', 'smtp://localhost:25/')) }}
 config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
-{% if zodb_type -%}
+config-zodb-dict = {{ dumps(zodb_dict) }}
+{% if storage_dict -%}
+config-zodb-zeo = ${request-zodb:connection-storage-dict}
 config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
 config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
-config-zodb-dict = ${request-zodb:connection-zodb-dict}
-config-zodb-storage-type = ${request-zodb:connection-zodb-storage-type}
-{% endif -%}
-{% if zodb_extern -%}
-config-zodb-extern = {{ dumps(zodb_extern) }}
 {% endif -%}
 software-type = zope
 
@@ -100,6 +122,7 @@ software-type = zope
 [{{ section_name }}]
 < = request-zope-base
 name = {{ partition_name }}
+{{ sla(partition_name) }}
 config-name = {{ dumps(custom_name) }}
 config-instance-count = {{ dumps(zope_parameter_dict.get('instance-count', 1)) }}
 config-thread-amount = {{ dumps(zope_parameter_dict.get('thread-amount', 1)) }}
@@ -108,7 +131,6 @@ config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longreque
 config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }}
 config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
 config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }}
-sla-computer_guid = {{ dumps(zope_parameter_dict.get('computer-guid', computer_id)) }}
 {% endfor -%}
 
 {# We need to concatenate lists that we cannot read as lists, so this gets hairy. -#}
@@ -138,7 +160,7 @@ config-url = ${request-balancer:{{ family_name }}-v6}
 < = request-common
 name = balancer
 software-type = balancer
-sla-computer_guid = {{ dumps(slapparameter_dict.get('balancer-computer-guid', computer_id)) }}
+{{ sla('balancer') }}
 return =
 {%- for family in zope_family_dict %}
   {{ family }}
@@ -159,7 +181,7 @@ config-ssl = {{ dumps(balancer_dict.get('ssl', {})) }}
 < = request-common
 software-url = {{ dumps(frontend_dict['software-url']) }}
 software-type = {{ dumps(frontend_dict.get('software-type', 'RootSoftwareInstance')) }}
-sla-instance_guid = {{ dumps(frontend_dict['instance-guid']) }}
+{{ sla('frontend', True) }}
 slave = true
 {% set config_dict = {
   'type': 'zope',
@@ -188,3 +210,5 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-
 
 [buildout]
 parts = publish
+
+{{- assert(not sla_dict) }}

stack/erp5/instance-zeo.cfg.in

 {% set next_port = slapparameter_dict['tcpv4-port'] -%}
+{% set ipv4 = (ipv4_set | list)[0] -%}
+{% set backup_periodicity = slapparameter_dict.get('backup-periodicity', 'daily') -%}
+{% set default_tidstorage_timestamp_path = slapparameter_dict.get('tidstorage-timestamp-path', buildout_directory ~ '/srv/backup/tidstorage') -%}
 {% set part_list = [] -%}
+{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
 {% set storage_dict = {} -%}
-{% set zodb_dict = {} -%}
-{% set tidstorage_dict = slapparameter_dict.get('backup', {}) -%}
-{% set known_tid_storage_identifier_dict = {} -%}
 {% set default_zodb_path = buildout_directory ~ '/srv/zodb' -%}
-{% set zodb_backup_path = buildout_directory ~ '/srv/backup/zodb' -%}
-{% set default_tidstorage_timestamp_path = buildout_directory ~ '/srv/backup/tidstorage' -%}
-{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
-{% set ipv4 = (ipv4_set | list)[0] -%}
-{% set tidstorage_ip = ipv4 -%}
-{% set tidstorage_port = 6001 -%}
+{% set default_backup_path = buildout_directory ~ '/srv/backup/zodb' -%}
 {% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
 
 [logrotate-base]
@@ -27,50 +23,33 @@ wrapper-path = ${directory:services}/${:base-name}
 binary-path = {{ bin_directory }}/runzeo
 ip = {{ ipv4 }}
 
-{% set backup_directory_list = [] -%}
-{% for export_id, mountpoint_dict in slapparameter_dict['zodb-dict'].items() -%}
-{%   do storage_dict.setdefault(
-       mountpoint_dict.get('storage-family', 'default').lower(), []
-     ).append((
-       export_id,
-       mountpoint_dict.get('mount-point', '/'),
-       mountpoint_dict.get('cache-size', -1),
-       mountpoint_dict.get('storage-dict', {}),
-     )) -%}
+{% set known_tid_storage_identifier_dict = {} -%}
+{% set zodb_dict = {} -%}
+{% for name, zodb in slapparameter_dict['zodb-dict'].iteritems() -%}
+{%   do zodb_dict.setdefault(zodb.get('family', 'default').lower(), []).append((name, zodb)) -%}
 {% endfor -%}
-{% for storage_family, export_list in storage_dict.items() -%}
-{%   set known_tid_storage_identifier_host = ((ipv4, next_port), ) -%}
-{%   for export_id, mount_point, cache_size, export_storage_dict in export_list -%}
-{%     do export_storage_dict.__setitem__('path', export_storage_dict.get('path', '%(zodb)s/%(name)s.fs') % {'zodb': default_zodb_path, 'name': export_id}) -%}
-{%     set client_dict = export_storage_dict.get('client', {}).copy() -%}
-{%     do client_dict.__setitem__('storage', export_id) -%}
-{#     XXX: I would like to raise if export_id is present in zodb_dict -#}
-{%     do zodb_dict.__setitem__(export_id, [
-         mount_point,
-         cache_size,
-         client_dict,
-       ]) -%}
-{%     if tidstorage_dict != None -%}
-{%       set backup_directory = tidstorage_dict.get('zodb-dict', {}).get(export_id, '%(backup)s/%(name)s') % {'backup': zodb_backup_path, 'name': export_id} -%}
-{%       do backup_directory_list.append(backup_directory) -%}
-{%       do known_tid_storage_identifier_dict.__setitem__(
-           json_module.dumps((known_tid_storage_identifier_host, export_id)), (
-             export_storage_dict['path'],
-             backup_directory,
-             mount_point,
-           ),
-         ) -%}
-{%     endif -%}
+{% set tidstorage_port = next_port + len(zodb_dict) -%}
+{% for family, zodb in zodb_dict.iteritems() -%}
+{%   set storage_list = [] -%}
+{%   set known_tid_storage_identifier_host = (ipv4, next_port), -%}
+{%   for name, zodb in zodb -%}
+{%     do storage_dict.__setitem__(name, {'server': ipv4 ~ ':' ~ next_port, 'storage': name}) %}
+{%     set path = zodb.get('path', '%(zodb)s/%(name)s.fs') % {'zodb': default_zodb_path, 'name': name} -%}
+{%     do storage_list.append((name, path)) -%}
+{%     set backup_directory = zodb.get('backup', '%(backup)s/%(name)s') % {'backup': default_backup_path, 'name': name} -%}
+{# BBB: No mount-point specified because they're meaningless for ZEO and
+        TIDStorage. Pass '' for compatibility, and not None
+        because this would disable TIDStorage bootstrapping. -#}
+{%     do known_tid_storage_identifier_dict.__setitem__(json_module.dumps(
+         (known_tid_storage_identifier_host, name)),
+         (path, backup_directory, '')) -%}
 {%   endfor -%}
-{%   set zeo_section_name = 'zeo-' ~ storage_family %}
+
+{%   set zeo_section_name = 'zeo-' ~ family %}
 [{{ zeo_section_name }}]
 < = zeo-base
-base-name = zeo-{{ storage_family }}
+base-name = zeo-{{ family }}
 port = {{ next_port }}
-{%   set storage_list = [] -%}
-{%   for storage_name, _, _, export_storage_dict in export_list -%}
-{%     do storage_list.append((storage_name, export_storage_dict['path'])) -%}
-{%   endfor -%}
 storage = {{ dumps(storage_list) }}
 
 [{{ section("logrotate-" ~ zeo_section_name) }}]
@@ -82,38 +61,39 @@ post = {{ bin_directory }}/slapos-kill --pidfile {{ "${" ~ zeo_section_name ~ ":
 [{{ section(zeo_section_name ~ "-promise") }}]
 recipe = slapos.cookbook:check_port_listening
 hostname = {{ "${" ~ zeo_section_name ~ ":ip}" }}
-port = {{ next_port }}
-path = ${directory:promises}/zeo-{{ storage_family }}
+port = {{ "${" ~ zeo_section_name ~ ":port}" }}
+path = ${directory:promises}/zeo-{{ family }}
 
-{%   set address = ipv4 ~ ':' ~ next_port -%}
-{%   for export in export_list -%}
-{%     do zodb_dict[export[0]][2].__setitem__('server', address) -%}
-{%   endfor -%}
 {%   set next_port = next_port + 1 -%}
 {% endfor -%}
 
-{% if tidstorage_dict != None -%}
+{% if backup_periodicity == 'never' -%}
+{%   set known_tid_storage_identifier_dict = () %}
+{% else -%}
 [tidstorage]
 recipe = slapos.cookbook:tidstorage
 known-tid-storage-identifier-dict = {{ dumps(known_tid_storage_identifier_dict) }}
 configuration-path = ${directory:etc}/tidstorage.py
-ip = {{ tidstorage_ip }}
+ip = {{ ipv4 }}
 port = {{ tidstorage_port }}
-timestamp-file-path = {{ dumps(tidstorage_dict.get('tidstorage-timestamp-path', '%(backup)s/repozo_tidstorage_timestamp.log' % {'backup': default_tidstorage_timestamp_path})) }}
-logfile-name = ${directory:log}/tidstorage.log
+timestamp-file-path = {{ dumps(slapparameter_dict.get('tidstorage-timestamp-path', '%(backup)s/repozo_tidstorage_timestamp.log') % {'backup': default_tidstorage_timestamp_path}) }}
+{# BBB: recipe requires logfile-name for nothing because tidstorage runs in foreground mode -#}
+logfile-name =
 pidfile-name = ${directory:run}/tidstorage.pid
 status-file = ${directory:tidstorage}/tidstorage.tid
 tidstorage-repozo-binary = {{ bin_directory }}/tidstorage_repozo
 tidstoraged-binary = {{ bin_directory }}/tidstoraged
 repozo-binary = {{ bin_directory }}/repozo
-tidstorage-wrapper = ${directory:services}/tidstoraged
 repozo-wrapper = ${buildout:bin-directory}/tidstorage-repozo
+{% if len(known_tid_storage_identifier_dict) > 1 -%}
+tidstorage-wrapper = ${directory:services}/tidstoraged
 
 [{{ section("promise-tidstorage") }}]
 recipe = slapos.cookbook:check_port_listening
 hostname = ${tidstorage:ip}
 port = ${tidstorage:port}
 path = ${directory:promises}/tidstorage
+{% endif -%}
 
 [{{ section("cron-entry-tidstorage-backup") }}]
 # TODO:
@@ -122,40 +102,20 @@ path = ${directory:promises}/tidstorage
 recipe = slapos.cookbook:cron.d
 cron-entries = ${cron:cron-entries}
 name = tidstorage
-time = {{ dumps(tidstorage_dict.get('periodicity', 'daily')) }}
+time = {{ dumps(backup_periodicity) }}
 command = ${tidstorage:repozo-wrapper}
-
-{% if backup_directory_list -%}
-[{{ section("cron-entry-purge-repozo-fsindex") }}]
-recipe = slapos.cookbook:cron.d
-cron-entries = ${cron:cron-entries}
-name = purge-repozo-fsindex
-time = ${cron-entry-tidstorage-backup:time}
-command = ${cron-script-purge-repozo-fsindex:wrapper-path}
-
-[cron-script-purge-repozo-fsindex]
-recipe = slapos.cookbook:wrapper
-command-line = "{{ parameter_dict['findutils-location'] }}/bin/find" {% for backup_directory in backup_directory_list %}'{{ backup_directory.replace("\\", "\\\\").replace("'", "\\'") }}' {% endfor %}-name "????-??-??-??-??-??.index" -daystart -mtime +2 -delete
-wrapper-path = ${directory:bin}/cron-purge-repozo-fsindex
-
-{% endif -%}
-[{{ section("logrotate-tidstorage") }}]
-< = logrotate-base
-name = tidstorage
-log = ${tidstorage:logfile-name}
-post = {{ bin_directory }}/slapos-kill --pidfile ${tidstorage:pidfile-name} -s HUP
-{% else -%}
-[tidstorage]
-ip =
-port =
 {% endif -%}
 
 [publish]
 recipe = slapos.cookbook:publish.serialised
-zodb-storage-type = zeoclient
-zodb-dict = {{ dumps(zodb_dict) }}
+storage-dict = {{ dumps(storage_dict) }}
+{% if len(known_tid_storage_identifier_dict) > 1 -%}
 tidstorage-ip = ${tidstorage:ip}
 tidstorage-port = ${tidstorage:port}
+{% else -%}
+tidstorage-ip =
+tidstorage-port =
+{% endif -%}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
@@ -166,7 +126,7 @@ promises = ${:etc}/promise
 var = ${buildout:directory}/var
 log = ${:var}/log
 run = ${:var}/run
-backup-zodb = {{ zodb_backup_path }}
+backup-zodb = {{ default_backup_path }}
 zodb = {{ default_zodb_path }}
 tidstorage = {{ default_tidstorage_timestamp_path }}
 

stack/erp5/instance-zope.cfg.in

 {% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
 {% set next_port = slapparameter_dict['port-base'] -%}
 {% set site_id = slapparameter_dict['site-id'] -%}
-{% set storage_type = slapparameter_dict['zodb-storage-type'] -%}
+{% set zodb_dict = slapparameter_dict['zodb-dict'] -%}
 {% set node_id_base = slapparameter_dict['name'] -%}
 {% set part_list = [] -%}
 {% set publish_list = [] -%}
-{% set zodb_dict = slapparameter_dict.get('zodb-dict', {}) -%}
 {% set longrequest_logger_base_path = buildout_directory ~ '/var/log/longrequest_logger_' -%}
 {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
 {% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
@@ -110,16 +109,6 @@ ipv4 = {{ ipv4 }}
 ipv6 = {{ ipv6 }}
 {% endif -%}
 
-{% if slapparameter_dict.get('tidstorage-ip') -%}
-[tidstorage]
-ipv4 = {{ dumps(slapparameter_dict['tidstorage-ip']) }}
-ipv4-port = {{ dumps(slapparameter_dict['tidstorage-port']) }}
-{% else -%}
-[tidstorage]
-ipv4 =
-ipv4-port =
-{% endif -%}
-
 {% set hosts_dict = {} -%}
 {% for alias, url in (
     ('erp5-memcached-volatile', slapparameter_dict['memcached-url']),
@@ -131,14 +120,11 @@ ipv4-port =
        urlparse.urlparse(url).hostname,
      )  -%}
 {%- endfor %}
-{# jinja2 does not support enumerate... -#}
-{% set catalog_counter = 0 %}
-{% for url in slapparameter_dict['mysql-url-list'] -%}
+{% for i, url in enumerate(slapparameter_dict['mysql-url-list']) -%}
 {%   do hosts_dict.__setitem__(
-       'erp5-catalog-' ~ catalog_counter,
+       'erp5-catalog-' ~ i,
        urlparse.urlparse(url).hostname,
      ) -%}
-{%   set catalog_counter = catalog_counter + 1 -%}
 {%- endfor %}
 {% do hosts_dict.update(slapparameter_dict['hosts-dict']) -%}
 [hosts-parameter]
@@ -182,20 +168,19 @@ bt5-repository =
 [zope-conf-parameter-base]
 ip = {{ ipv4 }}
 site-id = {{ site_id }}
-{% set zodb_list = slapparameter_dict.get('zodb-extern', []) -%}
-{% for key, (mount_point, cache_size, storage_dict) in zodb_dict.items() -%}
-{%   do zodb_list.append({'name': key, 'mount-point': mount_point, 'storage-type': storage_type, 'storage-dict': storage_dict}) -%}
-{%   if cache_size >= 0 -%}
-{%     do zodb_list[-1].__setitem__('cache-size', cache_size) -%}
-{%   endif -%}
+{% set storage_dict = {'neo': {}, 'zeo': slapparameter_dict.get('zodb-zeo', {})} -%}
+{% for name, zodb in zodb_dict.iteritems() -%}
+{%   do zodb.setdefault('storage-dict', {}).update(storage_dict[zodb['type']].get(name, {})) -%}
 {% endfor -%}
 developer-list = {{ dumps(slapparameter_dict['developer-list']) }}
 instance = ${directory:instance}
 instance-products = ${directory:instance-products}
 deadlock-path = /manage_debug_threads
 deadlock-debugger-password = {{ dumps(slapparameter_dict['deadlock-debugger-password']) }}
-tidstorage-ip = ${tidstorage:ipv4}
-tidstorage-port = ${tidstorage:ipv4-port}
+{% if slapparameter_dict.get('tidstorage-ip') -%}
+tidstorage-ip = {{ dumps(slapparameter_dict['tidstorage-ip']) }}
+tidstorage-port = {{ dumps(slapparameter_dict['tidstorage-port']) }}
+{% endif -%}
 promise-path = ${erp5-promise:promise-path}
 {% set thread_amount = slapparameter_dict['thread-amount'] -%}
 thread-amount = {{ thread_amount }}
@@ -237,17 +222,17 @@ event-log = ${directory:log}/{{ name }}-event.log
 z2-log = ${directory:log}/{{ name }}-Z2.log
 node-id = {{ dumps(node_id_base ~ '-' ~ index) }}
 {% set log_list = [] -%}
-{% set import_set = {} -%}
-{% for zodb in zodb_list -%}
-{%   if zodb.setdefault('storage-type', 'NEOStorage') == 'NEOStorage' -%}
-{%     do import_set.__setitem__('neo.client', None) -%}
-{%     set log = buildout_directory ~ '/var/log/' ~ name ~ '-neo-' ~ zodb.get('name', 'main') ~ '.log' -%}
+{% set import_set = set() -%}
+{% for db_name, zodb in zodb_dict.iteritems() -%}
+{%   if zodb['type'] == 'neo' -%}
+{%     do import_set.add('neo.client') -%}
+{%     set log = buildout_directory ~ '/var/log/' ~ name ~ '-neo-' ~ db_name ~ '.log' -%}
 {%     do log_list.append(log) -%}
-{%     do zodb.setdefault('storage-dict', {}).__setitem__('logfile', log) -%}
+{%     do zodb['storage-dict'].__setitem__('logfile', log) -%}
 {%   endif -%}
 {% endfor -%}
-import-list = {{ dumps(import_set.keys()) }}
-zodb-list = {{ dumps(zodb_list) }}
+import-list = {{ dumps(list(import_set)) }}
+zodb-dict = {{ dumps(zodb_dict) }}
 
 [{{ conf_name }}]
 < = zope-conf-base

stack/erp5/instance.cfg.in

@@ -71,6 +71,7 @@ filename = instance-erp5.cfg
 extensions = jinja2.ext.do
 extra-context =
     key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
+    import urlparse urlparse
 
 [dynamic-template-balancer-parameters]
 apache = {{ apache_location }}
@@ -92,7 +93,6 @@ extra-context =
 
 [dynamic-template-zeo-parameters]
 buildout-bin-directory = {{ buildout_bin_directory }}
-findutils-location = {{ findutils_location }}
 
 [dynamic-template-zeo]
 < = jinja2-template-base

stack/erp5/zope.conf.in

@@ -42,10 +42,9 @@ products {{ parameter_dict['instance-products'] }}
   secret {{ parameter_dict['deadlock-debugger-password'] }}
 </product-config>
 
-{% set tidstorage_ip = parameter_dict['tidstorage-ip'] -%}
-{% if tidstorage_ip -%}
+{% if 'tidstorage-ip' in parameter_dict -%}
 <product-config TIDStorage>
-  backend-ip {{ tidstorage_ip }}
+  backend-ip {{ parameter_dict['tidstorage-ip'] }}
   backend-port {{ parameter_dict['tidstorage-port'] }}
 </product-config>
 
@@ -104,11 +103,11 @@ products {{ parameter_dict['instance-products'] }}
 {% for m in parameter_dict['import-list'] -%}
 %import {{ m }}
 {% endfor -%}
-{% for zodb_dict in parameter_dict['zodb-list'] %}
-<zodb_db {{ zodb_dict.pop('name', 'main') }}>
-{%- set storage_type = zodb_dict.pop('storage-type', 'NEOStorage') %}
-{%- set storage_dict = zodb_dict.pop('storage-dict', {}) %}
-  mount-point {{ zodb_dict.pop('mount-point', '/') }}
+{% set type_dict = {'neo': 'NEOStorage', 'zeo': 'zeoclient'} %}
+{% for name, zodb_dict in parameter_dict['zodb-dict'].iteritems() %}
+<zodb_db {{ name }}>
+{%- set storage_type = type_dict[zodb_dict.pop('type')] %}
+{%- set storage_dict = zodb_dict.pop('storage-dict') %}
 {%- for key, value in zodb_dict.iteritems() %}
   {{ key }} {{ value }}
 {%-   endfor %}