NEO/ERP5: new schema of parameters, with overriding mechanism

The main change is a new and unique way to override parameters depending on where these parameters are used:
- for NEO, the goal is to avoid a huge `node_list` value (in particular with MyRocks settings), whereas most of the time all values in the list are the same
- for ERP5, we need a way to define different ZODB cache settings depending on the zope id (e.g. huge cache for some specialized zopes, a small cache for the validation node, etc. whereas currently a huge cache for all nodes would waste a lot of RAM)

For any `key=value` parameter that can be overridden, overriding is done as follows:
- `key` defines the default value (not required if the SR already defines one)
- `key!=[[pattern, value],...]`: the SR takes the value of the first 2-tuple whose *some reference* matches the pattern

For NEO:
- new `node_count` and `node` parameters
- `node!` can be used to override `node`: the *reference* is the partition reference (`node-<i>` for standalone NEO, and `neo-<i>` for ERP5)
- `node_list` is deprecated (if still passed, the new parameters must not be present)

For ERP5 ZODB settings, parameters can be overridden at both:
- zodb level (e.g. `pool-size`, `cache-size`... oh, I've just discovered there exists `cache-size-bytes`)
- storage level (e.g. `compress`, `cache-size`, etc. in the case of NEO)

Here, the *reference* is the zope id (e.g. `actitivies-0`)

The `pattern` is a Python regex that must match the whole reference.

Another not-so-small change is that the schema is fixed to only require NEO `cluster` parameter when instanciating a standalone NEO.

And a few fixes.

/reviewed-on nexedi/slapos!661

software/erp5/instance-erp5-input-schema.json

@@ -257,7 +257,7 @@
       "type": "object"
     },
     "zodb": {
-      "description": "Zope Object DataBase mountpoints. See https://github.com/zopefoundation/ZODB/blob/3.10/src/ZODB/component.xml for extra options.",
+      "description": "Zope Object DataBase mountpoints. See https://github.com/zopefoundation/ZODB/blob/4/src/ZODB/component.xml for extra options.",
       "items": {
         "required": [
           "type"
@@ -282,8 +282,13 @@
                 "type": "boolean"
               }
             },
+            "patternProperties": {
+              ".!$": {
+                "$ref": "#/properties/zodb/items/patternProperties/.!$"
+              }
+            },
             "additionalProperties": {
-              "type": "string"
+              "$ref": "#/properties/zodb/items/additionalProperties"
             },
             "type": "object"
           },
@@ -313,14 +318,33 @@
               },
               "server": {
                 "description": "Instantiate a server. If missing, 'storage-dict' must contain the necessary properties to mount the ZODB. Partitions references are 'neo-0', 'neo-1', ...",
-                "$ref": "../neoppod/instance-neo-input-schema.json"
+                "$ref": "../neoppod/instance-neo-input-schema.json#/definitions/neo-cluster"
               }
             }
           }
         ],
-        "additionalProperties": {
+        "patternProperties": {
+          ".!$": {
+            "description": "Override with the value of the first item whose zope id matches against the pattern.",
+            "items": {
+              "items": [
+                {
+                  "description": "Override pattern (Python regular expression).",
                   "type": "string"
                 },
+                {
+                  "description": "Override value (parameter for maching nodes).",
+                  "type": ["integer", "string"]
+                }
+              ],
+              "type": "array"
+            },
+            "type": "array"
+          }
+        },
+        "additionalProperties": {
+          "type": ["integer", "string"]
+        },
         "type": "object"
       },
       "type": "array"

software/erp5/instance-mariadb-schema.json

 {
   "$schema": "http://json-schema.org/draft-04/schema#",
-  "required": [
-    "tcpv4-port"
-  ],
   "type": "object",
   "properties": {
     "tcpv4-port": {

software/erp5/test/test/__init__.py

@@ -46,3 +46,13 @@ class ERP5InstanceTestCase(SlapOSInstanceTestCase):
     """Return the output paramters from the root partition"""
     return json.loads(
         self.computer_partition.getConnectionParameterDict()['_'])
+
+  def getComputerPartition(self, partition_reference):
+    for computer_partition in self.slap.computer.getComputerPartitionList():
+      if partition_reference == computer_partition.getInstanceParameter(
+          'instance_title'):
+        return computer_partition
+
+  def getComputerPartitionPath(self, partition_reference):
+    partition_id = self.getComputerPartition(partition_reference).getId()
+    return os.path.join(self.slap._instance_root, partition_id)

software/erp5/test/test/test_erp5.py

@@ -195,8 +195,8 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
     """
     # self.computer_partition_root_path is the path of root partition.
     # we want to assert that no scripts exist in any partition.
-    bin_programs = [os.path.basename(path) for path in
-      glob.glob("{}/../*/bin/*".format(self.computer_partition_root_path))]
+    bin_programs = map(os.path.basename,
+      glob.glob(self.computer_partition_root_path + "/../*/bin/*"))
 
     self.assertTrue(bin_programs) # just to check the glob was correct.
     self.assertNotIn('runUnitTest', bin_programs)
@@ -210,8 +210,99 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
     apache_process = psutil.Process(process_info['pid'])
     self.assertEqual(
         sorted([socket.AF_INET, socket.AF_INET6]),
-        sorted([
+        sorted(
             c.family
             for c in apache_process.connections()
             if c.status == 'LISTEN'
-        ]))
+        ))
+
+class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+  """Test override zope node parameters
+  """
+  __partition_reference__ = 'override'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    # The following example includes the most commonly used options,
+    # but not necessarily in a meaningful way.
+    return {'_': json.dumps({
+      "zodb": [{
+        "type": "zeo",
+        "server": {},
+        "cache-size-bytes": "20MB",
+        "cache-size-bytes!": [
+          ("bb-0", 1<<20),
+          ("bb-.*", "500MB"),
+        ],
+        "pool-timeout": "10m",
+        "storage-dict": {
+          "cache-size!": [
+            ("a-.*", "50MB"),
+          ],
+        },
+      }],
+      "zope-partition-dict": {
+          "a": {
+              "instance-count": 3,
+          },
+          "bb": {
+              "instance-count": 5,
+              "port-base": 2300,
+          },
+      },
+    })}
+
+  def test_zope_conf(self):
+    zeo_addr = json.loads(
+        self.getComputerPartition('zodb').getConnectionParameter('_')
+      )["storage-dict"]["root"]["server"]
+
+    def checkParameter(line, kw):
+      k, v = line.split()
+      self.assertFalse(k.endswith('!'), k)
+      try:
+        expected = kw.pop(k)
+      except KeyError:
+        if k == 'server':
+          return
+      self.assertIsNotNone(expected)
+      self.assertEqual(str(expected), v)
+
+    def checkConf(zodb, storage):
+      zodb["mount-point"] = "/"
+      zodb["pool-size"] = 4
+      zodb["pool-timeout"] = "10m"
+      storage["storage"] = "root"
+      storage["server"] = zeo_addr
+      with open('%s/etc/zope-%s.conf' % (partition, zope)) as f:
+        conf = map(str.strip, f.readlines())
+      i = conf.index("<zodb_db root>") + 1
+      conf = iter(conf[i:conf.index("</zodb_db>", i)])
+      for line in conf:
+        if line == '<zeoclient>':
+          for line in conf:
+            if line == '</zeoclient>':
+              break
+            checkParameter(line, storage)
+          for k, v in storage.iteritems():
+            self.assertIsNone(v, k)
+          del storage
+        else:
+          checkParameter(line, zodb)
+      for k, v in zodb.iteritems():
+        self.assertIsNone(v, k)
+
+    partition = self.getComputerPartitionPath('zope-a')
+    for zope in xrange(3):
+      checkConf({
+          "cache-size-bytes": "20MB",
+        }, {
+          "cache-size": "50MB",
+        })
+    partition = self.getComputerPartitionPath('zope-bb')
+    for zope in xrange(5):
+      checkConf({
+          "cache-size-bytes": "500MB" if zope else 1<<20,
+        }, {
+          "cache-size": None,
+        })

software/neoppod/buildout.hash.cfg

@@ -14,11 +14,11 @@
 # not need these here).
 [instance-common]
 filename = instance-common.cfg.in
-md5sum = 0a3a54fcc7be0bbd63cbd64f006ceebc
+md5sum = 80599fcc6e5d07270d7900aebfd62139
 
 [root-common]
 filename = root-common.cfg.in
-md5sum = ccc6e33412259415ec6c3452d37b77cc
+md5sum = c03fbfc9df9edc1ef60be970e0627c5e
 
 [instance-neo-admin]
 filename = instance-neo-admin.cfg.in
@@ -38,7 +38,7 @@ md5sum = 9f6f8f2b5f4cb0d97d50ffc1d3837e2f
 
 [template-neo]
 filename = instance.cfg.in
-md5sum = 83dc9faca482b2ddbd3d5fa968af7c33
+md5sum = 9e63e16eda75e73ad4ffb50afde0505d
 
 [cluster]
 filename = cluster.cfg.in

software/neoppod/instance-common.cfg.in

@@ -15,7 +15,9 @@ cert = ${slap-connection:cert-file}
 recipe = slapos.recipe.template:jinja2
 filename = ${:_buildout_section_name_}.cfg
 rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
-extensions = jinja2.ext.do
+extensions =
+    jinja2.ext.do
+    jinja2.ext.loopcontrols
 extra-context =
 context =
     key ipv4_set slap-configuration:ipv4

software/neoppod/instance-neo-input-schema.json

 {
-  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
   "description": "Parameters to instantiate a NEO cluster. See https://lab.nexedi.com/nexedi/neoppod/blob/master/neo.conf for more information.",
+  "definitions": {
+    "neo-cluster": {
       "additionalProperties": false,
-  "require": [
-    "cluster"
-  ],
       "properties": {
         "cluster": {
           "description": "Cluster unique identifier. Your last line of defense against mixing up NEO clusters and corrupting your data. Choose a unique value for each of your cluster. Space not allowed.",
@@ -72,10 +71,13 @@
         "_key": {
           "type": "string"
         },
-    "node-list": {
-      "description": "List of dictionaries containing parameters for each node.",
-      "items": {
-        "description": "Dictionary containing parameters required to configure individual nodes.",
+        "node-count": {
+          "description": "Number of nodes to deploy.",
+          "default": 1,
+          "type": "integer"
+        },
+        "node": {
+          "description": "Default node parameters.",
           "default": {},
           "additionalProperties": false,
           "properties": {
@@ -135,8 +137,41 @@
           },
           "type": "object"
         },
+        "node!": {
+          "description": "Node parameters are overridden by the value of the first item whose partition reference matches against the pattern.",
+          "items": {
+            "items": [
+              {
+                "description": "Override pattern (Python regular expression).",
+                "type": "string"
+              },
+              {
+                "allOf": [
+                  {
+                    "$ref": "#/definitions/neo-cluster/properties/node"
+                  },
+                  {
+                    "description": "Override value (parameters for maching nodes)."
+                  }
+                ]
+              }
+            ],
+            "type": "array"
+          },
           "type": "array"
         }
       },
       "type": "object"
+    }
+  },
+  "allOf": [
+    {
+      "$ref": "#/definitions/neo-cluster"
+    },
+    {
+      "required": [
+        "cluster"
+      ]
+    }
+  ]
 }

software/neoppod/instance.cfg.in

@@ -8,6 +8,7 @@ rendered = ${buildout:parts-directory}/${:_buildout_section_name_}.cfg
 <= jinja2-template-base
 template = {{ cluster }}
 extra-context =
+    import re re
     import urlparse urlparse
 import-list =
     rawfile root_common {{ root_common }}

software/neoppod/root-common.cfg.in

@@ -16,6 +16,20 @@ sla-computer_guid = ${slap-connection:computer-id}
 {% endif -%}
 {% endmacro -%}
 
+{% macro apply_overrides(dict_, reference) -%}
+{% for key in list(dict_) -%}
+{%   if key.endswith('!') -%}
+{%     for pattern, value in dict_.pop(key, ()) -%}
+{%       set m = re.match(pattern, reference) -%}
+{%       if m and m.group() == reference %}{# PY3: fullmatch -#}
+{%         do dict_.__setitem__(key[:-1], value) -%}
+{%         break -%}
+{%       endif -%}
+{%     endfor -%}
+{%   endif -%}
+{% endfor -%}
+{% endmacro -%}
+
 {% macro common_section() -%}
 [request-common-base]
 recipe = slapos.cookbook:request.serialised
@@ -52,7 +66,22 @@ config-ssl = {{ dumps((
 config-upstream-cluster = {{ dumps(parameter_dict.get('upstream-cluster', '')) }}
 config-upstream-masters = {{ dumps(parameter_dict.get('upstream-masters', '')) }}
 software-type = {{ software_type }}
-{% set node_list = parameter_dict.get('node-list', ({},)) -%}
+
+{% set node_list = parameter_dict.get('node-list') -%}
+{% if node_list == None -%}
+{%   set node_list = [] -%}
+{%   for i in range(parameter_dict.get('node-count', 1)) -%}
+{%     set x = parameter_dict.copy() -%}
+{%     do apply_overrides(x, prefix ~ i) -%}
+{%     do node_list.append(x.get('node', {})) -%}
+{%   endfor -%}
+{% else %}{# BBB -#}
+{%   do assert('node-count' not in parameter_dict) -%}
+{%   do assert('node' not in parameter_dict) -%}
+{%   do assert('node!' not in parameter_dict) -%}
+{% endif -%}
+{% do assert(node_list) -%}
+
 {% set storage_count = [] -%}
 {% for node in node_list -%}
 {%   do storage_count.append(node.get('storage-count', 1)) -%}

stack/erp5/buildout.hash.cfg

@@ -34,7 +34,7 @@ md5sum = e91c0fbd0df441884f7422fa7976053c
 
 [template-zope-conf]
 filename = zope.conf.in
-md5sum = 114e0ac43281b943931754ed317ebc36
+md5sum = 762897486b1e7e28b614224a9a577125
 
 [site-zcml]
 filename = site.zcml
@@ -70,7 +70,7 @@ md5sum = cc19560b9400cecbd23064d55c501eec
 
 [template]
 filename = instance.cfg.in
-md5sum = f6c6820f9b3653d0d5c29708606fc591
+md5sum = 520b6bf3461dddc9c8b862e50b14465d
 
 [monitor-template-dummy]
 filename = dummy.cfg
@@ -86,7 +86,7 @@ md5sum = 0648e38bd5d3a15bb9f93264932740b9
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = b1685783f4c93da918ccc83702559e6f
+md5sum = 8b4a15dca7e30ba5a792f1a9622216b0
 
 [template-balancer]
 filename = instance-balancer.cfg.in

stack/erp5/instance-zope.cfg.in

@@ -267,6 +267,11 @@ timerserver-interval = {{ dumps(timerserver_interval) }}
 [zope-conf-base]
 < = jinja2-template-base
 template = {{ parameter_dict['zope-conf-template'] }}
+extensions =
+    jinja2.ext.do
+    jinja2.ext.loopcontrols
+import-list =
+    rawfile root_common {{ root_common }}
 
 {% macro zope(
   index,
@@ -312,10 +317,10 @@ longrequest-logger-file =
 [{{ conf_name }}]
 < = zope-conf-base
 rendered = ${directory:etc}/{{ name }}.conf
-extensions = jinja2.ext.do
 context =
   section parameter_dict {{ conf_parameter_name }}
   import os os
+  import re re
 
 [{{ section(name) }}]
 < = runzope-base

stack/erp5/instance.cfg.in

@@ -46,6 +46,7 @@ extra-context =
     key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
     key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
     key openssl_location :openssl-location
+    import re re
     import urlparse urlparse
 import-list =
     file root_common context:root-common
@@ -111,6 +112,7 @@ template = {{ template_zope }}
 filename = instance-zope.cfg
 extra-context =
     key buildout_directory buildout:directory
+    key root_common context:root-common
     section parameter_dict dynamic-template-zope-parameters
     import urlparse urlparse
     import hashlib hashlib

stack/erp5/zope.conf.in

+{% set slapparameter_dict = {} %}{# dummy -#}
+{% import "root_common" as root_common with context -%}
+{% set node_id = parameter_dict['node-id'] -%}
 # Note: Environment is setup in running wrapper script, as zope.conf is read
 # too late for some components.
 %define INSTANCE {{ parameter_dict['instance'] }}
@@ -65,7 +68,7 @@ large-file-threshold {{ parameter_dict['large-file-threshold'] }}
 
 {% endif -%}
 <product-config CMFActivity>
-  node-id {{ parameter_dict['node-id'] }}
+  node-id {{ node_id }}
 </product-config>
 
 {% set timerserver_interval = parameter_dict['timerserver-interval'] -%}
@@ -145,10 +148,12 @@ large-file-threshold {{ parameter_dict['large-file-threshold'] }}
 <zodb_db {{ name }}>
 {%- set storage_type = type_dict[zodb_dict.pop('type')] %}
 {%- set storage_dict = zodb_dict.pop('storage-dict') %}
+{%- do root_common.apply_overrides(zodb_dict, node_id) %}
 {%- for key, value in zodb_dict.iteritems() %}
   {{ key }} {{ value }}
 {%- endfor %}
   <{{ storage_type }}>
+{%- do root_common.apply_overrides(storage_dict, node_id) %}
 {%- for key, value in storage_dict.iteritems() %}
     {{ key }} {{ value }}
 {%- endfor %}