Commit 036e518b authored by Julien Muchembled's avatar Julien Muchembled

ERP5: new parameters to override ZODB settings for any zope

parent 7ba3ff9e
...@@ -257,7 +257,7 @@ ...@@ -257,7 +257,7 @@
"type": "object" "type": "object"
}, },
"zodb": { "zodb": {
"description": "Zope Object DataBase mountpoints. See https://github.com/zopefoundation/ZODB/blob/3.10/src/ZODB/component.xml for extra options.", "description": "Zope Object DataBase mountpoints. See https://github.com/zopefoundation/ZODB/blob/4/src/ZODB/component.xml for extra options.",
"items": { "items": {
"required": [ "required": [
"type" "type"
...@@ -282,8 +282,13 @@ ...@@ -282,8 +282,13 @@
"type": "boolean" "type": "boolean"
} }
}, },
"patternProperties": {
".!$": {
"$ref": "#/properties/zodb/items/patternProperties/.!$"
}
},
"additionalProperties": { "additionalProperties": {
"type": "string" "$ref": "#/properties/zodb/items/additionalProperties"
}, },
"type": "object" "type": "object"
}, },
...@@ -318,9 +323,28 @@ ...@@ -318,9 +323,28 @@
} }
} }
], ],
"additionalProperties": { "patternProperties": {
".!$": {
"description": "Override with the value of the first item whose zope id matches against the pattern.",
"items": {
"items": [
{
"description": "Override pattern (Python regular expression).",
"type": "string" "type": "string"
}, },
{
"description": "Override value (parameter for maching nodes).",
"type": ["integer", "string"]
}
],
"type": "array"
},
"type": "array"
}
},
"additionalProperties": {
"type": ["integer", "string"]
},
"type": "object" "type": "object"
}, },
"type": "array" "type": "array"
......
...@@ -46,3 +46,13 @@ class ERP5InstanceTestCase(SlapOSInstanceTestCase): ...@@ -46,3 +46,13 @@ class ERP5InstanceTestCase(SlapOSInstanceTestCase):
"""Return the output paramters from the root partition""" """Return the output paramters from the root partition"""
return json.loads( return json.loads(
self.computer_partition.getConnectionParameterDict()['_']) self.computer_partition.getConnectionParameterDict()['_'])
def getComputerPartition(self, partition_reference):
for computer_partition in self.slap.computer.getComputerPartitionList():
if partition_reference == computer_partition.getInstanceParameter(
'instance_title'):
return computer_partition
def getComputerPartitionPath(self, partition_reference):
partition_id = self.getComputerPartition(partition_reference).getId()
return os.path.join(self.slap._instance_root, partition_id)
...@@ -195,8 +195,8 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix ...@@ -195,8 +195,8 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
""" """
# self.computer_partition_root_path is the path of root partition. # self.computer_partition_root_path is the path of root partition.
# we want to assert that no scripts exist in any partition. # we want to assert that no scripts exist in any partition.
bin_programs = [os.path.basename(path) for path in bin_programs = map(os.path.basename,
glob.glob("{}/../*/bin/*".format(self.computer_partition_root_path))] glob.glob(self.computer_partition_root_path + "/../*/bin/*"))
self.assertTrue(bin_programs) # just to check the glob was correct. self.assertTrue(bin_programs) # just to check the glob was correct.
self.assertNotIn('runUnitTest', bin_programs) self.assertNotIn('runUnitTest', bin_programs)
...@@ -210,8 +210,99 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix ...@@ -210,8 +210,99 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
apache_process = psutil.Process(process_info['pid']) apache_process = psutil.Process(process_info['pid'])
self.assertEqual( self.assertEqual(
sorted([socket.AF_INET, socket.AF_INET6]), sorted([socket.AF_INET, socket.AF_INET6]),
sorted([ sorted(
c.family c.family
for c in apache_process.connections() for c in apache_process.connections()
if c.status == 'LISTEN' if c.status == 'LISTEN'
])) ))
class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test override zope node parameters
"""
__partition_reference__ = 'override'
@classmethod
def getInstanceParameterDict(cls):
# The following example includes the most commonly used options,
# but not necessarily in a meaningful way.
return {'_': json.dumps({
"zodb": [{
"type": "zeo",
"server": {},
"cache-size-bytes": "20MB",
"cache-size-bytes!": [
("bb-0", 1<<20),
("bb-.*", "500MB"),
],
"pool-timeout": "10m",
"storage-dict": {
"cache-size!": [
("a-.*", "50MB"),
],
},
}],
"zope-partition-dict": {
"a": {
"instance-count": 3,
},
"bb": {
"instance-count": 5,
"port-base": 2300,
},
},
})}
def test_zope_conf(self):
zeo_addr = json.loads(
self.getComputerPartition('zodb').getConnectionParameter('_')
)["storage-dict"]["root"]["server"]
def checkParameter(line, kw):
k, v = line.split()
self.assertFalse(k.endswith('!'), k)
try:
expected = kw.pop(k)
except KeyError:
if k == 'server':
return
self.assertIsNotNone(expected)
self.assertEqual(str(expected), v)
def checkConf(zodb, storage):
zodb["mount-point"] = "/"
zodb["pool-size"] = 4
zodb["pool-timeout"] = "10m"
storage["storage"] = "root"
storage["server"] = zeo_addr
with open('%s/etc/zope-%s.conf' % (partition, zope)) as f:
conf = map(str.strip, f.readlines())
i = conf.index("<zodb_db root>") + 1
conf = iter(conf[i:conf.index("</zodb_db>", i)])
for line in conf:
if line == '<zeoclient>':
for line in conf:
if line == '</zeoclient>':
break
checkParameter(line, storage)
for k, v in storage.iteritems():
self.assertIsNone(v, k)
del storage
else:
checkParameter(line, zodb)
for k, v in zodb.iteritems():
self.assertIsNone(v, k)
partition = self.getComputerPartitionPath('zope-a')
for zope in xrange(3):
checkConf({
"cache-size-bytes": "20MB",
}, {
"cache-size": "50MB",
})
partition = self.getComputerPartitionPath('zope-bb')
for zope in xrange(5):
checkConf({
"cache-size-bytes": "500MB" if zope else 1<<20,
}, {
"cache-size": None,
})
...@@ -34,7 +34,7 @@ md5sum = e91c0fbd0df441884f7422fa7976053c ...@@ -34,7 +34,7 @@ md5sum = e91c0fbd0df441884f7422fa7976053c
[template-zope-conf] [template-zope-conf]
filename = zope.conf.in filename = zope.conf.in
md5sum = 114e0ac43281b943931754ed317ebc36 md5sum = 762897486b1e7e28b614224a9a577125
[site-zcml] [site-zcml]
filename = site.zcml filename = site.zcml
...@@ -70,7 +70,7 @@ md5sum = cc19560b9400cecbd23064d55c501eec ...@@ -70,7 +70,7 @@ md5sum = cc19560b9400cecbd23064d55c501eec
[template] [template]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = d53a7dbc374374b5c312e3f7e2c0a0ff md5sum = 520b6bf3461dddc9c8b862e50b14465d
[monitor-template-dummy] [monitor-template-dummy]
filename = dummy.cfg filename = dummy.cfg
...@@ -86,7 +86,7 @@ md5sum = 0648e38bd5d3a15bb9f93264932740b9 ...@@ -86,7 +86,7 @@ md5sum = 0648e38bd5d3a15bb9f93264932740b9
[template-zope] [template-zope]
filename = instance-zope.cfg.in filename = instance-zope.cfg.in
md5sum = b1685783f4c93da918ccc83702559e6f md5sum = 8b4a15dca7e30ba5a792f1a9622216b0
[template-balancer] [template-balancer]
filename = instance-balancer.cfg.in filename = instance-balancer.cfg.in
......
...@@ -267,6 +267,11 @@ timerserver-interval = {{ dumps(timerserver_interval) }} ...@@ -267,6 +267,11 @@ timerserver-interval = {{ dumps(timerserver_interval) }}
[zope-conf-base] [zope-conf-base]
< = jinja2-template-base < = jinja2-template-base
template = {{ parameter_dict['zope-conf-template'] }} template = {{ parameter_dict['zope-conf-template'] }}
extensions =
jinja2.ext.do
jinja2.ext.loopcontrols
import-list =
rawfile root_common {{ root_common }}
{% macro zope( {% macro zope(
index, index,
...@@ -312,10 +317,10 @@ longrequest-logger-file = ...@@ -312,10 +317,10 @@ longrequest-logger-file =
[{{ conf_name }}] [{{ conf_name }}]
< = zope-conf-base < = zope-conf-base
rendered = ${directory:etc}/{{ name }}.conf rendered = ${directory:etc}/{{ name }}.conf
extensions = jinja2.ext.do
context = context =
section parameter_dict {{ conf_parameter_name }} section parameter_dict {{ conf_parameter_name }}
import os os import os os
import re re
[{{ section(name) }}] [{{ section(name) }}]
< = runzope-base < = runzope-base
......
...@@ -112,6 +112,7 @@ template = {{ template_zope }} ...@@ -112,6 +112,7 @@ template = {{ template_zope }}
filename = instance-zope.cfg filename = instance-zope.cfg
extra-context = extra-context =
key buildout_directory buildout:directory key buildout_directory buildout:directory
key root_common context:root-common
section parameter_dict dynamic-template-zope-parameters section parameter_dict dynamic-template-zope-parameters
import urlparse urlparse import urlparse urlparse
import hashlib hashlib import hashlib hashlib
......
{% set slapparameter_dict = {} %}{# dummy -#}
{% import "root_common" as root_common with context -%}
{% set node_id = parameter_dict['node-id'] -%}
# Note: Environment is setup in running wrapper script, as zope.conf is read # Note: Environment is setup in running wrapper script, as zope.conf is read
# too late for some components. # too late for some components.
%define INSTANCE {{ parameter_dict['instance'] }} %define INSTANCE {{ parameter_dict['instance'] }}
...@@ -65,7 +68,7 @@ large-file-threshold {{ parameter_dict['large-file-threshold'] }} ...@@ -65,7 +68,7 @@ large-file-threshold {{ parameter_dict['large-file-threshold'] }}
{% endif -%} {% endif -%}
<product-config CMFActivity> <product-config CMFActivity>
node-id {{ parameter_dict['node-id'] }} node-id {{ node_id }}
</product-config> </product-config>
{% set timerserver_interval = parameter_dict['timerserver-interval'] -%} {% set timerserver_interval = parameter_dict['timerserver-interval'] -%}
...@@ -145,10 +148,12 @@ large-file-threshold {{ parameter_dict['large-file-threshold'] }} ...@@ -145,10 +148,12 @@ large-file-threshold {{ parameter_dict['large-file-threshold'] }}
<zodb_db {{ name }}> <zodb_db {{ name }}>
{%- set storage_type = type_dict[zodb_dict.pop('type')] %} {%- set storage_type = type_dict[zodb_dict.pop('type')] %}
{%- set storage_dict = zodb_dict.pop('storage-dict') %} {%- set storage_dict = zodb_dict.pop('storage-dict') %}
{%- do root_common.apply_overrides(zodb_dict, node_id) %}
{%- for key, value in zodb_dict.iteritems() %} {%- for key, value in zodb_dict.iteritems() %}
{{ key }} {{ value }} {{ key }} {{ value }}
{%- endfor %} {%- endfor %}
<{{ storage_type }}> <{{ storage_type }}>
{%- do root_common.apply_overrides(storage_dict, node_id) %}
{%- for key, value in storage_dict.iteritems() %} {%- for key, value in storage_dict.iteritems() %}
{{ key }} {{ value }} {{ key }} {{ value }}
{%- endfor %} {%- endfor %}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment