software/erp5: Adjust apachedex promise for haproxy

Since bd3c2b18 (Remove httpd from ERP5 and use haproxy instead, 2020-12-14)
timing in balancer access log files are in milliseconds, no longer in
microseconds as they use to be with apache.

Now apachedex supports generating reports from access log files with
milliseconds, as long as the expected format is passed with --logformat
argument.

We changed the default value of apachedex configuration to use the expected
--logformat argument, but users with a custom apachedex config passed as
instance parameters (monitor.apachedex-configuration) will have to update
their request parameters, to also pass --logformat.
Since this is breaking change, we take this opportunity to introduce another
breaking change, this parameter is not longer a string with all arguments
on the same line, which was problematic because these arguments contained
quotes and backslashes. The arguments are now passed as a list of strings
which will become the list of arguments passed to apachedex.

This is implemented by generating an apachedex config file, which simplifies
the generation of the wrapper. For this we also upate slapos.toolbox.

software/erp5/test/test/test_balancer.py

@@ -145,7 +145,15 @@ class BalancerTestCase(ERP5InstanceTestCase):
         # XXX what is this ? should probably not be needed here
         'name': cls.__name__,
         'monitor-passwd': 'secret',
-        'apachedex-configuration': '--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet',
+        'apachedex-configuration': [
+            '--logformat', '%h %l %u %t "%r" %>s %O "%{Referer}i" "%{User-Agent}i" %{ms}T',
+            '--erp5-base', '+erp5', '.*/VirtualHostRoot/erp5(/|\\?|$)',
+            '--base', '+other', '/',
+            '--skip-user-agent', 'Zabbix',
+            '--error-detail',
+            '--js-embed',
+            '--quiet',
+          ],
         'apachedex-promise-threshold': 100,
         'haproxy-server-check-path': '/',
         'zope-family-dict': {

stack/erp5/buildout.cfg

@@ -623,7 +623,7 @@ zope.app.publication = 3.14.0
 zope.app.testing = 3.8.1
 
 # Pinned versions
-APacheDEX = 1.6.2
+APacheDEX = 1.6.2 TODO
 Pillow = 6.2.2
 Products.CMFActionIcons = 2.1.3
 Products.DCWorkflowGraph = 0.4.1

stack/erp5/buildout.hash.cfg

@@ -78,7 +78,7 @@ md5sum = 68b329da9893e34099c7d8ad5cb9c940
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 5ea4bcdf74fb429f254af8e8fb7b38a3
+md5sum = b2259f6f6c620c9a1738937e6430d5b5
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -90,7 +90,7 @@ md5sum = c03f93f95333e6a61b857dcfab7f9c0e
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = 4a119083eab1eadbaf44468eb4f3381f
+md5sum = 8ad9137310ae0403d433bb3c0d93be9f
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -360,12 +360,22 @@ command = ${monitor-generate-apachedex-report-wrapper:wrapper-path}
 [monitor-generate-apachedex-report-wrapper]
 recipe = slapos.cookbook:wrapper
 wrapper-path = ${directory:bin}/${:command}
-command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration "${apachedex-parameters:configuration}"
+command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration ${apachedex-parameters:configuration}
 command = generate-apachedex-report
 
+[monitor-apachedex-report-config]
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:etc}/${:_buildout_section_name_}
+template = inline:
+  {% for line in slapparameter_dict['apachedex-configuration'] %}
+    {# apachedex config files use shlex.split, so we need to quote the arguments. #}
+    {# BBB: in python 3 we can use shlex.quote instead. #}
+    {{ repr(line.encode('utf-8')) }}
+  {% endfor %}
+
 [apachedex-parameters]
 apache-log-list = ${rsyslogd-cfg-parameter-dict:access-log-file}
-configuration = {{ slapparameter_dict['apachedex-configuration'] }}
+configuration = ${monitor-apachedex-report-config:rendered}
 promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
 
 [{{ section('monitor-promise-apachedex-result') }}]

stack/erp5/instance-erp5.cfg.in

@@ -351,8 +351,18 @@ config-name = ${:name}
 config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
 config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
 config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
-config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
-'--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
+config-apachedex-configuration = {{
+  dumps(
+    monitor_dict.get(
+      'apachedex-configuration',
+      [
+        '--logformat', '%h %l %u %t "%r" %>s %O "%{Referer}i" "%{User-Agent}i" %{ms}T',
+        '--erp5-base', '+erp5', '.*/VirtualHostRoot/erp5(/|\\?|$)',
+        '--base', '+other', '/',
+        '--skip-user-agent', 'Zabbix',
+        '--error-detail',
+        '--js-embed',
+        '--quiet'])) }}
 
 [request-frontend-base]
 {% if has_frontend -%}