software/erp5: Adjust apachedex promise for haproxy

Since bd3c2b18 (Remove httpd from ERP5 and use haproxy instead, 2020-12-14)
timing in balancer access log files are in milliseconds, no longer in
microseconds as they use to be with apache.

We update apachedex to version 1.8, where it supports generating reports
from access log files with milliseconds, as long as the expected format is
passed with --logformat argument.

We changed the default value of apachedex configuration to use the expected
--logformat argument, but users with a custom apachedex config passed as
instance parameters (monitor.apachedex-configuration) will have to update
their request parameters, to also pass --logformat.
Since this is breaking change, we take this opportunity to introduce another
breaking change, this parameter is not longer a string with all arguments
on the same line, which was problematic because these arguments contained
quotes and backslashes. The arguments are now passed as a list of strings
which will become the list of arguments passed to apachedex.

This is implemented by generating an apachedex config file, which simplifies
the generation of the wrapper. For this we also needed to upate
slapos.toolbox.

The same change is reflected to the overriden profiles in slapos-master,
but for slapos-master we don't pass the --logformat argument, because
slapos-master is using httpd, so the logs timings are still in microseconds.

software/erp5/test/test/test_balancer.py

@@ -145,7 +145,15 @@ class BalancerTestCase(ERP5InstanceTestCase):
         # XXX what is this ? should probably not be needed here
         'name': cls.__name__,
         'monitor-passwd': 'secret',
-        'apachedex-configuration': '--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet',
+        'apachedex-configuration': [
+            '--logformat', '%h %l %u %t "%r" %>s %O "%{Referer}i" "%{User-Agent}i" %{ms}T',
+            '--erp5-base', '+erp5', '.*/VirtualHostRoot/erp5(/|\\?|$)',
+            '--base', '+other', '/',
+            '--skip-user-agent', 'Zabbix',
+            '--error-detail',
+            '--js-embed',
+            '--quiet',
+          ],
         'apachedex-promise-threshold': 100,
         'haproxy-server-check-path': '/',
         'zope-family-dict': {

software/slapos-master/buildout.hash.cfg

@@ -14,11 +14,11 @@
 # not need these here).
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 2ef0ddc206c6b0982a37cfc21f23e423
+md5sum = 4d0839e359c98ba3cd516903b72f798b
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = 4998e62351f54700ee23a2ca8cd89329
+md5sum = 9c67c77eab5195b2674e340fb44c48a2
 
 [template-apache-backend-conf]
 filename = apache-backend.conf.in

software/slapos-master/instance-balancer.cfg.in

@@ -259,14 +259,24 @@ command = ${monitor-generate-apachedex-report-wrapper:wrapper-path}
 [monitor-generate-apachedex-report-wrapper]
 recipe = slapos.cookbook:wrapper
 wrapper-path = ${directory:bin}/${:command}
-command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration "${apachedex-parameters:configuration}"
+command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration ${apachedex-parameters:configuration}
 command = generate-apachedex-report
 
+[monitor-apachedex-report-config]
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:etc}/${:_buildout_section_name_}
+template = inline:
+  {% for line in slapparameter_dict['apachedex-configuration'] %}
+    {# apachedex config files use shlex.split, so we need to quote the arguments. #}
+    {# BBB: in python 3 we can use shlex.quote instead. #}
+    {{ repr(line.encode('utf-8')) }}
+  {% endfor %}
+
 [apachedex-parameters]
 # XXX - Sample log file with curent date: apache_access.log-%(date)s.gz
 # which will be equivalent to apache_access.log-20150112.gz if the date is 2015-01-12
 apache-log-list = ${apache-conf-parameter-dict:access-log}
-configuration = {{ slapparameter_dict['apachedex-configuration'] }}
+configuration = ${monitor-apachedex-report-config:rendered}
 promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
 
 [{{ section('monitor-promise-apachedex-result') }}]

software/slapos-master/instance-erp5.cfg.in

@@ -374,8 +374,17 @@ config-shared-certificate-authority-path = ${directory:ca-dir}
 config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
 config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
 config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
-config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
-'--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
+config-apachedex-configuration = {{
+  dumps(
+    monitor_dict.get(
+      'apachedex-configuration',
+      [
+        '--erp5-base', '+erp5', '.*/VirtualHostRoot/erp5(/|\\?|$)',
+        '--base', '+other', '/',
+        '--skip-user-agent', 'Zabbix',
+        '--error-detail',
+        '--js-embed',
+        '--quiet'])) }}
 
 [request-frontend-base]
 {% if has_frontend -%}

stack/erp5/buildout.cfg

@@ -623,7 +623,7 @@ zope.app.publication = 3.14.0
 zope.app.testing = 3.8.1
 
 # Pinned versions
-APacheDEX = 1.6.2
+APacheDEX = 1.8
 Pillow = 6.2.2
 Products.CMFActionIcons = 2.1.3
 Products.DCWorkflowGraph = 0.4.1

stack/erp5/buildout.hash.cfg

@@ -78,7 +78,7 @@ md5sum = 68b329da9893e34099c7d8ad5cb9c940
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 5ea4bcdf74fb429f254af8e8fb7b38a3
+md5sum = b2259f6f6c620c9a1738937e6430d5b5
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -90,7 +90,7 @@ md5sum = c03f93f95333e6a61b857dcfab7f9c0e
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = 4a119083eab1eadbaf44468eb4f3381f
+md5sum = 8ad9137310ae0403d433bb3c0d93be9f
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -360,12 +360,22 @@ command = ${monitor-generate-apachedex-report-wrapper:wrapper-path}
 [monitor-generate-apachedex-report-wrapper]
 recipe = slapos.cookbook:wrapper
 wrapper-path = ${directory:bin}/${:command}
-command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration "${apachedex-parameters:configuration}"
+command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration ${apachedex-parameters:configuration}
 command = generate-apachedex-report
 
+[monitor-apachedex-report-config]
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:etc}/${:_buildout_section_name_}
+template = inline:
+  {% for line in slapparameter_dict['apachedex-configuration'] %}
+    {# apachedex config files use shlex.split, so we need to quote the arguments. #}
+    {# BBB: in python 3 we can use shlex.quote instead. #}
+    {{ repr(line.encode('utf-8')) }}
+  {% endfor %}
+
 [apachedex-parameters]
 apache-log-list = ${rsyslogd-cfg-parameter-dict:access-log-file}
-configuration = {{ slapparameter_dict['apachedex-configuration'] }}
+configuration = ${monitor-apachedex-report-config:rendered}
 promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
 
 [{{ section('monitor-promise-apachedex-result') }}]

stack/erp5/instance-erp5.cfg.in

@@ -351,8 +351,18 @@ config-name = ${:name}
 config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
 config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
 config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
-config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
-'--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
+config-apachedex-configuration = {{
+  dumps(
+    monitor_dict.get(
+      'apachedex-configuration',
+      [
+        '--logformat', '%h %l %u %t "%r" %>s %O "%{Referer}i" "%{User-Agent}i" %{ms}T',
+        '--erp5-base', '+erp5', '.*/VirtualHostRoot/erp5(/|\\?|$)',
+        '--base', '+other', '/',
+        '--skip-user-agent', 'Zabbix',
+        '--error-detail',
+        '--js-embed',
+        '--quiet'])) }}
 
 [request-frontend-base]
 {% if has_frontend -%}