Commit e6087b7d authored by Jérome Perrin's avatar Jérome Perrin

software/erp5: Adjust apachedex promise for haproxy

Since bd3c2b18 (Remove httpd from ERP5 and use haproxy instead, 2020-12-14)
timing in balancer access log files are in milliseconds, no longer in
microseconds as they use to be with apache.

We update apachedex to version 1.8, where it supports generating reports
from access log files with milliseconds, as long as the expected format is
passed with --logformat argument.

We changed the default value of apachedex configuration to use the expected
--logformat argument, but users with a custom apachedex config passed as
instance parameters (monitor.apachedex-configuration) will have to update
their request parameters, to also pass --logformat.
Since this is breaking change, we take this opportunity to introduce another
breaking change, this parameter is not longer a string with all arguments
on the same line, which was problematic because these arguments contained
quotes and backslashes. The arguments are now passed as a list of strings
which will become the list of arguments passed to apachedex.

This is implemented by generating an apachedex config file, which simplifies
the generation of the wrapper. For this we also needed to upate
slapos.toolbox.

The same change is reflected to the overriden profiles in slapos-master,
but for slapos-master we don't pass the --logformat argument, because
slapos-master is using httpd, so the logs timings are still in microseconds.
parent ef781ea8
Pipeline #13255 failed with stage
in 0 seconds
...@@ -145,7 +145,15 @@ class BalancerTestCase(ERP5InstanceTestCase): ...@@ -145,7 +145,15 @@ class BalancerTestCase(ERP5InstanceTestCase):
# XXX what is this ? should probably not be needed here # XXX what is this ? should probably not be needed here
'name': cls.__name__, 'name': cls.__name__,
'monitor-passwd': 'secret', 'monitor-passwd': 'secret',
'apachedex-configuration': '--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet', 'apachedex-configuration': [
'--logformat', '%h %l %u %t "%r" %>s %O "%{Referer}i" "%{User-Agent}i" %{ms}T',
'--erp5-base', '+erp5', '.*/VirtualHostRoot/erp5(/|\\?|$)',
'--base', '+other', '/',
'--skip-user-agent', 'Zabbix',
'--error-detail',
'--js-embed',
'--quiet',
],
'apachedex-promise-threshold': 100, 'apachedex-promise-threshold': 100,
'haproxy-server-check-path': '/', 'haproxy-server-check-path': '/',
'zope-family-dict': { 'zope-family-dict': {
......
...@@ -14,11 +14,11 @@ ...@@ -14,11 +14,11 @@
# not need these here). # not need these here).
[template-erp5] [template-erp5]
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = 2ef0ddc206c6b0982a37cfc21f23e423 md5sum = 4d0839e359c98ba3cd516903b72f798b
[template-balancer] [template-balancer]
filename = instance-balancer.cfg.in filename = instance-balancer.cfg.in
md5sum = 4998e62351f54700ee23a2ca8cd89329 md5sum = 9c67c77eab5195b2674e340fb44c48a2
[template-apache-backend-conf] [template-apache-backend-conf]
filename = apache-backend.conf.in filename = apache-backend.conf.in
......
...@@ -259,14 +259,24 @@ command = ${monitor-generate-apachedex-report-wrapper:wrapper-path} ...@@ -259,14 +259,24 @@ command = ${monitor-generate-apachedex-report-wrapper:wrapper-path}
[monitor-generate-apachedex-report-wrapper] [monitor-generate-apachedex-report-wrapper]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/${:command} wrapper-path = ${directory:bin}/${:command}
command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration "${apachedex-parameters:configuration}" command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration ${apachedex-parameters:configuration}
command = generate-apachedex-report command = generate-apachedex-report
[monitor-apachedex-report-config]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:etc}/${:_buildout_section_name_}
template = inline:
{% for line in slapparameter_dict['apachedex-configuration'] %}
{# apachedex config files use shlex.split, so we need to quote the arguments. #}
{# BBB: in python 3 we can use shlex.quote instead. #}
{{ repr(line.encode('utf-8')) }}
{% endfor %}
[apachedex-parameters] [apachedex-parameters]
# XXX - Sample log file with curent date: apache_access.log-%(date)s.gz # XXX - Sample log file with curent date: apache_access.log-%(date)s.gz
# which will be equivalent to apache_access.log-20150112.gz if the date is 2015-01-12 # which will be equivalent to apache_access.log-20150112.gz if the date is 2015-01-12
apache-log-list = ${apache-conf-parameter-dict:access-log} apache-log-list = ${apache-conf-parameter-dict:access-log}
configuration = {{ slapparameter_dict['apachedex-configuration'] }} configuration = ${monitor-apachedex-report-config:rendered}
promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }} promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
[{{ section('monitor-promise-apachedex-result') }}] [{{ section('monitor-promise-apachedex-result') }}]
......
...@@ -374,8 +374,17 @@ config-shared-certificate-authority-path = ${directory:ca-dir} ...@@ -374,8 +374,17 @@ config-shared-certificate-authority-path = ${directory:ca-dir}
config-backend-path-dict = {{ dumps(zope_backend_path_dict) }} config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }} config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }} config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration', config-apachedex-configuration = {{
'--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }} dumps(
monitor_dict.get(
'apachedex-configuration',
[
'--erp5-base', '+erp5', '.*/VirtualHostRoot/erp5(/|\\?|$)',
'--base', '+other', '/',
'--skip-user-agent', 'Zabbix',
'--error-detail',
'--js-embed',
'--quiet'])) }}
[request-frontend-base] [request-frontend-base]
{% if has_frontend -%} {% if has_frontend -%}
......
...@@ -623,7 +623,7 @@ zope.app.publication = 3.14.0 ...@@ -623,7 +623,7 @@ zope.app.publication = 3.14.0
zope.app.testing = 3.8.1 zope.app.testing = 3.8.1
# Pinned versions # Pinned versions
APacheDEX = 1.6.2 APacheDEX = 1.8
Pillow = 6.2.2 Pillow = 6.2.2
Products.CMFActionIcons = 2.1.3 Products.CMFActionIcons = 2.1.3
Products.DCWorkflowGraph = 0.4.1 Products.DCWorkflowGraph = 0.4.1
......
...@@ -78,7 +78,7 @@ md5sum = 68b329da9893e34099c7d8ad5cb9c940 ...@@ -78,7 +78,7 @@ md5sum = 68b329da9893e34099c7d8ad5cb9c940
[template-erp5] [template-erp5]
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = 5ea4bcdf74fb429f254af8e8fb7b38a3 md5sum = b2259f6f6c620c9a1738937e6430d5b5
[template-zeo] [template-zeo]
filename = instance-zeo.cfg.in filename = instance-zeo.cfg.in
...@@ -90,7 +90,7 @@ md5sum = c03f93f95333e6a61b857dcfab7f9c0e ...@@ -90,7 +90,7 @@ md5sum = c03f93f95333e6a61b857dcfab7f9c0e
[template-balancer] [template-balancer]
filename = instance-balancer.cfg.in filename = instance-balancer.cfg.in
md5sum = 4a119083eab1eadbaf44468eb4f3381f md5sum = 8ad9137310ae0403d433bb3c0d93be9f
[template-haproxy-cfg] [template-haproxy-cfg]
filename = haproxy.cfg.in filename = haproxy.cfg.in
......
...@@ -360,12 +360,22 @@ command = ${monitor-generate-apachedex-report-wrapper:wrapper-path} ...@@ -360,12 +360,22 @@ command = ${monitor-generate-apachedex-report-wrapper:wrapper-path}
[monitor-generate-apachedex-report-wrapper] [monitor-generate-apachedex-report-wrapper]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/${:command} wrapper-path = ${directory:bin}/${:command}
command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration "${apachedex-parameters:configuration}" command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration ${apachedex-parameters:configuration}
command = generate-apachedex-report command = generate-apachedex-report
[monitor-apachedex-report-config]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:etc}/${:_buildout_section_name_}
template = inline:
{% for line in slapparameter_dict['apachedex-configuration'] %}
{# apachedex config files use shlex.split, so we need to quote the arguments. #}
{# BBB: in python 3 we can use shlex.quote instead. #}
{{ repr(line.encode('utf-8')) }}
{% endfor %}
[apachedex-parameters] [apachedex-parameters]
apache-log-list = ${rsyslogd-cfg-parameter-dict:access-log-file} apache-log-list = ${rsyslogd-cfg-parameter-dict:access-log-file}
configuration = {{ slapparameter_dict['apachedex-configuration'] }} configuration = ${monitor-apachedex-report-config:rendered}
promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }} promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
[{{ section('monitor-promise-apachedex-result') }}] [{{ section('monitor-promise-apachedex-result') }}]
......
...@@ -351,8 +351,18 @@ config-name = ${:name} ...@@ -351,8 +351,18 @@ config-name = ${:name}
config-backend-path-dict = {{ dumps(zope_backend_path_dict) }} config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }} config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }} config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration', config-apachedex-configuration = {{
'--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }} dumps(
monitor_dict.get(
'apachedex-configuration',
[
'--logformat', '%h %l %u %t "%r" %>s %O "%{Referer}i" "%{User-Agent}i" %{ms}T',
'--erp5-base', '+erp5', '.*/VirtualHostRoot/erp5(/|\\?|$)',
'--base', '+other', '/',
'--skip-user-agent', 'Zabbix',
'--error-detail',
'--js-embed',
'--quiet'])) }}
[request-frontend-base] [request-frontend-base]
{% if has_frontend -%} {% if has_frontend -%}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment