erp5: add monitoring instance & http log access

Conflicts:
	stack/erp5/buildout.cfg

stack/erp5/buildout.cfg

@@ -57,6 +57,8 @@ extends =
   ../../component/findutils/buildout.cfg
   ../../component/userhosts/buildout.cfg
   logrotate-base.cfg
+  ../../stack/monitor/buildout.cfg
+  ../../component/pycurl/buildout.cfg
 
 parts =
   erp5-util-develop
@@ -156,10 +158,15 @@ context =
     key slapos_core_version versions:slapos.core
     ${:extra-context}
 
+[template-http-monitor]
+< = download-base
+filename = instance-http-monitor.cfg.in
+md5sum = 16d1fc3cf26c3066ae612ded0ec64cd2
+
 [template-mariadb]
 < = download-base
 filename = instance-mariadb.cfg.in
-md5sum = 8a59e1730391fb3a859cd973cd7c3a5b
+md5sum = d42f6a8b11e5908b6a44ce2c3dae0367
 link-binary =
   ${coreutils:location}/bin/basename
   ${coreutils:location}/bin/cat
@@ -270,10 +277,12 @@ extra-context =
     key template_create_erp5_site_real template-create-erp5-site-real:target
     key template_erp5 template-erp5:target
     key template_haproxy_cfg template-haproxy-cfg:target
+    key template_http_monitor template-http-monitor:target
     key template_kumofs template-kumofs:target
     key template_logrotate_base template-logrotate-base:rendered
     key template_mariadb template-mariadb:target
     key template_mariadb_initial_setup template-mariadb-initial-setup:target
+    key template_monitor monitor-template:output
     key template_my_cnf template-my-cnf:target
     key template_runzope_userhosts_preloaded template-runzope-userhosts-preloaded:target
     key template_zeo template-zeo:target
@@ -298,7 +307,7 @@ md5sum = 651fa2ec2feb768e339936b7855774ca
 [template-zope]
 < = download-base
 filename = instance-zope.cfg.in
-md5sum = 7f185192e0536ecacc403cf6b1736d2d
+md5sum = 044fea3d2d13783a5509cbcffc2d197b
 link-binary =
   ${aspell:location}/bin/aspell
   ${coreutils:location}/bin/basename
@@ -536,6 +545,11 @@ eggs =
 # Needed for parsing .po files from our Localizer subset
   polib
 
+# Needed for monitoring tool
+  cns.recipe.symlink
+  collective.recipe.template
+  ${pycurl:egg}
+
 # parameterizing the version of the generated python interpreter name by the
 # python section version causes dependency between this egg section and the
 # installation of python, which we don't want on an instance
@@ -579,10 +593,10 @@ branch = erp5-cluster
 recipe = zc.recipe.egg
 eggs =
   ${lxml-python:egg}
-  slapos.toolbox
-  slapos.toolbox[zodbpack]
+  slapos.toolbox[check_web_page_http_cache_hit,zodbpack]
 
 scripts =
+  check-web-page-http-cache-hit
   is-local-tcp-port-opened
   onetimedownload
   slapos-kill

stack/erp5/instance-erp5.cfg.in

@@ -44,6 +44,7 @@ config-{{ option }} = {{ dumps(value) }}
 {% else -%}
 {% do zodb_extern[0] %}
 {% endif -%}
+{{ request('http-monitor', 'http-monitor', 'http-monitor', {}, {'monitor-url': True}) }}
 
 [inituser-password]
 {% set inituser_password = slapparameter_dict.get('inituser-password') -%}

stack/erp5/instance-http-monitor.cfg.in

+{% if software_type == slap_software_type -%}
+[buildout]
+parts =
+  certificate-authority
+  cron-entry-monitor
+  cron-entry-rss
+  cron
+  deploy-index
+  deploy-settings-cgi
+  deploy-status-cgi
+  deploy-status-history-cgi
+  setup-static-files
+  certificate-authority
+  zero-parameters
+  public-symlink
+  cgi-httpd-wrapper
+  cgi-httpd-graceful-wrapper
+  monitor-promise
+  monitor-instance-log-access
+  monitor-check-web-page-http-cache-hit
+  publish-http-monitor
+
+extends = {{ parameter_dict["template-monitor"] }}
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+
+[monitor-check-web-page-http-cache-hit]
+recipe = slapos.cookbook:wrapper
+command-line = "{{ parameter_dict['bin-directory'] }}/check-web-page-http-cache-hit" -K "${public:filename}"
+wrapper-path = ${monitor-directory:monitor-custom-scripts}/check-web-page-http-cache-hit
+
+[public]
+recipe = slapos.cookbook:zero-knowledge.write
+filename = knowledge0.cfg
+status-history-length = 5
+# Each one of the following 2 list keys should be defined in only one line
+# ex: resolve-list = www.example.com:80:127.0.0.1 www.perdu.com:80:10.0.0.1
+url-list =
+resolve-list =
+
+[zero-parameters]
+recipe = slapos.cookbook:zero-knowledge.read
+filename = ${public:filename}
+
+[publish-http-monitor]
+recipe = slapos.cookbook:publish.serialised
+monitor-url = ${monitor-parameters:url}
+
+{% endif %}

stack/erp5/instance-mariadb.cfg.in

@@ -31,6 +31,7 @@ recipe = slapos.cookbook:publish.serialised
 {% endmacro -%}
 database-list = {{ render_database_list(database_list) }}
 test-database-list = {{ render_database_list(test_database_list) }}
+monitor-url = ${publish-connection-informations:monitor_url}
 
 [simplefile]
 recipe = slapos.recipe.template:jinja2
@@ -217,8 +218,21 @@ command-line = "{{ parameter_dict['bin-directory'] }}/is-local-tcp-port-opened"
 wrapper-path = ${directory:promise}/mariadb
 parameters-extra = true
 
+[{{ section('monitor-current-log-access') }}]
+< = monitor-directory-access
+source = ${directory:log}
+
+[{{ section('monitor-backup-log-access') }}]
+< = monitor-directory-access
+source = ${logrotate-entry-mariadb:backup}
+
+[monitor-parameters]
+port = 3380
+
 [buildout]
-extends = {{ logrotate_cfg }}
+extends =
+  {{ logrotate_cfg }}
+  {{ parameter_dict['template-monitor'] }}
 parts +=
   publish-mariadb-url
   logrotate-entry-mariadb
@@ -226,4 +240,19 @@ parts +=
   update-mysql
   mysqld
   promise
+
+# Access to mariadb logs
+  certificate-authority
+  cron-entry-monitor
+  cron-entry-rss
+  deploy-index
+  setup-static-files
+  certificate-authority
+  public-symlink
+  cgi-httpd-wrapper
+  cgi-httpd-graceful-wrapper
+  monitor-promise
+  monitor-instance-log-access
+
+# Complete parts with sections
   {{ part_list | join('\n  ') }}

stack/erp5/instance-zope.cfg.in

@@ -78,6 +78,7 @@ ca-private = ${:ca-dir}/private
 ca-certs = ${:ca-dir}/certs
 ca-newcerts = ${:ca-dir}/newcerts
 ca-crl = ${:ca-dir}/crl
+logrotate-backup = ${:var}/logrotate
 
 [binary-link]
 recipe = slapos.cookbook:symbolic.link
@@ -98,6 +99,17 @@ recipe = slapos.cookbook:certificate_authority
 openssl-binary = ${binary-link:target-directory}/openssl
 wrapper = ${directory:services}/ca
 
+[{{ section('monitor-current-log-access') }}]
+< = monitor-directory-access
+source = ${directory:log}
+
+[{{ section('monitor-backup-log-access') }}]
+< = monitor-directory-access
+source = ${directory:logrotate-backup}
+
+[monitor-parameters]
+port = {{ slapparameter_dict['port-base'] + 5000 }}
+
 {% if use_ipv6 -%}
 {%   set ipv6 = (ipv6_set | list)[0] -%}
 [zeo-tunnel-base]
@@ -363,6 +375,7 @@ dict toward erp5 partition, violating the DRY principle and making the intent
 hard to guess.
 -#}
 hosts-dict = {{ dumps(hosts_dict) }}
+monitor-url = ${monitor-parameters:url}
 
 [erp5-promise]
 recipe = slapos.cookbook:erp5.promise
@@ -375,9 +388,24 @@ bt5 = {{ dumps(slapparameter_dict['bt5']) }}
 bt5-repository-url = {{ dumps(slapparameter_dict['bt5-repository-url']) }}
 
 [buildout]
-extends = {{ logrotate_cfg }}
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+extends =
+  {{ logrotate_cfg }}
+  {{ parameter_dict['template-monitor'] }}
 parts +=
   erp5-promise
+  certificate-authority
+  cron-entry-monitor
+  cron-entry-rss
+  deploy-index
+  setup-static-files
+  certificate-authority
+  public-symlink
+  cgi-httpd-wrapper
+  cgi-httpd-graceful-wrapper
+  monitor-promise
+  monitor-instance-log-access
   {{ part_list | join('\n  ') }}
   publish-zope
 versions = versions

stack/erp5/instance.cfg.in

@@ -118,6 +118,7 @@ jsl = {{ jsl_location }}
 link-binary = {{ dumps(zope_link_binary) }}
 userhosts = {{ userhosts_location }}
 runzope-userhosts-preloaded-template = {{ template_runzope_userhosts_preloaded }}
+template-monitor = {{ template_monitor }}
 
 [dynamic-template-zope]
 < = jinja2-template-base
@@ -152,6 +153,7 @@ template-my-cnf = {{ template_my_cnf }}
 template-mariadb-initial-setup = {{ template_mariadb_initial_setup }}
 link-binary = {{ dumps(mariadb_link_binary) }}
 bin-directory = {{ bin_directory }}
+template-monitor = {{ template_monitor }}
 
 [dynamic-template-mariadb]
 < = jinja2-template-base
@@ -174,6 +176,18 @@ filename = instance-create-erp5-site.cfg
 extra-context =
     section parameter_dict dynamic-template-create-erp5-site-parameters
 
+[dynamic-template-http-monitor-parameters]
+template-monitor = {{ template_monitor }}
+bin-directory = {{ bin_directory }}
+
+[dynamic-template-http-monitor]
+< = jinja2-template-base
+template = {{ template_http_monitor }}
+filename = instance-http-monitor.cfg
+extra-context =
+    section parameter_dict dynamic-template-http-monitor-parameters
+    raw software_type http-monitor
+
 [switch-softwaretype]
 recipe = slapos.cookbook:switch-softwaretype
 override = {{ dumps(override_switch_softwaretype |default) }}