Merge remote-tracking branch 'origin/master' into zope4

README.update-hash.rst

+update-hash
+===========
+
+``update-hash`` is a tool to assist software release developers in the management of ``buildout.hash.cfg`` files.
+
+A lot of recipes which uses hashing for referenced files. Updating the hash results in part uninstallation and installation, which is desired behaviour, as the file might have to be redownloaded. By using ``update-hash`` with ``buildout.hash.cfg`` the developer does not have to do the calculations and updates manually, just calling the tool is enough.
+
+Generally each Buildout profile which references some file shall use this approach to improve development process and minimise risk of using incorrect data from such entires.
+
+Working with ``buildout.hash.cfg``
+----------------------------------
+
+``buildout.hash.cfg`` files are buildout-style simplified configparser files to have a easy way to update MD5 hashes of provided files for download. They look like::
+
+  [section]
+  md5sum = <hash>
+  filename = <relative-path>
+
+Where ``<hash>`` is an automatically calculated checksum of ``<relative-path>``.
+
+Then ``buildout.hash.cfg`` can be included in software profile by ``extends`` of ``[buildout]`` section, and the section's ``md5sum`` and ``filename`` can be used.
+
+Special cases of ``filename`` key
+---------------------------------
+
+In case if section recipe has special unwanted behaviour for ``filename`` field the ``_update_hash_filename_`` key can be used like::
+
+  [section]
+  md5sum = <hash>
+  _update_hash_filename_ = <relative-path>
+
+Working with ``update-hash``
+----------------------------
+
+In order to update the ``buildout.hash.cfg`` one just need to call ``update-hash`` while being in the directory containing the file.

component/apache/buildout.cfg

@@ -39,9 +39,9 @@ configure-options =
 [apache]
 recipe = slapos.recipe.cmmi
 shared = true
-version = 2.4.43
+version = 2.4.46
 url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
-md5sum = 791c986b1e70fe61eb44060aacc89a64
+md5sum = 7d661ea5e736dac5e2761d9f49fe8361
 configure-options = --disable-static
                     --enable-authn-alias
                     --enable-bucketeer

component/golang/buildout.cfg

@@ -32,37 +32,10 @@ environment-extra =
 make-targets= cd src && ./make.bash && cp -alf .. ${:location}
 
 
-[golang19]
-<= golang-common
-url = https://dl.google.com/go/go1.9.7.src.tar.gz
-md5sum = 3c2cf876ed6612a022574a565206c6ea
-
-# go1.9 needs go1.4 to bootstrap
-environment-extra =
-  GOROOT_BOOTSTRAP=${golang14:location}
-
-[golang1.10]
-<= golang-common
-url = https://dl.google.com/go/go1.10.3.src.tar.gz
-md5sum = d15dfb264105c5e84fbe33f4a4aa5021
-
-# go1.10 needs go1.4 to bootstrap
-environment-extra =
-  GOROOT_BOOTSTRAP=${golang14:location}
-
-[golang1.11]
-<= golang-common
-url = https://dl.google.com/go/go1.11.4.src.tar.gz
-md5sum = a77697673215be465d1b583680ef2318
-
-# go1.11 needs go1.4 to bootstrap
-environment-extra =
-  GOROOT_BOOTSTRAP=${golang14:location}
-
 [golang1.12]
 <= golang-common
-url = https://dl.google.com/go/go1.12.9.src.tar.gz
-md5sum = 6132109d4050da349eadc9f7b0304ef4
+url = https://golang.org/dl/go1.12.17.src.tar.gz
+md5sum = 6b607fc795391dc609ffd79ebf41f080
 
 # go1.12 needs go1.4 to bootstrap
 environment-extra =
@@ -70,8 +43,8 @@ environment-extra =
 
 [golang1.13]
 <= golang-common
-url = https://dl.google.com/go/go1.13.9.src.tar.gz
-md5sum = 4ad8b04f962be93a32f3021e6f35b3b9
+url = https://golang.org/dl/go1.13.15.src.tar.gz
+md5sum = 4f4af14d88352a62761a9dcedf863ac0
 
 # go1.13 needs go1.4 to bootstrap
 environment-extra =
@@ -79,8 +52,8 @@ environment-extra =
 
 [golang1.14]
 <= golang-common
-url = https://dl.google.com/go/go1.14.3.src.tar.gz
-md5sum = 6b1fb42d219e2ea8925002013c76d4c7
+url = https://golang.org/dl/go1.14.9.src.tar.gz
+md5sum = 6f6dd1377421d27ca4bb607283b31738
 
 # go1.14 needs go1.4 to bootstrap
 environment-extra =
@@ -118,7 +91,7 @@ bin	= ${:directory}/bin
 depends = ${gowork.goinstall:recipe}
 
 # go version used for the workspace (possible to override in applications)
-golang  = ${golang1.10:location}
+golang  = ${golang1.14:location}
 
 # no special build flags by default
 buildflags =
@@ -138,7 +111,7 @@ recipe	= slapos.recipe.template
 url     = ${:_profile_base_location_}/goenv.sh.in
 output	= ${gowork:directory}/env.sh
 depends = ${gowork.mkdir:recipe}
-md5sum	= 6bcf96cf18ea68ce6e378ed8d49346a3
+md5sum	= f89553711ea95ad1f6727b76747a62bf
 
 [gowork.mkdir]
 # NOTE do not use slapos.cookbook:mkdirectory here - if anything in software (not instance)

component/golang/goenv.sh.in

@@ -12,6 +12,8 @@ export PKG_CONFIG_PATH=$(echo -n "${gowork:cpkgpath}" |tr '\n' ':'):$PKG_CONFIG_
 export GOPATH=$X:$GOPATH
 export PATH=$X/bin:$PATH
 export PS1="(`basename $X`) $PS1"
+export GOCACHE=$X/cache
+export GOENV=$X/goenv
 
 # strip trailing : from $GOPATH, $PKG_CONFIG_PATH
 GOPATH=$${GOPATH%:}

component/haproxy/buildout.cfg

@@ -13,8 +13,8 @@ parts = haproxy
 [haproxy]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.haproxy.org/download/2.0/src/haproxy-2.0.15.tar.gz
-md5sum = 59f892991476f08e2d16ac460c502f61
+url = http://www.haproxy.org/download/2.0/src/haproxy-2.0.17.tar.gz
+md5sum = 786a967c73cc1455c938d42fbe333bfe
 configure-command = true
 # for Linux kernel 2.6.28 and above, we use "linux-glibc" as the TARGET,
 # otherwise use "generic".

component/helloweb/buildout.cfg

@@ -25,7 +25,7 @@ parts =
 install =
     lab.nexedi.com/nexedi/helloweb/go/...
 
-golang  = ${golang1.12:location}
+golang  = ${golang1.14:location}
 
 # -*- go -*-
 [helloweb-go]

component/nginx/buildout.cfg

@@ -13,8 +13,8 @@ parts = nginx-output
 [nginx-common]
 recipe = slapos.recipe.cmmi
 shared = false
-url = https://nginx.org/download/nginx-1.17.6.tar.gz
-md5sum = 55022aa5715386c994f6773478822853
+url = https://nginx.org/download/nginx-1.19.2.tar.gz
+md5sum = 3dc55f6451ed6f819f1c796f4e5e9617
 
 [nginx]
 <= nginx-common
@@ -26,8 +26,8 @@ configure-options=
   --with-http_realip_module
   --with-mail
   --with-mail_ssl_module
-  --with-ld-opt="-L ${openssl-1.0:location}/lib -L ${pcre:location}/lib -L ${zlib:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib"
-  --with-cc-opt="-I ${openssl-1.0:location}/include -I ${pcre:location}/include -I ${zlib:location}/include"
+  --with-ld-opt="-L ${openssl:location}/lib -L ${pcre:location}/lib -L ${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib"
+  --with-cc-opt="-I ${openssl:location}/include -I ${pcre:location}/include -I ${zlib:location}/include"
 
 [nginx-dav-ext-module]
 recipe = slapos.recipe.build:download-unpacked
@@ -46,8 +46,8 @@ configure-options =
   --with-mail
   --with-mail_ssl_module
   --error-log-path=var/log/nginx.error.log
-  --with-ld-opt=" -L ${libexpat:location}/lib -L ${openssl-1.0:location}/lib -L ${pcre:location}/lib -L ${zlib:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib"
-  --with-cc-opt="-I ${libexpat:location}/include -I ${openssl-1.0:location}/include -I ${pcre:location}/include -I ${zlib:location}/include"
+  --with-ld-opt=" -L ${libexpat:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib -L ${zlib:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib"
+  --with-cc-opt="-I ${libexpat:location}/include -I ${openssl:location}/include -I ${pcre:location}/include -I ${zlib:location}/include"
   --with-http_dav_module 
   --add-module='${nginx-dav-ext-module:location}'
 
@@ -73,8 +73,8 @@ strip-top-level-dir = true
 configure-options=
   --with-ipv6
   --with-http_ssl_module
-  --with-ld-opt="-L ${zlib:location}/lib -L ${openssl-1.0:location}/lib -L ${pcre:location}/lib  -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib"
-  --with-cc-opt="-I ${pcre:location}/include -I ${openssl-1.0:location}/include -I ${zlib:location}/include"
+  --with-ld-opt="-L ${zlib:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib  -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib"
+  --with-cc-opt="-I ${pcre:location}/include -I ${openssl:location}/include -I ${zlib:location}/include"
   --add-module=${hexaglobe-nginx-module:location}/sub_module
 # --add-module=${hexaglobe-nginx-module:location}/nginx-upstream-fair
 
@@ -91,8 +91,8 @@ configure-options=
   --with-http_ssl_module
   --with-http_v2_module
   --with-http_gzip_static_module
-  --with-ld-opt="-L ${zlib:location}/lib -L ${openssl-1.0:location}/lib -L ${pcre:location}/lib  -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib"
-  --with-cc-opt="-I ${pcre:location}/include -I ${openssl-1.0:location}/include -I ${zlib:location}/include -Wno-error"
+  --with-ld-opt="-L ${zlib:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib  -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib"
+  --with-cc-opt="-I ${pcre:location}/include -I ${openssl:location}/include -I ${zlib:location}/include -Wno-error"
   --add-module=${nginx-push-stream-module:location}
 
 [nginx-push-stream-output]

component/nodejs/buildout.cfg

@@ -14,6 +14,12 @@ parts =
 [nodejs]
 <= nodejs-8.9.4
 
+[nodejs-12.18.3]
+<= nodejs-base
+openssl_location = ${openssl:location}
+version = v12.18.3
+md5sum = 28bf6a4d98b238403fa58a0805f4a979
+
 [nodejs-10.19.0]
 <= nodejs-base
 openssl_location = ${openssl:location}

component/openblas/buildout.cfg

@@ -15,23 +15,21 @@ patch-options = -p0
 patches =
   ${:_profile_base_location_}/disable_openmp_if_single_core.patch#ee553ccaf9dd4bc37374588e49956f62
 
-build-common-options = BINARY="$(uname -m | grep -q x86_64 && echo 64 || echo 32)" NO_STATIC=1 USE_OPENMP=1 USE_THREAD=1
+build-common-options = NO_STATIC=1 USE_OPENMP=1 USE_THREAD=1 DYNAMIC_ARCH=1
 
-# You can specify more options with openblas:build-ext-options parameter.
+# We build for multiple targets with runtime detection of the target CPU but
+# you can specify more options with openblas:build-ext-options parameter.
 # Example :
-# * to build generic binary that supports multiple architecture in one binary
-#   DYNAMIC_ARCH=1
-# * to specify target explicitly
-#   (see https://github.com/xianyi/OpenBLAS/blob/v0.2.15/TargetList.txt )
+# * to specify the oldest model you expect to encounter
+#   (see https://github.com/xianyi/OpenBLAS/blob/v0.2.18/TargetList.txt )
 #   TARGET=HASWELL
 build-ext-options =
 
 # Fortran is required for LAPACK, which is required for matplotlib.
 pre-configure = type gfortran
 
-# First try with auto-detected target and if it fails try TARGET=GENERIC.
 configure-command =
-  make ${:build-common-options} ${:build-ext-options} || (make -j1 clean && make ${:build-common-options} TARGET=GENERIC)
+  make ${:build-common-options} ${:build-ext-options}
 make-options =
   dummy
 make-targets =

component/pypy/buildout.cfg

@@ -23,7 +23,7 @@ make-targets = python setup.py install --install-lib @@LOCATION@@
 [pypy2]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.1-src.tar.bz2
+url = https://downloads.python.org/pypy/pypy2.7-v7.3.1-src.tar.bz2
 md5sum = 7608bd58940ffc5403632c2c786d83bb
 configure-command =
   sed -i '/"_tkinter":/s/^/#/' lib_pypy/tools/build_cffi_imports.py

component/rust/buildout.cfg

+[buildout]
+parts =
+  rustc
+
+extends =
+    ../cmake/buildout.cfg
+    ../curl/buildout.cfg
+    ../git/buildout.cfg
+    ../openssl/buildout.cfg
+    ../pkgconfig/buildout.cfg
+
+
+[rustc]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://static.rust-lang.org/dist/rustc-1.45.2-src.tar.gz
+md5sum = 1c67d7c3f211e49e12e7c20abab08e70
+# --sysconfdir is a workaround for https://github.com/rust-lang/rust/issues/63915
+configure-options = --enable-extended --sysconfdir=@@LOCATION@@/etc/
+environment = 
+    PATH=${cmake:location}/bin/:${curl:location}/bin/:${git:location}/bin/:${pkgconfig:location}/bin/:%(PATH)s
+    PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:

package.json

 {
   "dependencies": {
-    "eclint": "^2.8.1",
     "husky": "^1.1.2",
     "lint-staged": "^7.3.0"
   },
   "lint-staged": {
-    "*": [
-      "eclint fix",
-      "git add"
-    ],
     "*.json": [
       "python2 ./format-json",
       "git add"
     ],
-    "{components,software,stack}/**": [
+    "{component,software,stack}/**": [
       "python -c 'import sys, os.path, subprocess; [subprocess.check_call((\"python2\", \"./update-hash\", buildout_hash)) for buildout_hash in { os.path.join(os.path.dirname(staged), \"buildout.hash.cfg\") for staged in sys.argv[1:]} if os.path.exists(buildout_hash)]'",
       "python -c 'import sys, os.path, subprocess; [subprocess.check_call((\"git\", \"add\", buildout_hash)) for buildout_hash in { os.path.join(os.path.dirname(staged), \"buildout.hash.cfg\") for staged in sys.argv[1:]} if os.path.exists(buildout_hash)]'"
     ]

software/caddy-frontend/CHANGES.caddy_frontend.rst

@@ -3,6 +3,11 @@ Changes
 
 Here are listed the most important changes, which might affect upgrades.
 
+1.0.160 (2020-08-25)
+--------------------
+
+ * haproxy updated from 2.0.15 to 2.0.17 in order to fix issue while accessing inaccessible backends
+
 1.0.159 (2020-07-30)
 --------------------
 

software/caddy-frontend/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg.in
-md5sum = dae3bf6daf851b5610a1a6bd83057b29
+md5sum = 30eb6db0d2df9ffb3a50fb64e73df5be
 
 [template-common]
 filename = instance-common.cfg.in
@@ -22,7 +22,7 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
 
 [template-apache-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = 43b3763d4086283e9c7c886e3b89e54b
+md5sum = 1709085d62f46b22cbe9369fe324bb49
 
 [template-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
@@ -30,7 +30,7 @@ md5sum = 19debfbc27c464f451b1eb5bb5ce3c84
 
 [template-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = 8f518a7b543126ce91fe218255fb201c
+md5sum = da4e3de1c1861459273c6e8d32ead21e
 
 [template-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
@@ -70,7 +70,7 @@ md5sum = 975177dedf677d24e14cede5d13187ce
 
 [template-trafficserver-records-config]
 _update_hash_filename_ = templates/trafficserver/records.config.jinja2
-md5sum = f3f31188de56bb35383335b3219537f4
+md5sum = 56c7e9f5cc92f8c519f48496010e2078
 
 [template-trafficserver-storage-config]
 _update_hash_filename_ = templates/trafficserver/storage.config.jinja2
@@ -106,7 +106,7 @@ md5sum = 8c150e1e6c993708d31936742f3a7302
 
 [caddyprofiledeps-setup]
 filename = setup.py
-md5sum = d9b6476bb0b36cf463fddb00d41dfbaa
+md5sum = 8e1c6c06c09beb921965b3ce98c67c9e
 
 [caddyprofiledeps-dummy]
 filename = caddyprofiledummy.py
@@ -119,3 +119,7 @@ md5sum = f77744e52f9d028c39f99cfbf31eadaf
 [template-backend-haproxy-rsyslogd-conf]
 _update_hash_filename_ = templates/backend-haproxy-rsyslogd.conf.in
 md5sum = be899b04e1aa652ed510f20d4ea523dd
+
+[template-slave-introspection-httpd-nginx]
+_update_hash_filename_ = templates/slave-introspection-httpd-nginx.conf.in
+md5sum = 3067e6ba6c6901821d57d2109517d39c

software/caddy-frontend/common.cfg

@@ -12,6 +12,7 @@ extends =
   ../../component/xz-utils/buildout.cfg
   ../../component/rsyslogd/buildout.cfg
   ../../component/haproxy/buildout.cfg
+  ../../component/nginx/buildout.cfg
 
   ../../stack/caucase/buildout.cfg
 # Monitoring stack (keep on bottom)
@@ -94,6 +95,8 @@ logrotate_base_instance = ${template-logrotate-base:rendered}
 bin_directory = ${buildout:bin-directory}
 
 sixtunnel = ${6tunnel:location}
+nginx = ${nginx-output:nginx}
+nginx_mime = ${nginx-output:mime}
 caddy = ${caddy:output}
 caddy_location = ${caddy:location}
 haproxy_executable = ${haproxy:location}/sbin/haproxy
@@ -110,6 +113,7 @@ kedifa = ${:bin_directory}/kedifa
 kedifa-updater = ${:bin_directory}/kedifa-updater
 kedifa-csr = ${:bin_directory}/kedifa-csr
 xz_location = ${xz-utils:location}
+htpasswd = ${:bin_directory}/htpasswd
 
 monitor_template = ${monitor-template:output}
 template_backend_haproxy_configuration = ${template-backend-haproxy-configuration:target}
@@ -129,6 +133,7 @@ template_trafficserver_records_config = ${template-trafficserver-records-config:
 template_trafficserver_storage_config = ${template-trafficserver-storage-config:target}
 template_trafficserver_logging_config = ${template-trafficserver-logging-config:target}
 template_wrapper = ${template-wrapper:output}
+template_slave_introspection_httpd_nginx = ${template-slave-introspection-httpd-nginx:target}
 
 [template]
 recipe = slapos.recipe.template:jinja2
@@ -198,6 +203,9 @@ mode = 640
 [template-empty]
 <=download-template
 
+[template-slave-introspection-httpd-nginx]
+<=download-template
+
 [template-wrapper]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/templates/wrapper.in

software/caddy-frontend/instance-apache-frontend.cfg.in

@@ -52,6 +52,15 @@ parts =
   promise-backend-haproxy-https
   promise-backend-haproxy-configuration
 
+  slave-introspection-frontend
+  slave-introspection-graceful
+  promise-slave-introspection-https
+  promise-slave-introspection-configuration
+  logrotate-entry-slave-introspection
+
+[caddyprofiledeps]
+recipe = caddyprofiledeps
+
 # Create all needed directories
 [directory]
 recipe = slapos.cookbook:mkdirectory
@@ -83,6 +92,9 @@ csr_id = ${:srv}/csr_id
 caddy-csr_id = ${:etc}/caddy-csr_id
 caddy-csr_id-log = ${:log}/httpd-csr_id
 
+# slave introspection
+slave-introspection-var = ${:var}/slave-introspection
+
 [switch-caddy-softwaretype]
 recipe = slapos.cookbook:softwaretype
 single-default = ${dynamic-custom-personal-template-slave-list:rendered}
@@ -95,6 +107,14 @@ ip-access-certificate = ${self-signed-ip-access:certificate}
 caddy-directory = {{ parameter_dict['caddy_location'] }}
 caddy-ipv6 = {{ instance_parameter['ipv6-random'] }}
 caddy-https-port = ${configuration:port}
+nginx = {{ parameter_dict['nginx'] }}
+nginx_mime = {{ parameter_dict['nginx_mime'] }}
+htpasswd = {{ parameter_dict['htpasswd'] }}
+slave-introspection-template = {{ parameter_dict['template_slave_introspection_httpd_nginx'] }}
+slave-introspection-configuration = ${directory:etc}/slave-introspection-httpd-nginx.conf
+slave-introspection-https-port = ${configuration:slave-introspection-https-port}
+slave-introspection-secure_access = ${slave-introspection-frontend:connection-secure_access}
+slave-introspection-domain = ${slave-introspection-frontend:connection-domain}
 
 [self-signed-ip-access]
 # Self Signed certificate for HTTPS IP accesses to the frontend
@@ -258,6 +278,7 @@ stop-on-error = True
 
 [dynamic-custom-personal-template-slave-list]
 < = jinja2-template-base
+depends = ${caddyprofiledeps:recipe}
 template = {{ parameter_dict['template_slave_list'] }}
 filename = custom-personal-instance-slave-list.cfg
 slave_instance_list = {{ dumps(instance_parameter['slave-instance-list']) }}
@@ -276,6 +297,7 @@ extra-context =
     key caddy_configuration_directory caddy-directory:slave-configuration
     key backend_client_caucase_url :backend-client-caucase-url
     import urlparse_module urlparse
+    import furl_module furl
     key caddy_executable :caddy_executable
     key http_port configuration:plain_http_port
     key https_port configuration:port
@@ -400,6 +422,11 @@ not-found-file = ${caddy-directory:document-root}/${not-found-html:filename}
 master-certificate = ${caddy-directory:master-autocert-dir}/master.pem
 # Communication with ATS
 cache-port = ${trafficserver-variable:input-port}
+# slave instrspection
+slave-introspection-access-log = ${directory:log}/slave-introspection-access.log
+slave-introspection-error-log = ${directory:log}/slave-introspection-error.log
+slave-introspection-pid-file = ${directory:run}/slave-introspection.pid
+slave-introspection-graceful-command = ${slave-introspection-validate:rendered} && kill -HUP $(cat ${:slave-introspection-pid-file})
 
 # BBB: SlapOS Master non-zero knowledge BEGIN
 [get-self-signed-fallback-access]
@@ -892,6 +919,91 @@ extra-context =
     key http_port configuration:plain_http_port
     key https_port configuration:port
 
+[slave-introspection-frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Slave Introspection Frontend {{ instance_parameter['configuration.frontend-name'] }}
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config-url = https://[${slap-network-information:global-ipv6}]:{{ instance_parameter['configuration.slave-introspection-https-port'] }}/
+config-https-only = true
+return = domain secure_access
+
+[slave-introspection-configuration-state]
+<= jinja2-template-base
+template = {{ parameter_dict['template_configuration_state_script'] }}
+rendered = ${directory:bin}/${:_buildout_section_name_}
+mode = 0700
+
+path_list = ${frontend-configuration:slave-introspection-configuration} ${frontend-configuration:ip-access-certificate}
+sha256sum = {{ parameter_dict['sha256sum'] }}
+
+extra-context =
+    key path_list :path_list
+    key sha256sum :sha256sum
+    key signature_file :signature_file
+
+[slave-introspection-configuration-state-graceful]
+<= slave-introspection-configuration-state
+signature_file = ${directory:run}/slave_introspection_graceful_configuration_state_signature
+
+[slave-introspection-configuration-state-validate]
+<= slave-introspection-configuration-state
+signature_file = ${directory:run}/slave_introspection_validate_configuration_state_signature
+
+[slave-introspection-graceful]
+< = jinja2-template-base
+template = {{ parameter_dict['template_graceful_script'] }}
+rendered = ${directory:etc-run}/slave-introspection-safe-graceful
+mode = 0700
+
+extra-context =
+    key graceful_reload_command caddy-configuration:slave-introspection-graceful-command
+    key caddy_configuration_state slave-introspection-configuration-state-graceful:rendered
+
+[slave-introspection-validate]
+<= jinja2-template-base
+template = {{ parameter_dict['template_validate_script'] }}
+rendered = ${directory:bin}/slave-introspection-validate
+mode = 0700
+last_state_file = ${directory:run}/slave_introspection_configuration_last_state
+validate_command = {{ parameter_dict['nginx'] }} -c ${frontend-configuration:slave-introspection-configuration} -t
+extra-context =
+    key validate_command :validate_command
+    key configuration_state_command slave-introspection-configuration-state-validate:rendered
+    key last_state_file :last_state_file
+
+[promise-slave-introspection-configuration]
+<= monitor-promise-base
+module = validate_frontend_configuration
+name = slave-introspection-configuration.py
+config-verification-script = ${promise-slave-introspection-configuration-helper:rendered}
+
+[promise-slave-introspection-configuration-helper]
+< = jinja2-template-base
+template = {{ parameter_dict['template_empty'] }}
+rendered = ${directory:bin}/slave-introspection-read-last-configuration-state
+mode = 0700
+content =
+  #!/bin/sh
+  exit `cat ${slave-introspection-validate:last_state_file}`
+context =
+    key content :content
+
+[promise-slave-introspection-https]
+<= monitor-promise-base
+module = check_port_listening
+name = slave_introspection_https.py
+config-hostname = {{ instance_parameter['ipv6-random'] }}
+config-port = ${frontend-configuration:slave-introspection-https-port}
+
+[logrotate-entry-slave-introspection]
+<= logrotate-entry-base
+name = slave-introspection
+log = ${caddy-configuration:slave-introspection-access-log} ${caddy-configuration:slave-introspection-error-log}
+rotate-num = ${configuration:rotate-num}
+post = kill -USR1 $(cat ${caddy-configuration:slave-introspection-pid-file})
+
 [configuration]
 {%- for key, value in instance_parameter.iteritems() -%}
 {%-   if key.startswith('configuration.') %}

software/caddy-frontend/instance.cfg.in

@@ -41,6 +41,7 @@ template = {{ template_caddy_frontend }}
 filename = instance-caddy-frontend.cfg
 extensions = jinja2.ext.do
 extra-context =
+  import furl_module furl
   section parameter_dict dynamic-template-caddy-frontend-parameters
   raw software_type single-custom-personal
 caucase-jinja2-library = {{ caucase_jinja2_library }}
@@ -124,3 +125,4 @@ configuration.backend-haproxy-http-port = 21080
 configuration.backend-haproxy-https-port = 21443
 configuration.authenticate-to-backend = False
 configuration.rotate-num = 4000
+configuration.slave-introspection-https-port = 22443

software/caddy-frontend/setup.py

@@ -7,6 +7,8 @@ setup(
   name='caddyprofiledeps',
   install_requires=[
     'validators',
+    'furl',
+    'orderedmultidict',
   ],
   entry_points={
     'zc.buildout': [

software/caddy-frontend/software.cfg

@@ -27,3 +27,5 @@ smmap = 0.9.0
 numpy = 1.16.4
 
 websockify = 0.8.0
+furl = 2.1.0
+orderedmultidict = 1.0.1

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -36,6 +36,7 @@ context =
 # empty sections if no slaves are available
 [slave-log-directory-dict]
 [slave-password]
+[slave-htpasswd]
 
 {#- Loop thought slave list to set up slaves #}
 {%- set DEFAULT_PORT = {'http': 80, 'https': 443, '': None} %}
@@ -94,6 +95,7 @@ context =
 {%-   set slave_configuration_section_name = 'slave-instance-%s-configuration' % slave_reference %}
 {%-   set slave_logrotate_section = slave_reference + "-logs" %}
 {%-   set slave_password_section = slave_reference + "-password" %}
+{%-   set slave_htpasswd_section = slave_reference + "-htpasswd" %}
 {%-   set slave_ln_section = slave_reference + "-ln" %}
 {#-   extend parts #}
 {%-   do part_list.extend([slave_ln_section]) %}
@@ -118,7 +120,12 @@ context =
 {%-   do slave_instance.__setitem__('backend_log', slave_parameter_dict.get('backend_log')) %}
 {#-   Add slave log directory to the slave log access dict #}
 {%-   do slave_log_dict.__setitem__(slave_reference, slave_log_folder) %}
-{%-   set slave_log_access_url = 'https://' + slave_reference.lower() + ':${'+ slave_password_section +':passwd}@[' + frontend_configuration.get('caddy-ipv6') + ']:' + frontend_configuration.get('caddy-https-port') + '/' + slave_reference.lower() + '/' %}
+{%-   set furled = furl_module.furl(frontend_configuration['slave-introspection-secure_access']) %}
+{%-   do furled.set(username = slave_reference.lower()) %}
+{%-   do furled.set(password = '${'+ slave_password_section +':passwd}') %}
+{%-   do furled.set(path = slave_reference.lower() + '/') %}
+{#-   We unquote, as furl quotes automatically, but there is buildout value on purpose like ${...:...} in the passwod #}
+{%-   set slave_log_access_url = urlparse_module.unquote(furled.tostr()) %}
 {%-   do slave_publish_dict.__setitem__('log-access', slave_log_access_url) %}
 {%-   do slave_publish_dict.__setitem__('slave-reference', slave_reference) %}
 {%-   do slave_publish_dict.__setitem__('public-ipv4', public_ipv4) %}
@@ -139,6 +146,9 @@ context =
 [slave-password]
 {{ slave_reference }} = {{ '${' + slave_password_section + ':passwd}' }}
 
+[slave-htpasswd]
+{{ slave_reference }} = {{ '${' + slave_htpasswd_section + ':file}' }}
+
 {#-   Set slave logrotate entry #}
 [{{slave_logrotate_section}}]
 <= logrotate-entry-base
@@ -162,6 +172,13 @@ recipe = slapos.cookbook:generate.password
 storage-path = {{caddy_configuration_directory}}/.{{slave_reference}}.passwd
 bytes = 8
 
+[{{ slave_htpasswd_section }}]
+recipe = plone.recipe.command
+stop-on-error = True
+file = {{ caddy_configuration_directory }}/.{{ slave_reference }}.htpasswd
+command = {{ frontend_configuration['htpasswd'] }} -cb ${:file} {{ slave_reference.lower() }} {{ '${' + slave_password_section + ':passwd}' }}
+update-command = ${:command}
+
 {#-   ################################################## #}
 {#-   Set Slave Certificates if needed                   #}
 {#-   Set certificate key for custom configuration       #}
@@ -287,6 +304,7 @@ recipe = slapos.cookbook:mkdirectory
 {%- do part_list.append('slave-log-directories') %}
 
 {%- do part_list.append('caddy-log-access') %}
+{%- do part_list.append('slave-introspection') %}
 {#- ############################################## #}
 {#- ## Prepare virtualhost for slaves using cache  #}
 {#- Define IPv6 to IPV4 tunneling #}
@@ -331,6 +349,35 @@ extra-context =
     section slave_password slave-password
     section parameter_dict caddy-log-access-parameters
 
+[slave-introspection-parameters]
+local-ipv4 = {{ dumps(local_ipv4) }}
+global-ipv6 = {{ dumps(global_ipv6) }}
+https-port = {{ frontend_configuration['slave-introspection-https-port'] }}
+ip-access-certificate = {{ frontend_configuration.get('ip-access-certificate') }}
+nginx-mime = {{ frontend_configuration['nginx_mime'] }}
+access-log = {{ dumps(caddy_configuration['slave-introspection-access-log']) }}
+error-log = {{ dumps(caddy_configuration['slave-introspection-error-log']) }}
+var = {{ directory['slave-introspection-var'] }}
+pid = {{ caddy_configuration['slave-introspection-pid-file'] }}
+
+[slave-introspection-config]
+<= jinja2-template-base
+template = {{ frontend_configuration['slave-introspection-template'] }}
+rendered = {{ frontend_configuration['slave-introspection-configuration'] }}
+extra-context =
+    section slave_log_directory slave-log-directory-dict
+    section slave_htpasswd slave-htpasswd
+    section parameter_dict slave-introspection-parameters
+
+[slave-introspection]
+recipe = slapos.cookbook:wrapper
+command-line = {{ frontend_configuration['nginx'] }}
+  -c ${slave-introspection-config:rendered}
+
+wrapper-path = {{ directory['service'] }}/slave-instrospection-nginx
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+
+
 
 {#- Publish information for the instance #}
 [publish-caddy-information]

software/caddy-frontend/templates/slave-introspection-httpd-nginx.conf.in

+# Access log configuration
+daemon off;
+pid {{ parameter_dict['pid'] }};
+error_log {{ parameter_dict['error-log'] }};
+
+events {
+}
+
+http {
+  include {{ parameter_dict['nginx-mime'] }};
+  server {
+    server_name_in_redirect off;
+    port_in_redirect off;
+    error_log {{ parameter_dict['error-log'] }};
+    access_log {{ parameter_dict['access-log'] }};
+    listen [{{ parameter_dict['global-ipv6'] }}]:{{ parameter_dict['https-port'] }} ssl;
+    listen {{ parameter_dict['local-ipv4'] }}:{{ parameter_dict['https-port'] }} ssl;
+    ssl_certificate {{ parameter_dict['ip-access-certificate'] }};
+    ssl_certificate_key {{ parameter_dict['ip-access-certificate'] }};
+    default_type application/octet-stream;
+    client_body_temp_path {{ parameter_dict['var'] }} 1 2;
+    proxy_temp_path {{ parameter_dict['var'] }} 1 2;
+    fastcgi_temp_path {{ parameter_dict['var'] }} 1 2;
+    uwsgi_temp_path {{ parameter_dict['var'] }} 1 2;
+    scgi_temp_path {{ parameter_dict['var'] }} 1 2;
+  {% for slave, directory in slave_log_directory.iteritems() %}
+    location /{{ slave }} {
+      alias {{ directory }};
+      autoindex on;
+      autoindex_format json;
+      sendfile on;
+      sendfile_max_chunk 1m;
+      auth_basic "Log Access {{ slave }}";
+      auth_basic_user_file "{{ slave_htpasswd[slave] | trim }}";
+    }
+  {% endfor %}
+  }
+}

software/caddy-frontend/templates/trafficserver/records.config.jinja2

@@ -27,6 +27,7 @@ CONFIG proxy.config.http.cache.open_write_fail_action INT 2
 CONFIG proxy.config.body_factory.template_sets_dir STRING  {{ ats_configuration['templates-dir'] }}
 # Support stale-if-error by returning cached content on backend 5xx or unavailability
 CONFIG proxy.config.http.negative_revalidating_enabled INT 1
+CONFIG proxy.config.http.negative_revalidating_lifetime INT 86400
 ##############################################################################
 # Proxy users variables. Docs:
 #    https://docs.trafficserver.apache.org/records.config#proxy-user-variables

software/caddy-frontend/test/test.py

@@ -66,6 +66,7 @@ from cryptography.x509.oid import NameOID
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 from slapos.testing.utils import findFreeTCPPort
+from slapos.testing.utils import getPromisePluginParameterDict
 setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
         os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
@@ -243,44 +244,6 @@ def isHTTP2(domain, ip):
   return 'Using HTTP2, server supports multi-use' in err
 
 
-def getPluginParameterDict(software_path, filepath):
-  """Load the slapos monitor plugin and returns the configuration used by this plugin.
-
-  This allow to check that monitoring plugin are using a proper config.
-  """
-  # This is implemented by creating a wrapper script that loads the plugin wrapper
-  # script and returns its `extra_config_dict`. This might have to be adjusted if
-  # internals of slapos promise plugins change.
-
-  bin_file = os.path.join(software_path, 'bin', 'test-plugin-promise')
-
-  monitor_python_with_eggs = os.path.join(software_path, 'bin', 'monitor-pythonwitheggs')
-  if not os.path.exists(monitor_python_with_eggs):
-    raise ValueError("Monitoring stack's python does not exist at %s" % monitor_python_with_eggs)
-
-  with open(bin_file, 'w') as f:
-    f.write("""#!%s
-import os
-import importlib
-import sys
-import json
-
-filepath = sys.argv[1]
-sys.path[0:0] = [os.path.dirname(filepath)]
-filename = os.path.basename(filepath)
-module = importlib.import_module(os.path.splitext(filename)[0])
-
-print json.dumps(module.extra_config_dict)
-    """ % monitor_python_with_eggs)
-
-  os.chmod(bin_file, 0o755)
-  result = subprocess_output([bin_file, filepath]).strip()
-  try:
-    return json.loads(result)
-  except ValueError, e:
-    raise ValueError("%s\nResult was: %s" % (e, result))
-
-
 class TestDataMixin(object):
   def getTrimmedProcessInfo(self):
     return '\n'.join(sorted([
@@ -711,6 +674,15 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
       result.status_code,
       'While accessing %r of %r the status code was %r' % (
         url, frontend, result.status_code))
+    # check that the result is correct JSON, which allows to access
+    # information about all logs
+    self.assertEqual(
+      'application/json',
+      result.headers['Content-Type']
+    )
+    self.assertEqual(
+      sorted([q['name'] for q in result.json()]),
+      ['access.log', 'backend.log', 'error.log'])
     self.assertEqual(
       httplib.OK,
       requests.get(url + 'access.log', verify=False).status_code
@@ -721,7 +693,8 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     )
     # assert only for few tests, as backend log is not available for many of
     # them, as it's created on the fly
-    for test_name in ['test_url', 'test_auth_to_backend', 'test_compressed_result']:
+    for test_name in [
+      'test_url', 'test_auth_to_backend', 'test_compressed_result']:
       if self.id().endswith(test_name):
         self.assertEqual(
           httplib.OK,
@@ -3569,7 +3542,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
         'check-_monitor-ipv6-test-ipv6-packet-list-test.py'))[0]
     # get promise module and check that parameters are ok
     self.assertEqual(
-      getPluginParameterDict(self.software_path, monitor_file),
+      getPromisePluginParameterDict(monitor_file),
       {
         'frequency': '720',
         'address': 'monitor-ipv6-test'
@@ -3606,7 +3579,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
         'check-_monitor-ipv4-test-ipv4-packet-list-test.py'))[0]
     # get promise module and check that parameters are ok
     self.assertEqual(
-      getPluginParameterDict(self.software_path, monitor_file),
+      getPromisePluginParameterDict(monitor_file),
       {
         'frequency': '720',
         'ipv4': 'true',
@@ -4702,8 +4675,7 @@ class TestRe6stVerificationUrlDefaultSlave(SlaveHttpFrontendTestCase,
     re6st_connectivity_promise_file = re6st_connectivity_promise_list[0]
 
     self.assertEqual(
-      getPluginParameterDict(
-        self.software_path, re6st_connectivity_promise_file),
+      getPromisePluginParameterDict(re6st_connectivity_promise_file),
       {
         'url': 'http://[2001:67c:1254:4::1]/index.html',
       }
@@ -4757,8 +4729,7 @@ class TestRe6stVerificationUrlSlave(SlaveHttpFrontendTestCase,
     re6st_connectivity_promise_file = re6st_connectivity_promise_list[0]
 
     self.assertEqual(
-      getPluginParameterDict(
-        self.software_path, re6st_connectivity_promise_file),
+      getPromisePluginParameterDict(re6st_connectivity_promise_file),
       {
         'url': 'some-re6st-verification-url',
       }
@@ -6462,7 +6433,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
     # get promise module and check that parameters are ok
 
     self.assertEqual(
-      getPluginParameterDict(self.software_path, monitor_file),
+      getPromisePluginParameterDict(monitor_file),
       {
         'frequency': '720',
         'ipv4': 'true',
@@ -6506,7 +6477,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
         'check-_monitor-ipv6-test-unsafe-ipv6-packet-list-test.py'))[0]
     # get promise module and check that parameters are ok
     self.assertEqual(
-      getPluginParameterDict(self.software_path, monitor_file),
+      getPromisePluginParameterDict(monitor_file),
       {
         'frequency': '720',
         'address': '${section:option}\nafternewline ipv6'

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_file_list_log-CADDY.txt

@@ -23,5 +23,7 @@ T-2/var/log/httpd/_enable-http2-true_backend_log
 T-2/var/log/httpd/_enable-http2-true_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -23,5 +23,7 @@ T-2/var/log/httpd/_enable-http2-true_backend_log
 T-2/var/log/httpd/_enable-http2-true_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_file_list_log-CADDY.txt

@@ -23,5 +23,7 @@ T-2/var/log/httpd/_enable-http2-true_backend_log
 T-2/var/log/httpd/_enable-http2-true_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -23,5 +23,7 @@ T-2/var/log/httpd/_enable-http2-true_backend_log
 T-2/var/log/httpd/_enable-http2-true_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_file_list_log-CADDY.txt

@@ -11,5 +11,7 @@ T-2/var/log/frontend-error.log
 T-2/var/log/httpd-csr_id/expose-csr_id.log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_file_list_log-CADDY.txt

@@ -11,5 +11,7 @@ T-2/var/log/frontend-error.log
 T-2/var/log/httpd-csr_id/expose-csr_id.log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_file_list_log-CADDY.txt

@@ -13,5 +13,7 @@ T-2/var/log/httpd/_default_access_log
 T-2/var/log/httpd/_default_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-CADDY.txt

@@ -177,5 +177,7 @@ T-2/var/log/httpd/_url_https-url_backend_log
 T-2/var/log/httpd/_url_https-url_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_plugin-CADDY.txt

@@ -38,5 +38,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestSlave.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestSlaveCiphers.test_file_list_log-CADDY.txt

@@ -17,5 +17,7 @@ T-2/var/log/httpd/_own_ciphers_backend_log
 T-2/var/log/httpd/_own_ciphers_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveCiphers.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestSlaveCiphers.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestSlaveCiphers.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -177,5 +177,7 @@ T-2/var/log/httpd/_url_https-url_backend_log
 T-2/var/log/httpd/_url_https-url_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_plugin-CADDY.txt

@@ -38,5 +38,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibility.test_file_list_log-CADDY.txt

@@ -47,5 +47,7 @@ T-2/var/log/httpd/_type-notebook-ssl_from_slave_kedifa_overrides_backend_log
 T-2/var/log/httpd/_type-notebook-ssl_from_slave_kedifa_overrides_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibility.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibility.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibility.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_file_list_log-CADDY.txt

@@ -14,5 +14,7 @@ T-2/var/log/httpd/_ssl_from_master_kedifa_overrides_master_certificate_backend_l
 T-2/var/log/httpd/_ssl_from_master_kedifa_overrides_master_certificate_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_file_list_log-CADDY.txt

@@ -14,5 +14,7 @@ T-2/var/log/httpd/_ssl_from_master_backend_log
 T-2/var/log/httpd/_ssl_from_master_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/erp5/test/setup.py

@@ -48,6 +48,9 @@ setup(name=name,
         'psutil',
         'requests',
         'mysqlclient',
+        'backports.lzma',
+        'cryptography',
+        'pyOpenSSL',
         ],
       zip_safe=True,
       test_suite='test',

software/erp5/test/test/test_balancer.py

@@ -148,7 +148,6 @@ class TestFrontendXForwardedFor(ERP5InstanceTestCase):
       stderr=subprocess.STDOUT,
     )
     result = caucase_process.communicate()
-    print(result)
     csr_id = result[0].split()[0]
 
     subprocess.check_call(

software/erp5/test/test/test_erp5.py

@@ -50,20 +50,17 @@ class TestPublishedURLIsReachableMixin(object):
     # What happens is that instanciation just create the services, but does not
     # wait for ERP5 to be initialized. When this test run ERP5 instance is
     # instanciated, but zope is still busy creating the site and haproxy replies
-    # with 503 Service Unavailable, sometimes the first request is 404, so we
-    # retry in a loop.
-    # If we can move the "create site" in slapos node instance, then this retry loop
-    # would not be necessary.
+    # with 503 Service Unavailable when zope is not started yet, with 404 when
+    # erp5 site is not created, with 500 when mysql is not yet reachable, so we
+    # retry in a loop until we get a succesful response.
     for i in range(1, 60):
       r = requests.get(url, verify=False)  # XXX can we get CA from caucase already ?
-      if r.status_code in (requests.codes.service_unavailable,
-                           requests.codes.not_found):
+      if r.status_code != requests.codes.ok:
         delay = i * 2
         self.logger.warn("ERP5 was not available, sleeping for %ds and retrying", delay)
         time.sleep(delay)
         continue
-      if r.status_code != requests.codes.ok:
-        r.raise_for_status()
+      r.raise_for_status()
       break
 
     self.assertIn("ERP5", r.text)

software/erp5/test/test/test_mariadb.py

@@ -31,12 +31,19 @@ import json
 import glob
 from six.moves.urllib.parse import urlparse
 import socket
+import sys
 import time
 import contextlib
 import datetime
+import subprocess
+import gzip
 
+from backports import lzma
 import MySQLdb
 
+from slapos.testing.utils import CrontabMixin
+from slapos.testing.utils import getPromisePluginParameterDict
+
 from . import ERP5InstanceTestCase
 from . import setUpModule
 setUpModule  # pyflakes
@@ -56,8 +63,8 @@ class MariaDBTestCase(ERP5InstanceTestCase):
     return {
         'tcpv4-port': 3306,
         'max-connection-count': 5,
-        'max-slowqueries-threshold': 5,
-        'slowest-query-threshold': 10,
+        'max-slowqueries-threshold': 1,
+        'slowest-query-threshold': 0.1,
         # XXX what is this ? should probably not be needed here
         'name': cls.__name__,
         'monitor-passwd': 'secret',
@@ -86,6 +93,88 @@ class MariaDBTestCase(ERP5InstanceTestCase):
     )
 
 
+class TestCrontabs(MariaDBTestCase, CrontabMixin):
+
+  def test_full_backup(self):
+    self._executeCrontabAtDate('mariadb-backup', '2050-01-01')
+    with gzip.open(
+        os.path.join(
+            self.computer_partition_root_path,
+            'srv',
+            'backup',
+            'mariadb-full',
+            '20500101000000.sql.gz',
+        ),
+        'r') as dump:
+      self.assertIn('CREATE TABLE', dump.read())
+
+  def test_logrotate_and_slow_query_digest(self):
+    # slow query digest needs to run after logrotate, since it operates on the rotated
+    # file, so this tests both logrotate and slow query digest.
+
+    # run logrotate a first time so that it create state files
+    self._executeCrontabAtDate('logrotate', '2000-01-01')
+
+    # make two slow queries
+    cnx = self.getDatabaseConnection()
+    with contextlib.closing(cnx):
+      cnx.query("SELECT SLEEP(1.1)")
+      cnx.store_result()
+      cnx.query("SELECT SLEEP(1.2)")
+
+    # slow query crontab depends on crontab for log rotation
+    # to be executed first.
+    self._executeCrontabAtDate('logrotate', '2050-01-01')
+    # this logrotate leaves the log for the day as non compressed
+    rotated_log_file = os.path.join(
+        self.computer_partition_root_path,
+        'srv',
+        'backup',
+        'logrotate',
+        'mariadb_slowquery.log-20500101',
+    )
+    self.assertTrue(os.path.exists(rotated_log_file))
+
+    # then crontab to generate slow query report is executed
+    self._executeCrontabAtDate('generate-mariadb-slow-query-report', '2050-01-01')
+    # and it creates a report for the day
+    slow_query_report = os.path.join(
+        self.computer_partition_root_path,
+        'srv',
+        'monitor',
+        'private',
+        'slowquery_digest',
+        'slowquery_digest.txt-2050-01-01.xz',
+    )
+    with lzma.open(slow_query_report, 'r') as f:
+      # this is the hash for our "select sleep(n)" slow query
+      self.assertIn("ID 0xF9A57DD5A41825CA", f.read())
+
+    # on next day execution of logrotate, log files are compressed
+    self._executeCrontabAtDate('logrotate', '2050-01-02')
+    self.assertTrue(os.path.exists(rotated_log_file + '.xz'))
+    self.assertFalse(os.path.exists(rotated_log_file))
+
+    # there's a promise checking that the threshold is not exceeded
+    # and it reports a problem since we set a threshold of 1 slow query
+    check_slow_query_promise_plugin = getPromisePluginParameterDict(
+        os.path.join(
+            self.computer_partition_root_path,
+            'etc',
+            'plugin',
+            'check-slow-query-pt-digest-result.py',
+        ))
+    with self.assertRaises(subprocess.CalledProcessError) as error_context:
+      subprocess.check_output('faketime 2050-01-01 %s' % check_slow_query_promise_plugin['command'], shell=True)
+    self.assertEqual(
+        error_context.exception.output,
+"""\
+Threshold is lower than expected: 
+Expected total queries : 1.0 and current is: 2
+Expected slowest query : 0.1 and current is: 1
+""")
+
+
 class TestMariaDB(MariaDBTestCase):
   def test_utf8_collation(self):
     cnx = self.getDatabaseConnection()

software/erp5testnode/testsuite/dummy/software.cfg

@@ -11,7 +11,7 @@ parts =
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
 output = ${buildout:directory}/template.cfg
-md5sum = efd3b712a2294207f265a9c45648d5cf
+md5sum = 9fccb9600a085691ab0f9a20cd615c01
 mode = 0644
 
 [versions]

software/erp5testnode/testsuite/instance-resilient-test.cfg.jinja2

@@ -56,8 +56,12 @@ config-cpu-usage-ratio = 1
 config-software-root = ${buildout:directory}/../../soft
 config-buildout-shared-folder = ${buildout:directory}/../../shared
 config-no-ipv4-frontend = true
-# XXX hardcoded
-#config-frontend-domain = google.com
+
+# Use same repository / branch for the software installed in slaprunner
+# than the tested slaprunner itself.
+config-slapos-repository = {{ slapos_repository_url }}
+config-slapos-reference = {{ slapos_repository_branch }}
+
 # XXX Hack to deploy Root Instance on the same computer as the type-test Instance
 sla-computer_guid = ${slap-connection:computer-id}
 return = backend-url

software/erp5testnode/testsuite/slaprunner.cfg

@@ -7,7 +7,16 @@ parts += template-erp5testnode
 
 [template-resilient-test]
 filename = instance-resilient-test.cfg.jinja2 
-md5sum = cf34be404bdd93a72df9c593f8762cb2
+md5sum = be4adbc196f07f5e39ca78401dfa53ec
+
+# We have to use an extra level of indentation here because this is substituted
+# during software buildout to generate instance buildout, but the
+# slapos.recipe.template recipe doing the substitution does string replacements
+# without knowledge of the buildout syntax, so we want the second line to be
+# indented in the final generated instance buildout.
+extra-context =
+    raw slapos_repository_url ${slapos.cookbook-repository:repository}
+        raw slapos_repository_branch ${slapos.cookbook-repository:branch}
 
 [exporter-default-configuration]
 # Define shorter interaction to speed up tests

software/kvm/README.rst

@@ -18,9 +18,8 @@ KVM instance (1GB of RAM, 10GB of SSD, one core)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Note that the KVM instance will try to request a frontend slave instance in order
-to be accessible from IPv4.
+to be accessible from IPv4::
 
-::
   myawesomekvm = request(
       software_release=kvm,
       partition_reference="My awesome KVM",
@@ -92,16 +91,15 @@ KVM instance parameters:
 - keyboard-layout-language (default: fr)
     Change keyboard layout language (Change to en-us if you face some bad bihaviors)
     Language list: ['ar', 'da', 'de', 'de-ch', 'en-gb', 'en-us', 'es', 'et', 'fi',
-      'fo', 'fr', 'fr-be', 'fr-ca', 'fr-ch', 'hr', 'hu', 'is', 'it', 'ja', 'lt',
-      'lv', 'mk', 'nl', 'nl-be', 'no', 'pl', 'pt', 'pt-br', 'ru', 'sl', 'sv',
-      'th', 'tr']
+    'fo', 'fr', 'fr-be', 'fr-ca', 'fr-ch', 'hr', 'hu', 'is', 'it', 'ja', 'lt',
+    'lv', 'mk', 'nl', 'nl-be', 'no', 'pl', 'pt', 'pt-br', 'ru', 'sl', 'sv',
+    'th', 'tr']
 
 Resilient KVM instance
-~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~
 
-Like KVM instance, but backed-up (with history) in two places.
+Like KVM instance, but backed-up (with history) in two places::
 
-::
   kvm = 'https://lab.nexedi.com/nexedi/slapos/raw/slapos-0.188/software/kvm/software.cfg'
   myresilientkvm = request(
       software_release=kvm,
@@ -119,3 +117,18 @@ files for more instance parameters (cpu-count, ram-size, disk-size, specific loc
 
 Then, if you want one of the two clones to takeover, you need to login into
 the hosting machine, go to the partition of the clone, and invoke bin/takeover.
+
+Technical notes
+---------------
+
+Updating boot-image-url-select
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ * download the new OS installation image
+ * calculate it's sha512sum and store as <SHA512>
+ * calculate it's md5sum and store as <MD5>
+ * upload it to shacache
+ * construct download url: ``https://shacache.nxdcdn.com/<SHA512>#<MD5>``
+ * update the ``boot-image-url-select`` in:
+    * ``instance-kvm-input-schema.json``
+    * ``instance-kvm-cluster-input-schema.json``

software/kvm/buildout.hash.cfg

@@ -19,11 +19,11 @@ md5sum = e6d5c7bb627b4f1d3e7c99721b7c58fe
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = 50e78a2a34efe09afab161ae1c1efd46
+md5sum = 23493c541efef97ac5fe435114910b8e
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in
-md5sum = a4788112008cd0b38a57cd28f7252fbd
+md5sum = bdf8549a76ec61e125d51a05e611e004
 
 [template-kvm-resilient]
 filename = instance-kvm-resilient.cfg.jinja2
@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
 
 [template-kvm-run]
 filename = template/template-kvm-run.in
-md5sum = dd1f581f34cf5a0b627576771347c710
+md5sum = 4a6f149177a453a13436f320f6841518
 
 [template-kvm-controller]
 filename = template/kvm-controller-run.in
@@ -87,4 +87,4 @@ md5sum = 7e4b54f8172c364bd12d28b07f8b1600
 
 [image-download-config-creator]
 _update_hash_filename_ = template/image-download-config-creator.py
-md5sum = 7f6cd75096695922fddbba1c9292cef5
+md5sum = 54261e418ab9860efe73efd514c4d47f

software/kvm/instance-kvm-cluster-input-schema.json

@@ -42,12 +42,6 @@
           "type": "string",
           "format": "uri",
           "default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg"
-        },
-        "image-url-list": {
-          "title": "List of URLs images to download",
-          "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
-          "type": "string",
-          "textarea": "true"
         }
       },
       "type": "object"
@@ -508,6 +502,60 @@
               "description": "If the VM of cluster doesn't run Ansible and report status to this SlapOS instances, then this allow to disable ansible promise so your instance will not fail to check ansible promise.",
               "type": "boolean",
               "default": false
+            },
+            "boot-image-url-list": {
+              "title": "Boot image list",
+              "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
+              "type": "string",
+              "textarea": "true"
+            },
+            "boot-image-url-select": {
+              "title": "Boot image",
+              "type": "array",
+              "oneOf": [
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/0a6aee1d9aafc1ed095105c052f9fdd65ed00ea9274188c9cd0072c8e6838ab40e246d45a1e6956d74ef1b04a1fc042151762f25412e9ff0cbf49418eef7992e#a3ebc76aec372808ad80000108a2593a"
+                  ],
+                  "title": "Debian Buster 10.5 netinst x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/ce5ddfdbdaccdf929b7fe321212356347d82a02f6b7733427282b416f113d91e587682b003e9d376ac189c3b731595c50c236962aadf2720c16d9f36913577c0#23bf2a2d60271e553e63525e794415f1"
+                  ],
+                  "title": "Centos 8.2004 Minimal x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/302c990c6d69575ff24c96566e5c7e26bf36908abb0cd546e22687c46fb07bf8dba595bf77a9d4fd9ab63e75c0437c133f35462fd41ea77f6f616140cd0e5e6a#f3a306f40e4a313fb5a584d73b3dee8f"
+                  ],
+                  "title": "Ubuntu Focal 20.04.1 Live Server x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/6635269a7eb6fbd6b85fda40cd94f14a27bf53cb1fc82ffcce9fe386a025a43e1ab681db7e8cec50416bfbfc90262f0d95273686a101c74b3f17646f0a34c85b#3708a59af6cf820a95cafe0ae73ac399"
+                  ],
+                  "title": "openSUSE Leap 15.2 NET x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/fc17e8c6ae0790162f4beb8fa6226d945cff638429588999b3a08493ff27b280dc2939fba825ae04be1d9082ea8d7c3c002c5e4c39fbbcf88b8ab5104619e28a#ebcdb2223a77f098af3923fe1fa180aa"
+                  ],
+                  "title": "Arch Linux 2020.09.01 x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/c5a511f349a1146b615e6fab9c24f9be4362046adcf24f0ff82c470d361fac5f6628895e2110ebf8ff87db49d4c413a0a332699da6b1bec64275e0c17a15b999#ca7a1e555c04b4d9a549065fa2ddf713"
+                  ],
+                  "title": "Fedora Server 32-1.6 netinst x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/6c355def68b3c0427f21598cb054ffc893568902f205601ac60f192854769b31bc9cff8eeb6ce99ef975a8fb887d8d3e56fc6cd5ea5cb4b3bba1175c520047cb#57088b77f795ca44b00971e44782ee23"
+                  ],
+                  "title": "FreeBSD 12.1 RELEASE bootonly x86_64"
+                }
+              ]
             }
           },
           "type": "object"

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -127,9 +127,13 @@ config-document-host = ${apache-conf:ip}
 config-document-port = ${apache-conf:port}
 config-document-path = ${hash-code:passwd}
 config-keyboard-layout-language = {{ dumps(kvm_parameter_dict.get('keyboard-layout-language', 'fr')) }}
-{%- if 'image-url-list' in kvm_parameter_dict %}
+{%- if 'boot-image-url-list' in kvm_parameter_dict %}
 {#- play nice: if parameter was not constructed by the original request, do not send it at all #}
-config-image-url-list = {{ kvm_parameter_dict['image-url-list'] }}
+config-boot-image-url-list = {{ kvm_parameter_dict['boot-image-url-list'] }}
+{%- endif %}
+{%- if 'boot-image-url-select' in kvm_parameter_dict %}
+{#- play nice: if parameter was not constructed by the original request, do not send it at all #}
+config-boot-image-url-select = {{ kvm_parameter_dict['boot-image-url-select'] }}
 {%- endif %}
 config-type = cluster
 

software/kvm/instance-kvm-input-schema.json

 {
   "type": "object",
-  "$schema": "http://json-schema.org/draft-04/schema",
+  "$schema": "http://json-schema.org/draft-06/schema",
   "title": "Input Parameters",
   "properties": {
     "enable-device-hotplug": {
@@ -366,11 +366,59 @@
       "format": "uri",
       "default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg"
     },
-    "image-url-list": {
-      "title": "List of URLs images to download",
+    "boot-image-url-list": {
+      "title": "Boot image list",
       "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
       "type": "string",
       "textarea": "true"
+    },
+    "boot-image-url-select": {
+      "title": "Boot image",
+      "type": "array",
+      "oneOf": [
+        {
+          "const": [
+            "https://shacache.nxdcdn.com/0a6aee1d9aafc1ed095105c052f9fdd65ed00ea9274188c9cd0072c8e6838ab40e246d45a1e6956d74ef1b04a1fc042151762f25412e9ff0cbf49418eef7992e#a3ebc76aec372808ad80000108a2593a"
+          ],
+          "title": "Debian Buster 10.5 netinst x86_64"
+        },
+        {
+          "const": [
+            "https://shacache.nxdcdn.com/ce5ddfdbdaccdf929b7fe321212356347d82a02f6b7733427282b416f113d91e587682b003e9d376ac189c3b731595c50c236962aadf2720c16d9f36913577c0#23bf2a2d60271e553e63525e794415f1"
+          ],
+          "title": "Centos 8.2004 Minimal x86_64"
+        },
+        {
+          "const": [
+            "https://shacache.nxdcdn.com/302c990c6d69575ff24c96566e5c7e26bf36908abb0cd546e22687c46fb07bf8dba595bf77a9d4fd9ab63e75c0437c133f35462fd41ea77f6f616140cd0e5e6a#f3a306f40e4a313fb5a584d73b3dee8f"
+          ],
+          "title": "Ubuntu Focal 20.04.1 Live Server x86_64"
+        },
+        {
+          "const": [
+            "https://shacache.nxdcdn.com/6635269a7eb6fbd6b85fda40cd94f14a27bf53cb1fc82ffcce9fe386a025a43e1ab681db7e8cec50416bfbfc90262f0d95273686a101c74b3f17646f0a34c85b#3708a59af6cf820a95cafe0ae73ac399"
+          ],
+          "title": "openSUSE Leap 15.2 NET x86_64"
+        },
+        {
+          "const": [
+            "https://shacache.nxdcdn.com/fc17e8c6ae0790162f4beb8fa6226d945cff638429588999b3a08493ff27b280dc2939fba825ae04be1d9082ea8d7c3c002c5e4c39fbbcf88b8ab5104619e28a#ebcdb2223a77f098af3923fe1fa180aa"
+          ],
+          "title": "Arch Linux 2020.09.01 x86_64"
+        },
+        {
+          "const": [
+            "https://shacache.nxdcdn.com/c5a511f349a1146b615e6fab9c24f9be4362046adcf24f0ff82c470d361fac5f6628895e2110ebf8ff87db49d4c413a0a332699da6b1bec64275e0c17a15b999#ca7a1e555c04b4d9a549065fa2ddf713"
+          ],
+          "title": "Fedora Server 32-1.6 netinst x86_64"
+        },
+        {
+          "const": [
+            "https://shacache.nxdcdn.com/6c355def68b3c0427f21598cb054ffc893568902f205601ac60f192854769b31bc9cff8eeb6ce99ef975a8fb887d8d3e56fc6cd5ea5cb4b3bba1175c520047cb#57088b77f795ca44b00971e44782ee23"
+          ],
+          "title": "FreeBSD 12.1 RELEASE bootonly x86_64"
+        }
+      ]
     }
   }
 }

software/kvm/template/image-download-config-creator.py

@@ -24,7 +24,18 @@ if __name__ == "__main__":
     image_number = 0
     data = fh.read()
     configuration_dict['config-md5sum'] = hashlib.md5(data).hexdigest()
-    for entry in data.decode('utf-8').split():
+    if source_configuration.endswith('.json'):
+      data = data.strip()
+      data_list = []
+      if len(data):
+        try:
+          data_list = json.loads(data)
+        except Exception:
+          error_list.append('ERR: Data is not a JSON')
+          data_list = []
+    else:
+      data_list = data.decode('utf-8').split()
+    for entry in data_list:
       split_entry = entry.split('#')
       if len(split_entry) != 2:
         error_list.append('ERR: entry %r is incorrect' % (entry,))

software/kvm/template/template-kvm-run.in

@@ -99,7 +99,8 @@ enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lowe
 
 logfile = '{{ parameter_dict.get("log-file") }}'
 
-image_url_list_json_config = '{{ parameter_dict.get("image-url-list-json-config") }}'
+boot_image_url_list_json_config = '{{ parameter_dict.get("boot-image-url-list-json-config") }}'
+boot_image_url_select_json_config = '{{ parameter_dict.get("boot-image-url-select-json-config") }}'
 
 if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false':
   opener = FancyURLopener(context=ssl._create_unverified_context())
@@ -370,25 +371,41 @@ for nbd_ip, nbd_port in nbd_list:
           '-drive',
           'file=nbd:[%s]:%s,media=cdrom' % (nbd_ip, nbd_port)])
 else:
-  image_url_list_used = False
-  if image_url_list_json_config:
-    # Support image-url-list
-    with open(image_url_list_json_config) as fh:
+  index = 0
+  if boot_image_url_select_json_config:
+    # Support boot-image-url-select
+    with open(boot_image_url_select_json_config) as fh:
       image_config = json.load(fh)
     if image_config['error-amount'] == 0:
       for image in sorted(image_config['image-list'], key=lambda k: k['link']):
         link = os.path.join(image_config['destination-directory'], image['link'])
         if os.path.exists(link) and os.path.islink(link):
-          image_url_list_used = True
           kvm_argument_list.extend([
-            '-drive',
-            'file=%s,media=cdrom' % (link,)
+            '-drive', 'file=%s,media=cdrom,if=none,id=cdrom%s' % (link, index),
+            '-device', 'virtio-scsi-pci,id=scsi%s' % (index,),
+            '-device', 'scsi-cd,bus=scsi%s.0,drive=cdrom%s' % (index, index)
           ])
-  if not image_url_list_used:
-    # If no NBD is specified/available not downloadable image: use internal disk image
-    kvm_argument_list.extend([
-        '-drive', 'file=%s,media=cdrom' % default_cdrom_iso
-    ])
+          index += 1
+  if boot_image_url_list_json_config:
+    # Support boot-image-url-list
+    with open(boot_image_url_list_json_config) as fh:
+      image_config = json.load(fh)
+    if image_config['error-amount'] == 0:
+      for image in sorted(image_config['image-list'], key=lambda k: k['link']):
+        link = os.path.join(image_config['destination-directory'], image['link'])
+        if os.path.exists(link) and os.path.islink(link):
+          kvm_argument_list.extend([
+            '-drive', 'file=%s,media=cdrom,if=none,id=cdrom%s' % (link, index),
+            '-device', 'virtio-scsi-pci,id=scsi%s' % (index,),
+            '-device', 'scsi-cd,bus=scsi%s.0,drive=cdrom%s' % (index, index)
+          ])
+          index += 1
+  # Always add by default the default image
+  kvm_argument_list.extend([
+      '-drive', 'file=%s,media=cdrom,if=none,id=cdrom%s' % (default_cdrom_iso, index),
+      '-device', 'virtio-scsi-pci,id=scsi%s' % (index,),
+      '-device', 'scsi-cd,bus=scsi%s.0,drive=cdrom%s' % (index, index)
+  ])
 
 
 print('Starting KVM: \n %s' % ' '.join(kvm_argument_list))

software/monitor/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = 1b5d5a72d0d0e3156770ce8ecc5d19ce
+md5sum = 9ddae686379e8d747410c1adf82b47d6
 
 [template-monitor]
 _update_hash_filename_ = instance-monitor.cfg.jinja2
@@ -30,7 +30,7 @@ md5sum = a57106ee88ff3295b9ffce84105da79b
 
 [template-monitor-edgebot]
 _update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
-md5sum = c535f4df6388cdc3f6c1df2a91d6dd53
+md5sum = c1885a42aadd45bab3185a53258d4ff4
 
 [network-bench-cfg]
 filename = network_bench.cfg.in

software/monitor/instance-edgetest-input-schema.json

@@ -13,6 +13,12 @@
       "description": "Default HTTP code to check against (default: 200).",
       "type": "string"
     },
+    "check-http-header-dict": {
+      "default": "{}",
+      "title": "HTTP header dict to check",
+      "description": "JSON dict of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}",
+      "type": "object"
+    },
     "check-frontend-ip": {
       "default": "",
       "title": "Default space separated list of Frontend IPs to check",

software/monitor/instance-edgetest-slave-input-schema.json

@@ -12,6 +12,12 @@
       "description": "HTTP code to check against (default: comes from master partition).",
       "type": "string"
     },
+    "check-http-header-dict": {
+      "default": "Master default",
+      "title": "HTTP header dict to check",
+      "description": "JSON dict of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}",
+      "type": "object"
+    },
     "check-frontend-ip": {
       "default": "Master default",
       "title": "Space separated list of Frontend IPs to check",

software/monitor/instance-monitor-edgebot.cfg.jinja2

@@ -13,6 +13,9 @@
 {%-   if 'check-status-code' not in slave %}
 {%-     do slave.__setitem__('check-status-code', CONFIGURATION['check-status-code']) %}
 {%-   endif %}
+{%-   if 'check-http-header-dict' not in slave %}
+{%-     do slave.__setitem__('check-http-header-dict', CONFIGURATION['check-http-header-dict']) %}
+{%-   endif %}
 {%-   if 'check-certificate-expiration-days' not in slave %}
 {%-     do slave.__setitem__('check-certificate-expiration-days', CONFIGURATION['check-certificate-expiration-days']) %}
 {%-   endif %}
@@ -48,6 +51,7 @@ name = {{ safe_name }}.py
 config-report = http_query
 config-url = {{ slave['url'] }}
 config-status-code = {{ slave['check-status-code'] }}
+config-http-header-dict = {{ slave['check-http-header-dict'] }}
 config-certificate-expiration-days = {{ slave['check-certificate-expiration-days'] }}
 config-failure-amount = {{ slave['failure-amount'] }}
 config-maximum-elapsed-time = {{ slave['check-maximum-elapsed-time'] }}

software/monitor/instance.cfg

@@ -76,6 +76,7 @@ key = $${slap-connection:key-file}
 cert = $${slap-connection:cert-file}
 # Defaults
 configuration.check-status-code = 200
+configuration.check-http-header-dict = {}
 configuration.nameserver =
 configuration.check-frontend-ip =
 configuration.check-certificate-expiration-days = 15

software/monitor/software.cfg

@@ -78,9 +78,9 @@ scripts =
 [versions]
 slapos.recipe.template = 4.4
 
-surykatka = 0.4.2
+surykatka = 0.5.0
 
-# For surykatka 0.4.2
+# For surykatka 0.5.0
 click = 7.0
 certifi = 2019.11.28
 chardet = 3.0.4

software/monitor/test/test.py

@@ -470,6 +470,90 @@ URL =
     )
 
 
+class TestEdgeCheckHTTPHeaderDict(EdgeSlaveMixin, SlapOSInstanceTestCase):
+  surykatka_dict = {
+    2: {'expected_ini': """[SURYKATKA]
+INTERVAL = 120
+TIMEOUT = 4
+SQLITE = %(db_file)s
+URL =
+  https://www.erp5.com/
+  https://www.erp5.org/"""}
+  }
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'check-http-header-dict':
+        '{"B": "BBB"}',
+    }
+
+  def assertSurykatkaPromises(self):
+    self.assertPromiseContent(
+      'http-query-backend-http-header-promise.py',
+      "'ip-list': ''")
+    self.assertPromiseContent(
+      'http-query-backend-http-header-promise.py',
+      "'report': 'http_query'")
+    self.assertPromiseContent(
+      'http-query-backend-http-header-promise.py',
+      "'status-code': '200'")
+    self.assertPromiseContent(
+      'http-query-backend-http-header-promise.py',
+      "'http-header-dict': '{\"A\": \"AAA\"}'")
+    self.assertPromiseContent(
+      'http-query-backend-http-header-promise.py',
+      "'certificate-expiration-days': '15'")
+    self.assertPromiseContent(
+      'http-query-backend-http-header-promise.py',
+      "'url': 'https://www.erp5.org/'")
+    self.assertPromiseContent(
+      'http-query-backend-http-header-promise.py',
+      "'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
+    )
+    self.assertPromiseContent(
+      'http-query-backend-http-header-promise.py',
+      "'failure-amount': '2'"
+    )
+
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'ip-list': ''")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'report': 'http_query'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'status-code': '200'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'http-header-dict': '{\"B\": \"BBB\"}'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'certificate-expiration-days': '15'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'url': 'https://www.erp5.com/'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
+    )
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'failure-amount': '2'"
+    )
+
+  def requestEdgetestSlaves(self):
+    self.requestEdgetestSlave(
+      'backend',
+      {'url': 'https://www.erp5.com/'},
+    )
+    self.requestEdgetestSlave(
+      'backend-http-header',
+      {'url': 'https://www.erp5.org/', 'check-http-header-dict': '{"A": "AAA"}'},
+    )
+
+
 class TestEdgeCheckCertificateExpirationDays(
   EdgeSlaveMixin, SlapOSInstanceTestCase):
   surykatka_dict = {