Update Release Candidate

README.update-hash.rst

+update-hash
+===========
+
+``update-hash`` is a tool to assist software release developers in the management of ``buildout.hash.cfg`` files.
+
+A lot of recipes which uses hashing for referenced files. Updating the hash results in part uninstallation and installation, which is desired behaviour, as the file might have to be redownloaded. By using ``update-hash`` with ``buildout.hash.cfg`` the developer does not have to do the calculations and updates manually, just calling the tool is enough.
+
+Generally each Buildout profile which references some file shall use this approach to improve development process and minimise risk of using incorrect data from such entires.
+
+Working with ``buildout.hash.cfg``
+----------------------------------
+
+``buildout.hash.cfg`` files are buildout-style simplified configparser files to have a easy way to update MD5 hashes of provided files for download. They look like::
+
+  [section]
+  md5sum = <hash>
+  filename = <relative-path>
+
+Where ``<hash>`` is an automatically calculated checksum of ``<relative-path>``.
+
+Then ``buildout.hash.cfg`` can be included in software profile by ``extends`` of ``[buildout]`` section, and the section's ``md5sum`` and ``filename`` can be used.
+
+Special cases of ``filename`` key
+---------------------------------
+
+In case if section recipe has special unwanted behaviour for ``filename`` field the ``_update_hash_filename_`` key can be used like::
+
+  [section]
+  md5sum = <hash>
+  _update_hash_filename_ = <relative-path>
+
+Working with ``update-hash``
+----------------------------
+
+In order to update the ``buildout.hash.cfg`` one just need to call ``update-hash`` while being in the directory containing the file.

component/golang/buildout.cfg

@@ -32,37 +32,10 @@ environment-extra =
 make-targets= cd src && ./make.bash && cp -alf .. ${:location}
 
 
-[golang19]
-<= golang-common
-url = https://dl.google.com/go/go1.9.7.src.tar.gz
-md5sum = 3c2cf876ed6612a022574a565206c6ea
-
-# go1.9 needs go1.4 to bootstrap
-environment-extra =
-  GOROOT_BOOTSTRAP=${golang14:location}
-
-[golang1.10]
-<= golang-common
-url = https://dl.google.com/go/go1.10.3.src.tar.gz
-md5sum = d15dfb264105c5e84fbe33f4a4aa5021
-
-# go1.10 needs go1.4 to bootstrap
-environment-extra =
-  GOROOT_BOOTSTRAP=${golang14:location}
-
-[golang1.11]
-<= golang-common
-url = https://dl.google.com/go/go1.11.4.src.tar.gz
-md5sum = a77697673215be465d1b583680ef2318
-
-# go1.11 needs go1.4 to bootstrap
-environment-extra =
-  GOROOT_BOOTSTRAP=${golang14:location}
-
 [golang1.12]
 <= golang-common
-url = https://dl.google.com/go/go1.12.9.src.tar.gz
-md5sum = 6132109d4050da349eadc9f7b0304ef4
+url = https://golang.org/dl/go1.12.17.src.tar.gz
+md5sum = 6b607fc795391dc609ffd79ebf41f080
 
 # go1.12 needs go1.4 to bootstrap
 environment-extra =
@@ -70,8 +43,8 @@ environment-extra =
 
 [golang1.13]
 <= golang-common
-url = https://dl.google.com/go/go1.13.9.src.tar.gz
-md5sum = 4ad8b04f962be93a32f3021e6f35b3b9
+url = https://golang.org/dl/go1.13.15.src.tar.gz
+md5sum = 4f4af14d88352a62761a9dcedf863ac0
 
 # go1.13 needs go1.4 to bootstrap
 environment-extra =
@@ -79,8 +52,8 @@ environment-extra =
 
 [golang1.14]
 <= golang-common
-url = https://dl.google.com/go/go1.14.3.src.tar.gz
-md5sum = 6b1fb42d219e2ea8925002013c76d4c7
+url = https://golang.org/dl/go1.14.9.src.tar.gz
+md5sum = 6f6dd1377421d27ca4bb607283b31738
 
 # go1.14 needs go1.4 to bootstrap
 environment-extra =
@@ -118,7 +91,7 @@ bin	= ${:directory}/bin
 depends = ${gowork.goinstall:recipe}
 
 # go version used for the workspace (possible to override in applications)
-golang  = ${golang1.10:location}
+golang  = ${golang1.14:location}
 
 # no special build flags by default
 buildflags =

component/helloweb/buildout.cfg

@@ -25,7 +25,7 @@ parts =
 install =
     lab.nexedi.com/nexedi/helloweb/go/...
 
-golang  = ${golang1.12:location}
+golang  = ${golang1.14:location}
 
 # -*- go -*-
 [helloweb-go]

component/nginx/buildout.cfg

@@ -13,8 +13,8 @@ parts = nginx-output
 [nginx-common]
 recipe = slapos.recipe.cmmi
 shared = false
-url = https://nginx.org/download/nginx-1.17.6.tar.gz
-md5sum = 55022aa5715386c994f6773478822853
+url = https://nginx.org/download/nginx-1.19.2.tar.gz
+md5sum = 3dc55f6451ed6f819f1c796f4e5e9617
 
 [nginx]
 <= nginx-common
@@ -26,8 +26,8 @@ configure-options=
   --with-http_realip_module
   --with-mail
   --with-mail_ssl_module
-  --with-ld-opt="-L ${openssl-1.0:location}/lib -L ${pcre:location}/lib -L ${zlib:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib"
-  --with-cc-opt="-I ${openssl-1.0:location}/include -I ${pcre:location}/include -I ${zlib:location}/include"
+  --with-ld-opt="-L ${openssl:location}/lib -L ${pcre:location}/lib -L ${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib"
+  --with-cc-opt="-I ${openssl:location}/include -I ${pcre:location}/include -I ${zlib:location}/include"
 
 [nginx-dav-ext-module]
 recipe = slapos.recipe.build:download-unpacked
@@ -46,8 +46,8 @@ configure-options =
   --with-mail
   --with-mail_ssl_module
   --error-log-path=var/log/nginx.error.log
-  --with-ld-opt=" -L ${libexpat:location}/lib -L ${openssl-1.0:location}/lib -L ${pcre:location}/lib -L ${zlib:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib"
-  --with-cc-opt="-I ${libexpat:location}/include -I ${openssl-1.0:location}/include -I ${pcre:location}/include -I ${zlib:location}/include"
+  --with-ld-opt=" -L ${libexpat:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib -L ${zlib:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib"
+  --with-cc-opt="-I ${libexpat:location}/include -I ${openssl:location}/include -I ${pcre:location}/include -I ${zlib:location}/include"
   --with-http_dav_module 
   --add-module='${nginx-dav-ext-module:location}'
 
@@ -73,8 +73,8 @@ strip-top-level-dir = true
 configure-options=
   --with-ipv6
   --with-http_ssl_module
-  --with-ld-opt="-L ${zlib:location}/lib -L ${openssl-1.0:location}/lib -L ${pcre:location}/lib  -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib"
-  --with-cc-opt="-I ${pcre:location}/include -I ${openssl-1.0:location}/include -I ${zlib:location}/include"
+  --with-ld-opt="-L ${zlib:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib  -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib"
+  --with-cc-opt="-I ${pcre:location}/include -I ${openssl:location}/include -I ${zlib:location}/include"
   --add-module=${hexaglobe-nginx-module:location}/sub_module
 # --add-module=${hexaglobe-nginx-module:location}/nginx-upstream-fair
 
@@ -91,8 +91,8 @@ configure-options=
   --with-http_ssl_module
   --with-http_v2_module
   --with-http_gzip_static_module
-  --with-ld-opt="-L ${zlib:location}/lib -L ${openssl-1.0:location}/lib -L ${pcre:location}/lib  -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib"
-  --with-cc-opt="-I ${pcre:location}/include -I ${openssl-1.0:location}/include -I ${zlib:location}/include -Wno-error"
+  --with-ld-opt="-L ${zlib:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib  -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib"
+  --with-cc-opt="-I ${pcre:location}/include -I ${openssl:location}/include -I ${zlib:location}/include -Wno-error"
   --add-module=${nginx-push-stream-module:location}
 
 [nginx-push-stream-output]

component/nodejs/buildout.cfg

@@ -14,6 +14,12 @@ parts =
 [nodejs]
 <= nodejs-8.9.4
 
+[nodejs-12.18.3]
+<= nodejs-base
+openssl_location = ${openssl:location}
+version = v12.18.3
+md5sum = 28bf6a4d98b238403fa58a0805f4a979
+
 [nodejs-10.19.0]
 <= nodejs-base
 openssl_location = ${openssl:location}

component/openblas/buildout.cfg

@@ -15,23 +15,21 @@ patch-options = -p0
 patches =
   ${:_profile_base_location_}/disable_openmp_if_single_core.patch#ee553ccaf9dd4bc37374588e49956f62
 
-build-common-options = BINARY="$(uname -m | grep -q x86_64 && echo 64 || echo 32)" NO_STATIC=1 USE_OPENMP=1 USE_THREAD=1
+build-common-options = NO_STATIC=1 USE_OPENMP=1 USE_THREAD=1 DYNAMIC_ARCH=1
 
-# You can specify more options with openblas:build-ext-options parameter.
+# We build for multiple targets with runtime detection of the target CPU but
+# you can specify more options with openblas:build-ext-options parameter.
 # Example :
-# * to build generic binary that supports multiple architecture in one binary
-#   DYNAMIC_ARCH=1
-# * to specify target explicitly
-#   (see https://github.com/xianyi/OpenBLAS/blob/v0.2.15/TargetList.txt )
+# * to specify the oldest model you expect to encounter
+#   (see https://github.com/xianyi/OpenBLAS/blob/v0.2.18/TargetList.txt )
 #   TARGET=HASWELL
 build-ext-options =
 
 # Fortran is required for LAPACK, which is required for matplotlib.
 pre-configure = type gfortran
 
-# First try with auto-detected target and if it fails try TARGET=GENERIC.
 configure-command =
-  make ${:build-common-options} ${:build-ext-options} || (make -j1 clean && make ${:build-common-options} TARGET=GENERIC)
+  make ${:build-common-options} ${:build-ext-options}
 make-options =
   dummy
 make-targets =

component/pypy/buildout.cfg

@@ -23,7 +23,7 @@ make-targets = python setup.py install --install-lib @@LOCATION@@
 [pypy2]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.1-src.tar.bz2
+url = https://downloads.python.org/pypy/pypy2.7-v7.3.1-src.tar.bz2
 md5sum = 7608bd58940ffc5403632c2c786d83bb
 configure-command =
   sed -i '/"_tkinter":/s/^/#/' lib_pypy/tools/build_cffi_imports.py

component/rust/buildout.cfg

+[buildout]
+parts =
+  rustc
+
+extends =
+    ../cmake/buildout.cfg
+    ../curl/buildout.cfg
+    ../git/buildout.cfg
+    ../openssl/buildout.cfg
+    ../pkgconfig/buildout.cfg
+
+
+[rustc]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://static.rust-lang.org/dist/rustc-1.45.2-src.tar.gz
+md5sum = 1c67d7c3f211e49e12e7c20abab08e70
+# --sysconfdir is a workaround for https://github.com/rust-lang/rust/issues/63915
+configure-options = --enable-extended --sysconfdir=@@LOCATION@@/etc/
+environment = 
+    PATH=${cmake:location}/bin/:${curl:location}/bin/:${git:location}/bin/:${pkgconfig:location}/bin/:%(PATH)s
+    PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:

software/caddy-frontend/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg.in
-md5sum = dae3bf6daf851b5610a1a6bd83057b29
+md5sum = 30eb6db0d2df9ffb3a50fb64e73df5be
 
 [template-common]
 filename = instance-common.cfg.in
@@ -22,7 +22,7 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
 
 [template-apache-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = 43b3763d4086283e9c7c886e3b89e54b
+md5sum = 1709085d62f46b22cbe9369fe324bb49
 
 [template-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
@@ -30,7 +30,7 @@ md5sum = 19debfbc27c464f451b1eb5bb5ce3c84
 
 [template-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = 8f518a7b543126ce91fe218255fb201c
+md5sum = da4e3de1c1861459273c6e8d32ead21e
 
 [template-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
@@ -106,7 +106,7 @@ md5sum = 8c150e1e6c993708d31936742f3a7302
 
 [caddyprofiledeps-setup]
 filename = setup.py
-md5sum = d9b6476bb0b36cf463fddb00d41dfbaa
+md5sum = 8e1c6c06c09beb921965b3ce98c67c9e
 
 [caddyprofiledeps-dummy]
 filename = caddyprofiledummy.py
@@ -119,3 +119,7 @@ md5sum = f77744e52f9d028c39f99cfbf31eadaf
 [template-backend-haproxy-rsyslogd-conf]
 _update_hash_filename_ = templates/backend-haproxy-rsyslogd.conf.in
 md5sum = be899b04e1aa652ed510f20d4ea523dd
+
+[template-slave-introspection-httpd-nginx]
+_update_hash_filename_ = templates/slave-introspection-httpd-nginx.conf.in
+md5sum = 3067e6ba6c6901821d57d2109517d39c

software/caddy-frontend/common.cfg

@@ -12,6 +12,7 @@ extends =
   ../../component/xz-utils/buildout.cfg
   ../../component/rsyslogd/buildout.cfg
   ../../component/haproxy/buildout.cfg
+  ../../component/nginx/buildout.cfg
 
   ../../stack/caucase/buildout.cfg
 # Monitoring stack (keep on bottom)
@@ -94,6 +95,8 @@ logrotate_base_instance = ${template-logrotate-base:rendered}
 bin_directory = ${buildout:bin-directory}
 
 sixtunnel = ${6tunnel:location}
+nginx = ${nginx-output:nginx}
+nginx_mime = ${nginx-output:mime}
 caddy = ${caddy:output}
 caddy_location = ${caddy:location}
 haproxy_executable = ${haproxy:location}/sbin/haproxy
@@ -110,6 +113,7 @@ kedifa = ${:bin_directory}/kedifa
 kedifa-updater = ${:bin_directory}/kedifa-updater
 kedifa-csr = ${:bin_directory}/kedifa-csr
 xz_location = ${xz-utils:location}
+htpasswd = ${:bin_directory}/htpasswd
 
 monitor_template = ${monitor-template:output}
 template_backend_haproxy_configuration = ${template-backend-haproxy-configuration:target}
@@ -129,6 +133,7 @@ template_trafficserver_records_config = ${template-trafficserver-records-config:
 template_trafficserver_storage_config = ${template-trafficserver-storage-config:target}
 template_trafficserver_logging_config = ${template-trafficserver-logging-config:target}
 template_wrapper = ${template-wrapper:output}
+template_slave_introspection_httpd_nginx = ${template-slave-introspection-httpd-nginx:target}
 
 [template]
 recipe = slapos.recipe.template:jinja2
@@ -198,6 +203,9 @@ mode = 640
 [template-empty]
 <=download-template
 
+[template-slave-introspection-httpd-nginx]
+<=download-template
+
 [template-wrapper]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/templates/wrapper.in

software/caddy-frontend/instance-apache-frontend.cfg.in

@@ -52,6 +52,15 @@ parts =
   promise-backend-haproxy-https
   promise-backend-haproxy-configuration
 
+  slave-introspection-frontend
+  slave-introspection-graceful
+  promise-slave-introspection-https
+  promise-slave-introspection-configuration
+  logrotate-entry-slave-introspection
+
+[caddyprofiledeps]
+recipe = caddyprofiledeps
+
 # Create all needed directories
 [directory]
 recipe = slapos.cookbook:mkdirectory
@@ -83,6 +92,9 @@ csr_id = ${:srv}/csr_id
 caddy-csr_id = ${:etc}/caddy-csr_id
 caddy-csr_id-log = ${:log}/httpd-csr_id
 
+# slave introspection
+slave-introspection-var = ${:var}/slave-introspection
+
 [switch-caddy-softwaretype]
 recipe = slapos.cookbook:softwaretype
 single-default = ${dynamic-custom-personal-template-slave-list:rendered}
@@ -95,6 +107,14 @@ ip-access-certificate = ${self-signed-ip-access:certificate}
 caddy-directory = {{ parameter_dict['caddy_location'] }}
 caddy-ipv6 = {{ instance_parameter['ipv6-random'] }}
 caddy-https-port = ${configuration:port}
+nginx = {{ parameter_dict['nginx'] }}
+nginx_mime = {{ parameter_dict['nginx_mime'] }}
+htpasswd = {{ parameter_dict['htpasswd'] }}
+slave-introspection-template = {{ parameter_dict['template_slave_introspection_httpd_nginx'] }}
+slave-introspection-configuration = ${directory:etc}/slave-introspection-httpd-nginx.conf
+slave-introspection-https-port = ${configuration:slave-introspection-https-port}
+slave-introspection-secure_access = ${slave-introspection-frontend:connection-secure_access}
+slave-introspection-domain = ${slave-introspection-frontend:connection-domain}
 
 [self-signed-ip-access]
 # Self Signed certificate for HTTPS IP accesses to the frontend
@@ -258,6 +278,7 @@ stop-on-error = True
 
 [dynamic-custom-personal-template-slave-list]
 < = jinja2-template-base
+depends = ${caddyprofiledeps:recipe}
 template = {{ parameter_dict['template_slave_list'] }}
 filename = custom-personal-instance-slave-list.cfg
 slave_instance_list = {{ dumps(instance_parameter['slave-instance-list']) }}
@@ -276,6 +297,7 @@ extra-context =
     key caddy_configuration_directory caddy-directory:slave-configuration
     key backend_client_caucase_url :backend-client-caucase-url
     import urlparse_module urlparse
+    import furl_module furl
     key caddy_executable :caddy_executable
     key http_port configuration:plain_http_port
     key https_port configuration:port
@@ -400,6 +422,11 @@ not-found-file = ${caddy-directory:document-root}/${not-found-html:filename}
 master-certificate = ${caddy-directory:master-autocert-dir}/master.pem
 # Communication with ATS
 cache-port = ${trafficserver-variable:input-port}
+# slave instrspection
+slave-introspection-access-log = ${directory:log}/slave-introspection-access.log
+slave-introspection-error-log = ${directory:log}/slave-introspection-error.log
+slave-introspection-pid-file = ${directory:run}/slave-introspection.pid
+slave-introspection-graceful-command = ${slave-introspection-validate:rendered} && kill -HUP $(cat ${:slave-introspection-pid-file})
 
 # BBB: SlapOS Master non-zero knowledge BEGIN
 [get-self-signed-fallback-access]
@@ -892,6 +919,91 @@ extra-context =
     key http_port configuration:plain_http_port
     key https_port configuration:port
 
+[slave-introspection-frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Slave Introspection Frontend {{ instance_parameter['configuration.frontend-name'] }}
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config-url = https://[${slap-network-information:global-ipv6}]:{{ instance_parameter['configuration.slave-introspection-https-port'] }}/
+config-https-only = true
+return = domain secure_access
+
+[slave-introspection-configuration-state]
+<= jinja2-template-base
+template = {{ parameter_dict['template_configuration_state_script'] }}
+rendered = ${directory:bin}/${:_buildout_section_name_}
+mode = 0700
+
+path_list = ${frontend-configuration:slave-introspection-configuration} ${frontend-configuration:ip-access-certificate}
+sha256sum = {{ parameter_dict['sha256sum'] }}
+
+extra-context =
+    key path_list :path_list
+    key sha256sum :sha256sum
+    key signature_file :signature_file
+
+[slave-introspection-configuration-state-graceful]
+<= slave-introspection-configuration-state
+signature_file = ${directory:run}/slave_introspection_graceful_configuration_state_signature
+
+[slave-introspection-configuration-state-validate]
+<= slave-introspection-configuration-state
+signature_file = ${directory:run}/slave_introspection_validate_configuration_state_signature
+
+[slave-introspection-graceful]
+< = jinja2-template-base
+template = {{ parameter_dict['template_graceful_script'] }}
+rendered = ${directory:etc-run}/slave-introspection-safe-graceful
+mode = 0700
+
+extra-context =
+    key graceful_reload_command caddy-configuration:slave-introspection-graceful-command
+    key caddy_configuration_state slave-introspection-configuration-state-graceful:rendered
+
+[slave-introspection-validate]
+<= jinja2-template-base
+template = {{ parameter_dict['template_validate_script'] }}
+rendered = ${directory:bin}/slave-introspection-validate
+mode = 0700
+last_state_file = ${directory:run}/slave_introspection_configuration_last_state
+validate_command = {{ parameter_dict['nginx'] }} -c ${frontend-configuration:slave-introspection-configuration} -t
+extra-context =
+    key validate_command :validate_command
+    key configuration_state_command slave-introspection-configuration-state-validate:rendered
+    key last_state_file :last_state_file
+
+[promise-slave-introspection-configuration]
+<= monitor-promise-base
+module = validate_frontend_configuration
+name = slave-introspection-configuration.py
+config-verification-script = ${promise-slave-introspection-configuration-helper:rendered}
+
+[promise-slave-introspection-configuration-helper]
+< = jinja2-template-base
+template = {{ parameter_dict['template_empty'] }}
+rendered = ${directory:bin}/slave-introspection-read-last-configuration-state
+mode = 0700
+content =
+  #!/bin/sh
+  exit `cat ${slave-introspection-validate:last_state_file}`
+context =
+    key content :content
+
+[promise-slave-introspection-https]
+<= monitor-promise-base
+module = check_port_listening
+name = slave_introspection_https.py
+config-hostname = {{ instance_parameter['ipv6-random'] }}
+config-port = ${frontend-configuration:slave-introspection-https-port}
+
+[logrotate-entry-slave-introspection]
+<= logrotate-entry-base
+name = slave-introspection
+log = ${caddy-configuration:slave-introspection-access-log} ${caddy-configuration:slave-introspection-error-log}
+rotate-num = ${configuration:rotate-num}
+post = kill -USR1 $(cat ${caddy-configuration:slave-introspection-pid-file})
+
 [configuration]
 {%- for key, value in instance_parameter.iteritems() -%}
 {%-   if key.startswith('configuration.') %}

software/caddy-frontend/instance.cfg.in

@@ -41,6 +41,7 @@ template = {{ template_caddy_frontend }}
 filename = instance-caddy-frontend.cfg
 extensions = jinja2.ext.do
 extra-context =
+  import furl_module furl
   section parameter_dict dynamic-template-caddy-frontend-parameters
   raw software_type single-custom-personal
 caucase-jinja2-library = {{ caucase_jinja2_library }}
@@ -124,3 +125,4 @@ configuration.backend-haproxy-http-port = 21080
 configuration.backend-haproxy-https-port = 21443
 configuration.authenticate-to-backend = False
 configuration.rotate-num = 4000
+configuration.slave-introspection-https-port = 22443

software/caddy-frontend/setup.py

@@ -7,6 +7,8 @@ setup(
   name='caddyprofiledeps',
   install_requires=[
     'validators',
+    'furl',
+    'orderedmultidict',
   ],
   entry_points={
     'zc.buildout': [

software/caddy-frontend/software.cfg

@@ -27,3 +27,5 @@ smmap = 0.9.0
 numpy = 1.16.4
 
 websockify = 0.8.0
+furl = 2.1.0
+orderedmultidict = 1.0.1

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -36,6 +36,7 @@ context =
 # empty sections if no slaves are available
 [slave-log-directory-dict]
 [slave-password]
+[slave-htpasswd]
 
 {#- Loop thought slave list to set up slaves #}
 {%- set DEFAULT_PORT = {'http': 80, 'https': 443, '': None} %}
@@ -94,6 +95,7 @@ context =
 {%-   set slave_configuration_section_name = 'slave-instance-%s-configuration' % slave_reference %}
 {%-   set slave_logrotate_section = slave_reference + "-logs" %}
 {%-   set slave_password_section = slave_reference + "-password" %}
+{%-   set slave_htpasswd_section = slave_reference + "-htpasswd" %}
 {%-   set slave_ln_section = slave_reference + "-ln" %}
 {#-   extend parts #}
 {%-   do part_list.extend([slave_ln_section]) %}
@@ -118,7 +120,12 @@ context =
 {%-   do slave_instance.__setitem__('backend_log', slave_parameter_dict.get('backend_log')) %}
 {#-   Add slave log directory to the slave log access dict #}
 {%-   do slave_log_dict.__setitem__(slave_reference, slave_log_folder) %}
-{%-   set slave_log_access_url = 'https://' + slave_reference.lower() + ':${'+ slave_password_section +':passwd}@[' + frontend_configuration.get('caddy-ipv6') + ']:' + frontend_configuration.get('caddy-https-port') + '/' + slave_reference.lower() + '/' %}
+{%-   set furled = furl_module.furl(frontend_configuration['slave-introspection-secure_access']) %}
+{%-   do furled.set(username = slave_reference.lower()) %}
+{%-   do furled.set(password = '${'+ slave_password_section +':passwd}') %}
+{%-   do furled.set(path = slave_reference.lower() + '/') %}
+{#-   We unquote, as furl quotes automatically, but there is buildout value on purpose like ${...:...} in the passwod #}
+{%-   set slave_log_access_url = urlparse_module.unquote(furled.tostr()) %}
 {%-   do slave_publish_dict.__setitem__('log-access', slave_log_access_url) %}
 {%-   do slave_publish_dict.__setitem__('slave-reference', slave_reference) %}
 {%-   do slave_publish_dict.__setitem__('public-ipv4', public_ipv4) %}
@@ -139,6 +146,9 @@ context =
 [slave-password]
 {{ slave_reference }} = {{ '${' + slave_password_section + ':passwd}' }}
 
+[slave-htpasswd]
+{{ slave_reference }} = {{ '${' + slave_htpasswd_section + ':file}' }}
+
 {#-   Set slave logrotate entry #}
 [{{slave_logrotate_section}}]
 <= logrotate-entry-base
@@ -162,6 +172,13 @@ recipe = slapos.cookbook:generate.password
 storage-path = {{caddy_configuration_directory}}/.{{slave_reference}}.passwd
 bytes = 8
 
+[{{ slave_htpasswd_section }}]
+recipe = plone.recipe.command
+stop-on-error = True
+file = {{ caddy_configuration_directory }}/.{{ slave_reference }}.htpasswd
+command = {{ frontend_configuration['htpasswd'] }} -cb ${:file} {{ slave_reference.lower() }} {{ '${' + slave_password_section + ':passwd}' }}
+update-command = ${:command}
+
 {#-   ################################################## #}
 {#-   Set Slave Certificates if needed                   #}
 {#-   Set certificate key for custom configuration       #}
@@ -287,6 +304,7 @@ recipe = slapos.cookbook:mkdirectory
 {%- do part_list.append('slave-log-directories') %}
 
 {%- do part_list.append('caddy-log-access') %}
+{%- do part_list.append('slave-introspection') %}
 {#- ############################################## #}
 {#- ## Prepare virtualhost for slaves using cache  #}
 {#- Define IPv6 to IPV4 tunneling #}
@@ -331,6 +349,35 @@ extra-context =
     section slave_password slave-password
     section parameter_dict caddy-log-access-parameters
 
+[slave-introspection-parameters]
+local-ipv4 = {{ dumps(local_ipv4) }}
+global-ipv6 = {{ dumps(global_ipv6) }}
+https-port = {{ frontend_configuration['slave-introspection-https-port'] }}
+ip-access-certificate = {{ frontend_configuration.get('ip-access-certificate') }}
+nginx-mime = {{ frontend_configuration['nginx_mime'] }}
+access-log = {{ dumps(caddy_configuration['slave-introspection-access-log']) }}
+error-log = {{ dumps(caddy_configuration['slave-introspection-error-log']) }}
+var = {{ directory['slave-introspection-var'] }}
+pid = {{ caddy_configuration['slave-introspection-pid-file'] }}
+
+[slave-introspection-config]
+<= jinja2-template-base
+template = {{ frontend_configuration['slave-introspection-template'] }}
+rendered = {{ frontend_configuration['slave-introspection-configuration'] }}
+extra-context =
+    section slave_log_directory slave-log-directory-dict
+    section slave_htpasswd slave-htpasswd
+    section parameter_dict slave-introspection-parameters
+
+[slave-introspection]
+recipe = slapos.cookbook:wrapper
+command-line = {{ frontend_configuration['nginx'] }}
+  -c ${slave-introspection-config:rendered}
+
+wrapper-path = {{ directory['service'] }}/slave-instrospection-nginx
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+
+
 
 {#- Publish information for the instance #}
 [publish-caddy-information]

software/caddy-frontend/templates/slave-introspection-httpd-nginx.conf.in

+# Access log configuration
+daemon off;
+pid {{ parameter_dict['pid'] }};
+error_log {{ parameter_dict['error-log'] }};
+
+events {
+}
+
+http {
+  include {{ parameter_dict['nginx-mime'] }};
+  server {
+    server_name_in_redirect off;
+    port_in_redirect off;
+    error_log {{ parameter_dict['error-log'] }};
+    access_log {{ parameter_dict['access-log'] }};
+    listen [{{ parameter_dict['global-ipv6'] }}]:{{ parameter_dict['https-port'] }} ssl;
+    listen {{ parameter_dict['local-ipv4'] }}:{{ parameter_dict['https-port'] }} ssl;
+    ssl_certificate {{ parameter_dict['ip-access-certificate'] }};
+    ssl_certificate_key {{ parameter_dict['ip-access-certificate'] }};
+    default_type application/octet-stream;
+    client_body_temp_path {{ parameter_dict['var'] }} 1 2;
+    proxy_temp_path {{ parameter_dict['var'] }} 1 2;
+    fastcgi_temp_path {{ parameter_dict['var'] }} 1 2;
+    uwsgi_temp_path {{ parameter_dict['var'] }} 1 2;
+    scgi_temp_path {{ parameter_dict['var'] }} 1 2;
+  {% for slave, directory in slave_log_directory.iteritems() %}
+    location /{{ slave }} {
+      alias {{ directory }};
+      autoindex on;
+      autoindex_format json;
+      sendfile on;
+      sendfile_max_chunk 1m;
+      auth_basic "Log Access {{ slave }}";
+      auth_basic_user_file "{{ slave_htpasswd[slave] | trim }}";
+    }
+  {% endfor %}
+  }
+}

software/caddy-frontend/test/test.py

@@ -674,6 +674,15 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
       result.status_code,
       'While accessing %r of %r the status code was %r' % (
         url, frontend, result.status_code))
+    # check that the result is correct JSON, which allows to access
+    # information about all logs
+    self.assertEqual(
+      'application/json',
+      result.headers['Content-Type']
+    )
+    self.assertEqual(
+      sorted([q['name'] for q in result.json()]),
+      ['access.log', 'backend.log', 'error.log'])
     self.assertEqual(
       httplib.OK,
       requests.get(url + 'access.log', verify=False).status_code
@@ -684,7 +693,8 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     )
     # assert only for few tests, as backend log is not available for many of
     # them, as it's created on the fly
-    for test_name in ['test_url', 'test_auth_to_backend', 'test_compressed_result']:
+    for test_name in [
+      'test_url', 'test_auth_to_backend', 'test_compressed_result']:
       if self.id().endswith(test_name):
         self.assertEqual(
           httplib.OK,

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_file_list_log-CADDY.txt

@@ -23,5 +23,7 @@ T-2/var/log/httpd/_enable-http2-true_backend_log
 T-2/var/log/httpd/_enable-http2-true_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -23,5 +23,7 @@ T-2/var/log/httpd/_enable-http2-true_backend_log
 T-2/var/log/httpd/_enable-http2-true_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_file_list_log-CADDY.txt

@@ -23,5 +23,7 @@ T-2/var/log/httpd/_enable-http2-true_backend_log
 T-2/var/log/httpd/_enable-http2-true_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -23,5 +23,7 @@ T-2/var/log/httpd/_enable-http2-true_backend_log
 T-2/var/log/httpd/_enable-http2-true_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_file_list_log-CADDY.txt

@@ -11,5 +11,7 @@ T-2/var/log/frontend-error.log
 T-2/var/log/httpd-csr_id/expose-csr_id.log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_file_list_log-CADDY.txt

@@ -11,5 +11,7 @@ T-2/var/log/frontend-error.log
 T-2/var/log/httpd-csr_id/expose-csr_id.log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_file_list_log-CADDY.txt

@@ -13,5 +13,7 @@ T-2/var/log/httpd/_default_access_log
 T-2/var/log/httpd/_default_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-CADDY.txt

@@ -177,5 +177,7 @@ T-2/var/log/httpd/_url_https-url_backend_log
 T-2/var/log/httpd/_url_https-url_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_plugin-CADDY.txt

@@ -38,5 +38,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestSlave.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestSlaveCiphers.test_file_list_log-CADDY.txt

@@ -17,5 +17,7 @@ T-2/var/log/httpd/_own_ciphers_backend_log
 T-2/var/log/httpd/_own_ciphers_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveCiphers.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestSlaveCiphers.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestSlaveCiphers.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -177,5 +177,7 @@ T-2/var/log/httpd/_url_https-url_backend_log
 T-2/var/log/httpd/_url_https-url_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_plugin-CADDY.txt

@@ -38,5 +38,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibility.test_file_list_log-CADDY.txt

@@ -47,5 +47,7 @@ T-2/var/log/httpd/_type-notebook-ssl_from_slave_kedifa_overrides_backend_log
 T-2/var/log/httpd/_type-notebook-ssl_from_slave_kedifa_overrides_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibility.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibility.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibility.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_file_list_log-CADDY.txt

@@ -14,5 +14,7 @@ T-2/var/log/httpd/_ssl_from_master_kedifa_overrides_master_certificate_backend_l
 T-2/var/log/httpd/_ssl_from_master_kedifa_overrides_master_certificate_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_file_list_log-CADDY.txt

@@ -14,5 +14,7 @@ T-2/var/log/httpd/_ssl_from_master_backend_log
 T-2/var/log/httpd/_ssl_from_master_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
 T-2/var/log/trafficserver/manager.log
 T-2/var/log/trafficserver/traffic.out

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_file_list_plugin-CADDY.txt

@@ -36,5 +36,7 @@ T-2/etc/plugin/monitor-bootstrap-status.py
 T-2/etc/plugin/monitor-http-frontend.py
 T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
 T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
 T-2/etc/plugin/trafficserver-cache-availability.py
 T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_file_list_run-CADDY.txt

@@ -9,3 +9,6 @@ T-2/var/run/bhlog.sck
 T-2/var/run/graceful_configuration_state_signature
 T-2/var/run/httpd.pid
 T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_supervisor_state-CADDY.txt

@@ -33,5 +33,7 @@ T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
 T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
 T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
 T-2:trafficserver-{hash-generic}-on-watch RUNNING
 T-2:trafficserver-reload EXITED

software/kvm/README.rst

@@ -18,9 +18,8 @@ KVM instance (1GB of RAM, 10GB of SSD, one core)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Note that the KVM instance will try to request a frontend slave instance in order
-to be accessible from IPv4.
+to be accessible from IPv4::
 
-::
   myawesomekvm = request(
       software_release=kvm,
       partition_reference="My awesome KVM",
@@ -97,11 +96,10 @@ KVM instance parameters:
     'th', 'tr']
 
 Resilient KVM instance
-~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~
 
-Like KVM instance, but backed-up (with history) in two places.
+Like KVM instance, but backed-up (with history) in two places::
 
-::
   kvm = 'https://lab.nexedi.com/nexedi/slapos/raw/slapos-0.188/software/kvm/software.cfg'
   myresilientkvm = request(
       software_release=kvm,
@@ -119,3 +117,18 @@ files for more instance parameters (cpu-count, ram-size, disk-size, specific loc
 
 Then, if you want one of the two clones to takeover, you need to login into
 the hosting machine, go to the partition of the clone, and invoke bin/takeover.
+
+Technical notes
+---------------
+
+Updating boot-image-url-select
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ * download the new OS installation image
+ * calculate it's sha512sum and store as <SHA512>
+ * calculate it's md5sum and store as <MD5>
+ * upload it to shacache
+ * construct download url: ``https://shacache.nxdcdn.com/<SHA512>#<MD5>``
+ * update the ``boot-image-url-select`` in:
+    * ``instance-kvm-input-schema.json``
+    * ``instance-kvm-cluster-input-schema.json``

software/kvm/buildout.hash.cfg

@@ -19,11 +19,11 @@ md5sum = e6d5c7bb627b4f1d3e7c99721b7c58fe
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = 50e78a2a34efe09afab161ae1c1efd46
+md5sum = 23493c541efef97ac5fe435114910b8e
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in
-md5sum = a4788112008cd0b38a57cd28f7252fbd
+md5sum = bdf8549a76ec61e125d51a05e611e004
 
 [template-kvm-resilient]
 filename = instance-kvm-resilient.cfg.jinja2
@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
 
 [template-kvm-run]
 filename = template/template-kvm-run.in
-md5sum = dd1f581f34cf5a0b627576771347c710
+md5sum = 4a6f149177a453a13436f320f6841518
 
 [template-kvm-controller]
 filename = template/kvm-controller-run.in
@@ -87,4 +87,4 @@ md5sum = 7e4b54f8172c364bd12d28b07f8b1600
 
 [image-download-config-creator]
 _update_hash_filename_ = template/image-download-config-creator.py
-md5sum = 7f6cd75096695922fddbba1c9292cef5
+md5sum = 54261e418ab9860efe73efd514c4d47f

software/kvm/instance-kvm-cluster-input-schema.json

@@ -42,12 +42,6 @@
           "type": "string",
           "format": "uri",
           "default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg"
-        },
-        "image-url-list": {
-          "title": "List of URLs images to download",
-          "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
-          "type": "string",
-          "textarea": "true"
         }
       },
       "type": "object"
@@ -508,6 +502,60 @@
               "description": "If the VM of cluster doesn't run Ansible and report status to this SlapOS instances, then this allow to disable ansible promise so your instance will not fail to check ansible promise.",
               "type": "boolean",
               "default": false
+            },
+            "boot-image-url-list": {
+              "title": "Boot image list",
+              "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
+              "type": "string",
+              "textarea": "true"
+            },
+            "boot-image-url-select": {
+              "title": "Boot image",
+              "type": "array",
+              "oneOf": [
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/0a6aee1d9aafc1ed095105c052f9fdd65ed00ea9274188c9cd0072c8e6838ab40e246d45a1e6956d74ef1b04a1fc042151762f25412e9ff0cbf49418eef7992e#a3ebc76aec372808ad80000108a2593a"
+                  ],
+                  "title": "Debian Buster 10.5 netinst x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/ce5ddfdbdaccdf929b7fe321212356347d82a02f6b7733427282b416f113d91e587682b003e9d376ac189c3b731595c50c236962aadf2720c16d9f36913577c0#23bf2a2d60271e553e63525e794415f1"
+                  ],
+                  "title": "Centos 8.2004 Minimal x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/302c990c6d69575ff24c96566e5c7e26bf36908abb0cd546e22687c46fb07bf8dba595bf77a9d4fd9ab63e75c0437c133f35462fd41ea77f6f616140cd0e5e6a#f3a306f40e4a313fb5a584d73b3dee8f"
+                  ],
+                  "title": "Ubuntu Focal 20.04.1 Live Server x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/6635269a7eb6fbd6b85fda40cd94f14a27bf53cb1fc82ffcce9fe386a025a43e1ab681db7e8cec50416bfbfc90262f0d95273686a101c74b3f17646f0a34c85b#3708a59af6cf820a95cafe0ae73ac399"
+                  ],
+                  "title": "openSUSE Leap 15.2 NET x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/fc17e8c6ae0790162f4beb8fa6226d945cff638429588999b3a08493ff27b280dc2939fba825ae04be1d9082ea8d7c3c002c5e4c39fbbcf88b8ab5104619e28a#ebcdb2223a77f098af3923fe1fa180aa"
+                  ],
+                  "title": "Arch Linux 2020.09.01 x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/c5a511f349a1146b615e6fab9c24f9be4362046adcf24f0ff82c470d361fac5f6628895e2110ebf8ff87db49d4c413a0a332699da6b1bec64275e0c17a15b999#ca7a1e555c04b4d9a549065fa2ddf713"
+                  ],
+                  "title": "Fedora Server 32-1.6 netinst x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/6c355def68b3c0427f21598cb054ffc893568902f205601ac60f192854769b31bc9cff8eeb6ce99ef975a8fb887d8d3e56fc6cd5ea5cb4b3bba1175c520047cb#57088b77f795ca44b00971e44782ee23"
+                  ],
+                  "title": "FreeBSD 12.1 RELEASE bootonly x86_64"
+                }
+              ]
             }
           },
           "type": "object"

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -127,9 +127,13 @@ config-document-host = ${apache-conf:ip}
 config-document-port = ${apache-conf:port}
 config-document-path = ${hash-code:passwd}
 config-keyboard-layout-language = {{ dumps(kvm_parameter_dict.get('keyboard-layout-language', 'fr')) }}
-{%- if 'image-url-list' in kvm_parameter_dict %}
+{%- if 'boot-image-url-list' in kvm_parameter_dict %}
 {#- play nice: if parameter was not constructed by the original request, do not send it at all #}
-config-image-url-list = {{ kvm_parameter_dict['image-url-list'] }}
+config-boot-image-url-list = {{ kvm_parameter_dict['boot-image-url-list'] }}
+{%- endif %}
+{%- if 'boot-image-url-select' in kvm_parameter_dict %}
+{#- play nice: if parameter was not constructed by the original request, do not send it at all #}
+config-boot-image-url-select = {{ kvm_parameter_dict['boot-image-url-select'] }}
 {%- endif %}
 config-type = cluster
 

software/kvm/instance-kvm-input-schema.json

 {
   "type": "object",
-  "$schema": "http://json-schema.org/draft-04/schema",
+  "$schema": "http://json-schema.org/draft-06/schema",
   "title": "Input Parameters",
   "properties": {
     "enable-device-hotplug": {
@@ -366,11 +366,59 @@
       "format": "uri",
       "default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg"
     },
-    "image-url-list": {
-      "title": "List of URLs images to download",
+    "boot-image-url-list": {
+      "title": "Boot image list",
       "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
       "type": "string",
       "textarea": "true"
+    },
+    "boot-image-url-select": {
+      "title": "Boot image",
+      "type": "array",
+      "oneOf": [
+        {
+          "const": [
+            "https://shacache.nxdcdn.com/0a6aee1d9aafc1ed095105c052f9fdd65ed00ea9274188c9cd0072c8e6838ab40e246d45a1e6956d74ef1b04a1fc042151762f25412e9ff0cbf49418eef7992e#a3ebc76aec372808ad80000108a2593a"
+          ],
+          "title": "Debian Buster 10.5 netinst x86_64"
+        },
+        {
+          "const": [
+            "https://shacache.nxdcdn.com/ce5ddfdbdaccdf929b7fe321212356347d82a02f6b7733427282b416f113d91e587682b003e9d376ac189c3b731595c50c236962aadf2720c16d9f36913577c0#23bf2a2d60271e553e63525e794415f1"
+          ],
+          "title": "Centos 8.2004 Minimal x86_64"
+        },
+        {
+          "const": [
+            "https://shacache.nxdcdn.com/302c990c6d69575ff24c96566e5c7e26bf36908abb0cd546e22687c46fb07bf8dba595bf77a9d4fd9ab63e75c0437c133f35462fd41ea77f6f616140cd0e5e6a#f3a306f40e4a313fb5a584d73b3dee8f"
+          ],
+          "title": "Ubuntu Focal 20.04.1 Live Server x86_64"
+        },
+        {
+          "const": [
+            "https://shacache.nxdcdn.com/6635269a7eb6fbd6b85fda40cd94f14a27bf53cb1fc82ffcce9fe386a025a43e1ab681db7e8cec50416bfbfc90262f0d95273686a101c74b3f17646f0a34c85b#3708a59af6cf820a95cafe0ae73ac399"
+          ],
+          "title": "openSUSE Leap 15.2 NET x86_64"
+        },
+        {
+          "const": [
+            "https://shacache.nxdcdn.com/fc17e8c6ae0790162f4beb8fa6226d945cff638429588999b3a08493ff27b280dc2939fba825ae04be1d9082ea8d7c3c002c5e4c39fbbcf88b8ab5104619e28a#ebcdb2223a77f098af3923fe1fa180aa"
+          ],
+          "title": "Arch Linux 2020.09.01 x86_64"
+        },
+        {
+          "const": [
+            "https://shacache.nxdcdn.com/c5a511f349a1146b615e6fab9c24f9be4362046adcf24f0ff82c470d361fac5f6628895e2110ebf8ff87db49d4c413a0a332699da6b1bec64275e0c17a15b999#ca7a1e555c04b4d9a549065fa2ddf713"
+          ],
+          "title": "Fedora Server 32-1.6 netinst x86_64"
+        },
+        {
+          "const": [
+            "https://shacache.nxdcdn.com/6c355def68b3c0427f21598cb054ffc893568902f205601ac60f192854769b31bc9cff8eeb6ce99ef975a8fb887d8d3e56fc6cd5ea5cb4b3bba1175c520047cb#57088b77f795ca44b00971e44782ee23"
+          ],
+          "title": "FreeBSD 12.1 RELEASE bootonly x86_64"
+        }
+      ]
     }
   }
 }

software/kvm/instance-kvm.cfg.jinja2

@@ -9,7 +9,8 @@
 {% set instance_type = slapparameter_dict.get('type', 'standalone') -%}
 {% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
 {% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%}
-{% set image_url_list_enabled = 'image-url-list' in slapparameter_dict %}
+{% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %}
+{% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %}
 {% set cpu_max_count = dumps(slapparameter_dict.get('cpu-max-count', int(slapparameter_dict.get('cpu-count', 1)) + 1)) %}
 {% set ram_max_size = dumps(slapparameter_dict.get('ram-max-size', int(slapparameter_dict.get('ram-size', 1024)) + 512)) %}
 {% set extends_list = [] -%}
@@ -54,10 +55,15 @@ public = ${:srv}/public/
 cron-entries = ${:etc}/cron.d
 crontabs = ${:etc}/crontabs
 cronstamps = ${:etc}/cronstamps
-{%- if image_url_list_enabled %}
-image-repository = ${:srv}/image-repository
-image-url-list-var = ${:var}/image-url-list
-image-url-list-expose = ${monitor-directory:private}/image-url-list
+{%- if boot_image_url_list_enabled %}
+boot-image-url-list-repository = ${:srv}/boot-image-url-list-repository
+boot-image-url-list-var = ${:var}/boot-image-url-list
+boot-image-url-list-expose = ${monitor-directory:private}/boot-image-url-list
+{%- endif %}
+{%- if boot_image_url_select_enabled %}
+boot-image-url-select-repository = ${:srv}/boot-image-url-select-repository
+boot-image-url-select-var = ${:var}/boot-image-url-select
+boot-image-url-select-expose = ${monitor-directory:private}/boot-image-url-select
 {%- endif %}
 
 [create-mac]
@@ -73,9 +79,116 @@ recipe = slapos.cookbook:generate.password
 storage-path = ${directory:srv}/passwd
 bytes = 8
 
-{% if image_url_list_enabled %}
-## image-url-list support BEGIN
-[empty-file-state-base-promise]
+{% if boot_image_url_select_enabled %}
+## boot-image-url-select support BEGIN
+[empty-file-state-base-select-promise]
+<= monitor-promise-base
+module = check_file_state
+name = ${:_buildout_section_name_}.py
+config-state = empty
+# It's very hard to put the username and password correctly, after schema://
+# and before the host, as it's not the way how one can use monitor provided
+# information, so just show the information in the URL
+config-url = ${monitor-base:base-url}/private/boot-image-url-select/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
+
+[boot-image-url-select-source-config]
+# generates configuration of the image from the user parameter
+# special "magic" is used, to properly support multiline boot-image-url-select
+# but in the same time correctly generate the configuration file
+recipe = slapos.recipe.template:jinja2
+template = inline:
+{#- Do special trick to support boot-image-url-select being None, if key is present with value "" #}
+{%- raw %}
+  {%- set boot_image_url_select = slap_parameter.get('boot-image-url-select') or '' %}
+  {%- if boot_image_url_select == 'None' %}
+  {#- That's insane protection, is it 'None' because of type = array? #}
+  {%-   set boot_image_url_select = '' %}
+  {%- endif %}
+  {{ boot_image_url_select }}
+{% endraw -%}
+context =
+  section slap_parameter slap-parameter
+rendered = ${directory:etc}/boot-image-url-select.json
+
+[boot-image-url-select-processed-config]
+# compares if the current configuration has been used by
+# the boot-image-url-select-download, if not, exposes it as not empty file with
+# information
+recipe = slapos.recipe.build
+install =
+  import os
+  import hashlib
+  if not os.path.exists(location):
+    os.mkdir(location)
+  with open('${:state-file}', 'w') as state_handler:
+    try:
+      with open('${:config-file}', 'rb') as config_handler, open('${:processed-md5sum}') as processed_handler:
+        config_md5sum = hashlib.md5(config_handler.read()).hexdigest()
+        processed_md5sum = processed_handler.read()
+        if config_md5sum == processed_md5sum:
+          state_handler.write('')
+        else:
+          state_handler.write('config %s != processed %s' % (config_md5sum, processed_md5sum))
+    except Exception as e:
+      state_handler.write(str(e))
+
+update = ${:install}
+config-file = ${boot-image-url-select-source-config:rendered}
+state-filename = boot-image-url-select-processed-config.state
+state-file = ${directory:boot-image-url-select-expose}/${:state-filename}
+processed-md5sum = ${directory:boot-image-url-select-var}/update-image-processed.md5sum
+
+[boot-image-url-select-processed-config-promise]
+# promise to check if the configuration provided by the user has been already
+# processed by the boot-image-url-select-download script, which runs asynchronously
+<= empty-file-state-base-select-promise
+filename = ${boot-image-url-select-processed-config:state-filename}
+config-filename = ${boot-image-url-select-processed-config:state-file}
+
+[boot-image-url-select-json-config]
+# generates json configuration from user configuration
+recipe = plone.recipe.command
+command = {{ python_executable }} {{ image_download_config_creator }} ${boot-image-url-select-source-config:rendered} ${:rendered} ${directory:boot-image-url-select-repository} ${:error-state-file}
+update-command = ${:command}
+rendered = ${directory:boot-image-url-select-var}/boot-image-url-select.json
+error-state-filename = boot-image-url-select-json-config-error.txt
+error-state-file = ${directory:boot-image-url-select-expose}/${:error-state-filename}
+
+[boot-image-url-select-config-state-promise]
+# promise to check if configuration has been parsed without errors
+<= empty-file-state-base-select-promise
+filename = ${boot-image-url-select-json-config:error-state-filename}
+config-filename = ${boot-image-url-select-json-config:error-state-file}
+
+[boot-image-url-select-download-wrapper]
+# wrapper to execute boot-image-url-select-download on each run
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:scripts}/boot-image-url-select-updater
+command-line = {{ python_executable }} {{ image_download_controller }} ${boot-image-url-select-json-config:rendered} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${boot-image-url-select-processed-config:processed-md5sum}
+md5sum-state-filename = boot-image-url-select-download-controller-md5sum-fail.json
+md5sum-state-file = ${directory:boot-image-url-select-expose}/${:md5sum-state-filename}
+error-state-filename = boot-image-url-select-download-controller-error.text
+error-state-file = ${directory:boot-image-url-select-expose}/${:error-state-filename}
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+
+[boot-image-url-select-download-md5sum-promise]
+# promise to report errors with problems with calculating md5sum of the
+# downloaded images
+<= empty-file-state-base-select-promise
+filename = ${boot-image-url-select-download-wrapper:md5sum-state-filename}
+config-filename = ${boot-image-url-select-download-wrapper:md5sum-state-file}
+
+[boot-image-url-select-download-state-promise]
+# promise to report errors during download
+<= empty-file-state-base-select-promise
+filename = ${boot-image-url-select-download-wrapper:error-state-filename}
+config-filename = ${boot-image-url-select-download-wrapper:error-state-file}
+## boot-image-url-select support END
+{% endif %} {# if boot_image_url_select_enabled #}
+
+{% if boot_image_url_list_enabled %}
+## boot-image-url-list support BEGIN
+[empty-file-state-base-list-promise]
 <= monitor-promise-base
 module = check_file_state
 name = ${:_buildout_section_name_}.py
@@ -83,25 +196,25 @@ config-state = empty
 # It's very hard to put the username and password correctly, after schema://
 # and before the host, as it's not the way how one can use monitor provided
 # information, so just show the information in the URL
-config-url = ${monitor-base:base-url}/private/image-url-list/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
+config-url = ${monitor-base:base-url}/private/boot-image-url-list/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
 
-[image-url-list-source-config]
+[boot-image-url-list-source-config]
 # generates configuration of the image from the user parameter
-# special "magic" is used, to properly support multiline image-url-list
+# special "magic" is used, to properly support multiline boot-image-url-list
 # but in the same time correctly generate the configuration file
 recipe = slapos.recipe.template:jinja2
 template = inline:
-{#- Do special trick to support image-url-list being None, if key is present with value "" #}
+{#- Do special trick to support boot-image-url-list being None, if key is present with value "" #}
 {%- raw %}
-  {{ slap_parameter.get('image-url-list') or '' }}
+  {{ slap_parameter.get('boot-image-url-list') or '' }}
 {% endraw -%}
 context =
   section slap_parameter slap-parameter
-rendered = ${directory:etc}/image-url-list.conf
+rendered = ${directory:etc}/boot-image-url-list.conf
 
-[image-url-list-processed-config]
+[boot-image-url-list-processed-config]
 # compares if the current configuration has been used by
-# the image-url-list-download, if not, exposes it as not empty file with
+# the boot-image-url-list-download, if not, exposes it as not empty file with
 # information
 recipe = slapos.recipe.build
 install =
@@ -122,58 +235,58 @@ install =
       state_handler.write(str(e))
 
 update = ${:install}
-config-file = ${image-url-list-source-config:rendered}
-state-filename = image-url-list-processed-config.state
-state-file = ${directory:image-url-list-expose}/${:state-filename}
-processed-md5sum = ${directory:image-url-list-var}/update-image-processed.md5sum
+config-file = ${boot-image-url-list-source-config:rendered}
+state-filename = boot-image-url-list-processed-config.state
+state-file = ${directory:boot-image-url-list-expose}/${:state-filename}
+processed-md5sum = ${directory:boot-image-url-list-var}/update-image-processed.md5sum
 
-[image-url-list-processed-config-promise]
+[boot-image-url-list-processed-config-promise]
 # promise to check if the configuration provided by the user has been already
-# processed by the image-url-list-download script, which runs asynchronously
-<= empty-file-state-base-promise
-filename = ${image-url-list-processed-config:state-filename}
-config-filename = ${image-url-list-processed-config:state-file}
+# processed by the boot-image-url-list-download script, which runs asynchronously
+<= empty-file-state-base-list-promise
+filename = ${boot-image-url-list-processed-config:state-filename}
+config-filename = ${boot-image-url-list-processed-config:state-file}
 
-[image-url-list-json-config]
+[boot-image-url-list-json-config]
 # generates json configuration from user configuration
 recipe = plone.recipe.command
-command = {{ python_executable }} {{ image_download_config_creator }} ${image-url-list-source-config:rendered} ${:rendered} ${directory:image-repository} ${:error-state-file}
+command = {{ python_executable }} {{ image_download_config_creator }} ${boot-image-url-list-source-config:rendered} ${:rendered} ${directory:boot-image-url-list-repository} ${:error-state-file}
 update-command = ${:command}
-rendered = ${directory:image-url-list-var}/image-url-list.json
-error-state-filename = image-url-list-json-config-error.txt
-error-state-file = ${directory:image-url-list-expose}/${:error-state-filename}
+rendered = ${directory:boot-image-url-list-var}/boot-image-url-list.json
+error-state-filename = boot-image-url-list-json-config-error.txt
+error-state-file = ${directory:boot-image-url-list-expose}/${:error-state-filename}
 
-[image-url-list-config-state-promise]
+[boot-image-url-list-config-state-promise]
 # promise to check if configuration has been parsed without errors
-<= empty-file-state-base-promise
-filename = ${image-url-list-json-config:error-state-filename}
-config-filename = ${image-url-list-json-config:error-state-file}
+<= empty-file-state-base-list-promise
+filename = ${boot-image-url-list-json-config:error-state-filename}
+config-filename = ${boot-image-url-list-json-config:error-state-file}
 
-[image-url-list-download-wrapper]
-# wrapper to execute image-url-list-download on each run
+[boot-image-url-list-download-wrapper]
+# wrapper to execute boot-image-url-list-download on each run
 recipe = slapos.cookbook:wrapper
-wrapper-path = ${directory:scripts}/image-updater
-command-line = {{ python_executable }} {{ image_download_controller }} ${image-url-list-json-config:rendered} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${image-url-list-processed-config:processed-md5sum}
-md5sum-state-filename = image-download-controller-md5sum-fail.json
-md5sum-state-file = ${directory:image-url-list-expose}/${:md5sum-state-filename}
-error-state-filename = image-download-controller-error.text
-error-state-file = ${directory:image-url-list-expose}/${:error-state-filename}
+wrapper-path = ${directory:scripts}/boot-image-url-list-updater
+command-line = {{ python_executable }} {{ image_download_controller }} ${boot-image-url-list-json-config:rendered} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${boot-image-url-list-processed-config:processed-md5sum}
+md5sum-state-filename = boot-image-url-list-download-controller-md5sum-fail.json
+md5sum-state-file = ${directory:boot-image-url-list-expose}/${:md5sum-state-filename}
+error-state-filename = boot-image-url-list-download-controller-error.text
+error-state-file = ${directory:boot-image-url-list-expose}/${:error-state-filename}
 hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
-[image-url-list-download-md5sum-promise]
+[boot-image-url-list-download-md5sum-promise]
 # promise to report errors with problems with calculating md5sum of the
 # downloaded images
-<= empty-file-state-base-promise
-filename = ${image-url-list-download-wrapper:md5sum-state-filename}
-config-filename = ${image-url-list-download-wrapper:md5sum-state-file}
+<= empty-file-state-base-list-promise
+filename = ${boot-image-url-list-download-wrapper:md5sum-state-filename}
+config-filename = ${boot-image-url-list-download-wrapper:md5sum-state-file}
 
-[image-url-list-download-state-promise]
+[boot-image-url-list-download-state-promise]
 # promise to report errors during download
-<= empty-file-state-base-promise
-filename = ${image-url-list-download-wrapper:error-state-filename}
-config-filename = ${image-url-list-download-wrapper:error-state-file}
-## image-url-list support END
-{% endif %} {# if image_url_list_enabled #}
+<= empty-file-state-base-list-promise
+filename = ${boot-image-url-list-download-wrapper:error-state-filename}
+config-filename = ${boot-image-url-list-download-wrapper:error-state-file}
+## boot-image-url-list support END
+{% endif %} {# if boot_image_url_list_enabled #}
 
 [kvm-controller-parameter-dict]
 python-path = {{ python_eggs_executable }}
@@ -195,10 +308,15 @@ vnc-ip = ${:ipv4}
 vnc-port = 5901
 
 default-cdrom-iso = {{ debian_amd64_netinst_location }}
-{% if image_url_list_enabled %}
-image-url-list-json-config = ${image-url-list-json-config:rendered}
+{% if boot_image_url_list_enabled %}
+boot-image-url-list-json-config = ${boot-image-url-list-json-config:rendered}
 {% else %}
-image-url-list-json-config =
+boot-image-url-list-json-config =
+{% endif %}
+{% if boot_image_url_select_enabled %}
+boot-image-url-select-json-config = ${boot-image-url-select-json-config:rendered}
+{% else %}
+boot-image-url-select-json-config =
 {% endif %}
 nbd-host = ${slap-parameter:nbd-host}
 nbd-port = ${slap-parameter:nbd-port}
@@ -823,7 +941,7 @@ nbd-port = 1024
 nbd-host =
 nbd2-port = 1024
 nbd2-host =
-image-url-list =
+boot-image-url-list =
 
 enable-device-hotplug = False
 ram-size = 1024
@@ -873,7 +991,7 @@ keyboard-layout-language = fr
 {% set key_list =  v.split('\n') -%}
 {{ k }} =
   {{ key_list | join('\n  ') }}
-{% elif k == 'image-url-list' %}
+{% elif k == 'boot-image-url-list' %}
 {# needs to decorate possibly multiline or maybe unsafe value #}
 {{ k }} = {{ dumps(v) }}
 {% else -%}
@@ -950,12 +1068,19 @@ parts =
   cron-service
   cron-entry-logrotate
   frontend-promise
-{% if image_url_list_enabled %}
-  image-url-list-download-wrapper
-  image-url-list-config-state-promise
-  image-url-list-download-md5sum-promise
-  image-url-list-download-state-promise
-  image-url-list-processed-config-promise
+{% if boot_image_url_list_enabled %}
+  boot-image-url-list-download-wrapper
+  boot-image-url-list-config-state-promise
+  boot-image-url-list-download-md5sum-promise
+  boot-image-url-list-download-state-promise
+  boot-image-url-list-processed-config-promise
+{% endif %}
+{% if boot_image_url_select_enabled %}
+  boot-image-url-select-download-wrapper
+  boot-image-url-select-config-state-promise
+  boot-image-url-select-download-md5sum-promise
+  boot-image-url-select-download-state-promise
+  boot-image-url-select-processed-config-promise
 {% endif %}
 {% if additional_frontend %}
   frontend-additional-promise

software/kvm/template/image-download-config-creator.py

@@ -24,7 +24,18 @@ if __name__ == "__main__":
     image_number = 0
     data = fh.read()
     configuration_dict['config-md5sum'] = hashlib.md5(data).hexdigest()
-    for entry in data.decode('utf-8').split():
+    if source_configuration.endswith('.json'):
+      data = data.strip()
+      data_list = []
+      if len(data):
+        try:
+          data_list = json.loads(data)
+        except Exception:
+          error_list.append('ERR: Data is not a JSON')
+          data_list = []
+    else:
+      data_list = data.decode('utf-8').split()
+    for entry in data_list:
       split_entry = entry.split('#')
       if len(split_entry) != 2:
         error_list.append('ERR: entry %r is incorrect' % (entry,))

software/kvm/template/template-kvm-run.in

@@ -99,7 +99,8 @@ enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lowe
 
 logfile = '{{ parameter_dict.get("log-file") }}'
 
-image_url_list_json_config = '{{ parameter_dict.get("image-url-list-json-config") }}'
+boot_image_url_list_json_config = '{{ parameter_dict.get("boot-image-url-list-json-config") }}'
+boot_image_url_select_json_config = '{{ parameter_dict.get("boot-image-url-select-json-config") }}'
 
 if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false':
   opener = FancyURLopener(context=ssl._create_unverified_context())
@@ -370,24 +371,40 @@ for nbd_ip, nbd_port in nbd_list:
           '-drive',
           'file=nbd:[%s]:%s,media=cdrom' % (nbd_ip, nbd_port)])
 else:
-  image_url_list_used = False
-  if image_url_list_json_config:
-    # Support image-url-list
-    with open(image_url_list_json_config) as fh:
+  index = 0
+  if boot_image_url_select_json_config:
+    # Support boot-image-url-select
+    with open(boot_image_url_select_json_config) as fh:
       image_config = json.load(fh)
     if image_config['error-amount'] == 0:
       for image in sorted(image_config['image-list'], key=lambda k: k['link']):
         link = os.path.join(image_config['destination-directory'], image['link'])
         if os.path.exists(link) and os.path.islink(link):
-          image_url_list_used = True
           kvm_argument_list.extend([
-            '-drive',
-            'file=%s,media=cdrom' % (link,)
+            '-drive', 'file=%s,media=cdrom,if=none,id=cdrom%s' % (link, index),
+            '-device', 'virtio-scsi-pci,id=scsi%s' % (index,),
+            '-device', 'scsi-cd,bus=scsi%s.0,drive=cdrom%s' % (index, index)
           ])
-  if not image_url_list_used:
-    # If no NBD is specified/available not downloadable image: use internal disk image
+          index += 1
+  if boot_image_url_list_json_config:
+    # Support boot-image-url-list
+    with open(boot_image_url_list_json_config) as fh:
+      image_config = json.load(fh)
+    if image_config['error-amount'] == 0:
+      for image in sorted(image_config['image-list'], key=lambda k: k['link']):
+        link = os.path.join(image_config['destination-directory'], image['link'])
+        if os.path.exists(link) and os.path.islink(link):
+          kvm_argument_list.extend([
+            '-drive', 'file=%s,media=cdrom,if=none,id=cdrom%s' % (link, index),
+            '-device', 'virtio-scsi-pci,id=scsi%s' % (index,),
+            '-device', 'scsi-cd,bus=scsi%s.0,drive=cdrom%s' % (index, index)
+          ])
+          index += 1
+  # Always add by default the default image
   kvm_argument_list.extend([
-        '-drive', 'file=%s,media=cdrom' % default_cdrom_iso
+      '-drive', 'file=%s,media=cdrom,if=none,id=cdrom%s' % (default_cdrom_iso, index),
+      '-device', 'virtio-scsi-pci,id=scsi%s' % (index,),
+      '-device', 'scsi-cd,bus=scsi%s.0,drive=cdrom%s' % (index, index)
   ])
 
 

software/kvm/test/test.py

@@ -486,8 +486,30 @@ class TestInstanceNbdServer(InstanceTestCase):
 
 
 @skipUnlessKvm
-class TestImageUrlList(InstanceTestCase):
-  __partition_reference__ = 'iul'
+class TestBootImageUrlList(InstanceTestCase):
+  __partition_reference__ = 'biul'
+
+  # variations
+  key = 'boot-image-url-list'
+  test_input = "%s#%s\n%s#%s"
+  image_directory = 'boot-image-url-list-repository'
+  config_state_promise = 'boot-image-url-list-config-state-promise.py'
+  download_md5sum_promise = 'boot-image-url-list-download-md5sum-promise.py'
+  download_state_promise = 'boot-image-url-list-download-state-promise.py'
+
+  bad_value = "jsutbad"
+  incorrect_md5sum_value_image = "%s#"
+  incorrect_md5sum_value = "url#asdasd"
+  single_image_value = "%s#%s"
+  unreachable_host_value = "evennotahost#%s"
+  too_many_image_value = """
+      image1#11111111111111111111111111111111
+      image2#22222222222222222222222222222222
+      image3#33333333333333333333333333333333
+      image4#44444444444444444444444444444444
+      image5#55555555555555555555555555555555
+      image6#66666666666666666666666666666666
+      """
 
   @classmethod
   def getInstanceSoftwareType(cls):
@@ -501,7 +523,7 @@ class TestImageUrlList(InstanceTestCase):
   def tearDown(self):
     # clean up the instance for other tests
     # 1st remove all images...
-    self.rerequestInstance({'image-url-list': ''})
+    self.rerequestInstance({self.key: ''})
     self.slap.waitForInstance(max_retry=10)
     # 2nd ...move instance to "default" state
     self.rerequestInstance({})
@@ -535,7 +557,7 @@ class TestImageUrlList(InstanceTestCase):
 
   def test(self):
     partition_parameter_kw = {
-      'image-url-list': "%s#%s\n%s#%s" % (
+      self.key: self.test_input % (
         self.fake_image, self.fake_image_md5sum, self.fake_image2,
         self.fake_image2_md5sum)
     }
@@ -543,7 +565,7 @@ class TestImageUrlList(InstanceTestCase):
     self.slap.waitForInstance(max_retry=10)
     # check that image is correctly downloaded and linked
     image_repository = os.path.join(
-      self.computer_partition_root_path, 'srv', 'image-repository')
+      self.computer_partition_root_path, 'srv', self.image_directory)
     image = os.path.join(image_repository, self.fake_image_md5sum)
     image_link = os.path.join(image_repository, 'image_001')
     self.assertTrue(os.path.exists(image))
@@ -562,19 +584,28 @@ class TestImageUrlList(InstanceTestCase):
     self.assertTrue(os.path.islink(image2_link))
     self.assertEqual(os.readlink(image2_link), image2)
 
-    # check that the image is NOT YET available in kvm
+    def getRunningImageList():
+      running_image_list = []
       with self.slap.instance_supervisor_rpc as instance_supervisor:
         kvm_pid = [q for q in instance_supervisor.getAllProcessInfo()
                    if 'kvm-' in q['name']][0]['pid']
         kvm_process = psutil.Process(kvm_pid)
-      cmd_line = ''.join(kvm_process.cmdline())
-      self.assertNotIn(
-        'srv/image-repository/image_001,media=cdrom',
-        cmd_line
-      )
-      self.assertNotIn(
-        'srv/image-repository/image_002,media=cdrom',
-        cmd_line
+        software_root = '/'.join([
+          self.slap.software_directory,
+          hashlib.md5(self.getSoftwareURL().encode('utf-8')).hexdigest()])
+        for entry in kvm_process.cmdline():
+          if entry.startswith('file') and 'media=cdrom' in entry:
+            # do cleanups
+            entry = entry.replace(software_root, '')
+            entry = entry.replace(self.computer_partition_root_path, '')
+            running_image_list.append(entry)
+      return running_image_list
+
+    # check that the image is NOT YET available in kvm
+    self.assertEqual(
+      ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
+       'media=cdrom,if=none,id=cdrom0'],
+      getRunningImageList()
     )
 
     # mimic the requirement: restart the instance by requesting it stopped and
@@ -584,30 +615,42 @@ class TestImageUrlList(InstanceTestCase):
     self.rerequestInstance(partition_parameter_kw, state='started')
     self.slap.waitForInstance(max_retry=1)
 
-    # now the image is available in the kvm
-    with self.slap.instance_supervisor_rpc as instance_supervisor:
-      kvm_pid = [q for q in instance_supervisor.getAllProcessInfo()
-                 if 'kvm-' in q['name']][0]['pid']
-      kvm_process = psutil.Process(kvm_pid)
-      cmd_line = ''.join(kvm_process.cmdline())
-      self.assertIn(
-        'srv/image-repository/image_001,media=cdrom',
-        cmd_line
-      )
-      self.assertIn(
-        'srv/image-repository/image_002,media=cdrom',
-        cmd_line
+    # now the image is available in the kvm, and its above default image
+    self.assertEqual(
+      [
+        'file=/srv/%s/image_001,media=cdrom,if=none,id=cdrom0' % (
+          self.image_directory,),
+        'file=/srv/%s/image_002,media=cdrom,if=none,id=cdrom1' % (
+          self.image_directory,),
+        'file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
+        'media=cdrom,if=none,id=cdrom2'
+      ],
+      getRunningImageList()
     )
 
     # cleanup of images works, also asserts that configuration changes are
     # reflected
-    self.rerequestInstance({'image-url-list': ''})
+    self.rerequestInstance({self.key: ''})
     self.slap.waitForInstance(max_retry=2)
     self.assertEqual(
       os.listdir(image_repository),
       []
     )
 
+    # mimic the requirement: restart the instance by requesting it stopped and
+    # then started started, like user have to do it
+    self.rerequestInstance(partition_parameter_kw, state='stopped')
+    self.slap.waitForInstance(max_retry=1)
+    self.rerequestInstance(partition_parameter_kw, state='started')
+    self.slap.waitForInstance(max_retry=1)
+
+    # again only default image is available in the running process
+    self.assertEqual(
+      ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
+       'media=cdrom,if=none,id=cdrom0'],
+      getRunningImageList()
+    )
+
   def assertPromiseFails(self, promise):
     monitor_run_promise = os.path.join(
       self.computer_partition_root_path, 'software_release', 'bin',
@@ -625,58 +668,180 @@ class TestImageUrlList(InstanceTestCase):
 
   def test_bad_parameter(self):
     self.rerequestInstance({
-      'image-url-list': "jsutbad"
+      self.key: self.bad_value
     })
     self.raising_waitForInstance(3)
-    self.assertPromiseFails('image-url-list-config-state-promise.py')
+    self.assertPromiseFails(self.config_state_promise)
 
   def test_incorrect_md5sum(self):
     self.rerequestInstance({
-      'image-url-list': "%s#" % (self.fake_image,)
+      self.key: self.incorrect_md5sum_value_image % (self.fake_image,)
     })
     self.raising_waitForInstance(3)
-    self.assertPromiseFails('image-url-list-config-state-promise.py')
+    self.assertPromiseFails(self.config_state_promise)
     self.rerequestInstance({
-      'image-url-list': "url#asdasd"
+      self.key: self.incorrect_md5sum_value
     })
     self.raising_waitForInstance(3)
-    self.assertPromiseFails('image-url-list-config-state-promise.py')
+    self.assertPromiseFails(self.config_state_promise)
 
   def test_not_matching_md5sum(self):
     self.rerequestInstance({
-      'image-url-list': "%s#%s" % (
+      self.key: self.single_image_value % (
         self.fake_image, self.fake_image_wrong_md5sum)
     })
     self.raising_waitForInstance(3)
-    self.assertPromiseFails('image-url-list-download-md5sum-promise.py')
-    self.assertPromiseFails('image-url-list-download-state-promise.py')
+    self.assertPromiseFails(self.download_md5sum_promise)
+    self.assertPromiseFails(self.download_state_promise)
 
   def test_unreachable_host(self):
     self.rerequestInstance({
-      'image-url-list': "evennotahost#%s" % (
+      self.key: self.unreachable_host_value % (
         self.fake_image_md5sum,)
     })
     self.raising_waitForInstance(3)
-    self.assertPromiseFails('image-url-list-download-state-promise.py')
+    self.assertPromiseFails(self.download_state_promise)
 
   def test_too_many_images(self):
     self.rerequestInstance({
-      'image-url-list': """
-      image1#11111111111111111111111111111111
-      image2#22222222222222222222222222222222
-      image3#33333333333333333333333333333333
-      image4#44444444444444444444444444444444
-      image5#55555555555555555555555555555555
-      image6#66666666666666666666666666666666
-      """
+      self.key: self.too_many_image_value
     })
     self.raising_waitForInstance(3)
-    self.assertPromiseFails('image-url-list-config-state-promise.py')
+    self.assertPromiseFails(self.config_state_promise)
 
 
 @skipUnlessKvm
-class TestImageUrlListKvmCluster(InstanceTestCase):
-  __partition_reference__ = 'iulkc'
+class TestBootImageUrlSelect(TestBootImageUrlList):
+  __partition_reference__ = 'bius'
+
+  # variations
+  key = 'boot-image-url-select'
+  test_input = '["%s#%s", "%s#%s"]'
+  image_directory = 'boot-image-url-select-repository'
+  config_state_promise = 'boot-image-url-select-config-state-promise.py'
+  download_md5sum_promise = 'boot-image-url-select-download-md5sum-promise.py'
+  download_state_promise = 'boot-image-url-select-download-state-promise.py'
+
+  bad_value = '["jsutbad"]'
+  incorrect_md5sum_value_image = '["%s#"]'
+  incorrect_md5sum_value = '["url#asdasd"]'
+  single_image_value = '["%s#%s"]'
+  unreachable_host_value = '["evennotahost#%s"]'
+  too_many_image_value = """[
+      "image1#11111111111111111111111111111111",
+      "image2#22222222222222222222222222222222",
+      "image3#33333333333333333333333333333333",
+      "image4#44444444444444444444444444444444",
+      "image5#55555555555555555555555555555555",
+      "image6#66666666666666666666666666666666"
+      ]"""
+
+  def test_not_json(self):
+    self.rerequestInstance({
+      self.key: 'notjson#notjson'
+    })
+    self.raising_waitForInstance(3)
+    self.assertPromiseFails(self.config_state_promise)
+
+  def test_together(self):
+    partition_parameter_kw = {
+      'boot-image-url-list': "%s#%s" % (
+        self.fake_image, self.fake_image_md5sum),
+      'boot-image-url-select': '["%s#%s"]' % (
+        self.fake_image, self.fake_image_md5sum)
+    }
+    self.rerequestInstance(partition_parameter_kw)
+    self.slap.waitForInstance(max_retry=10)
+    # check that image is correctly downloaded and linked
+    for image_directory in [
+      'boot-image-url-list-repository', 'boot-image-url-select-repository']:
+      image_repository = os.path.join(
+        self.computer_partition_root_path, 'srv', image_directory)
+      image = os.path.join(image_repository, self.fake_image_md5sum)
+      image_link = os.path.join(image_repository, 'image_001')
+      self.assertTrue(os.path.exists(image))
+      with open(image, 'rb') as fh:
+        image_md5sum = hashlib.md5(fh.read()).hexdigest()
+      self.assertEqual(image_md5sum, self.fake_image_md5sum)
+      self.assertTrue(os.path.islink(image_link))
+      self.assertEqual(os.readlink(image_link), image)
+
+    def getRunningImageList():
+      running_image_list = []
+      with self.slap.instance_supervisor_rpc as instance_supervisor:
+        kvm_pid = [q for q in instance_supervisor.getAllProcessInfo()
+                   if 'kvm-' in q['name']][0]['pid']
+        kvm_process = psutil.Process(kvm_pid)
+        software_root = '/'.join([
+          self.slap.software_directory,
+          hashlib.md5(self.getSoftwareURL().encode('utf-8')).hexdigest()])
+        for entry in kvm_process.cmdline():
+          if entry.startswith('file') and 'media=cdrom' in entry:
+            # do cleanups
+            entry = entry.replace(software_root, '')
+            entry = entry.replace(self.computer_partition_root_path, '')
+            running_image_list.append(entry)
+      return running_image_list
+
+    # check that the image is NOT YET available in kvm
+    self.assertEqual(
+      ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
+       'media=cdrom,if=none,id=cdrom0'],
+      getRunningImageList()
+    )
+
+    # mimic the requirement: restart the instance by requesting it stopped and
+    # then started started, like user have to do it
+    self.rerequestInstance(partition_parameter_kw, state='stopped')
+    self.slap.waitForInstance(max_retry=1)
+    self.rerequestInstance(partition_parameter_kw, state='started')
+    self.slap.waitForInstance(max_retry=1)
+
+    # now the image is available in the kvm, and its above default image
+    self.assertEqual(
+      [
+        'file=/srv/boot-image-url-select-repository/image_001,media=cdrom,'
+        'if=none,id=cdrom0',
+        'file=/srv/boot-image-url-list-repository/image_001,media=cdrom,'
+        'if=none,id=cdrom1',
+        'file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
+        'media=cdrom,if=none,id=cdrom2'
+      ],
+      getRunningImageList()
+    )
+
+    # cleanup of images works, also asserts that configuration changes are
+    # reflected
+    self.rerequestInstance(
+      {'boot-image-url-list': '', 'boot-image-url-select': ''})
+    self.slap.waitForInstance(max_retry=2)
+    for image_directory in [
+      'boot-image-url-list-repository', 'boot-image-url-select-repository']:
+      image_repository = os.path.join(
+        self.computer_partition_root_path, 'srv', image_directory)
+      self.assertEqual(
+        os.listdir(image_repository),
+        []
+      )
+
+    # mimic the requirement: restart the instance by requesting it stopped and
+    # then started started, like user have to do it
+    self.rerequestInstance(partition_parameter_kw, state='stopped')
+    self.slap.waitForInstance(max_retry=1)
+    self.rerequestInstance(partition_parameter_kw, state='started')
+    self.slap.waitForInstance(max_retry=1)
+
+    # again only default image is available in the running process
+    self.assertEqual(
+      ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
+       'media=cdrom,if=none,id=cdrom0'],
+      getRunningImageList()
+    )
+
+
+@skipUnlessKvm
+class TestBootImageUrlListKvmCluster(InstanceTestCase):
+  __partition_reference__ = 'biulkc'
 
   @classmethod
   def getInstanceSoftwareType(cls):
@@ -693,18 +858,22 @@ class TestImageUrlListKvmCluster(InstanceTestCase):
       "2a74b7675a8d8ca20476dba6e",)
   fake_image2_md5sum = "d4316a4d05f527d987b9d6e43e4c2bc6"
 
+  input_value = "%s#%s"
+  key = 'boot-image-url-list'
+  config_file_name = 'boot-image-url-list.conf'
+
   @classmethod
   def getInstanceParameterDict(cls):
     return {'_': json.dumps({
       "kvm-partition-dict": {
         "KVM0": {
             "disable-ansible-promise": True,
-            "image-url-list": "%s#%s" % (
+            cls.key: cls.input_value % (
               cls.fake_image, cls.fake_image_md5sum)
         },
         "KVM1": {
             "disable-ansible-promise": True,
-            "image-url-list": "%s#%s" % (
+            cls.key: cls.input_value % (
               cls.fake_image2, cls.fake_image2_md5sum)
         }
       }
@@ -715,22 +884,31 @@ class TestImageUrlListKvmCluster(InstanceTestCase):
     #       we assume ordering of the cluster requests
     KVM0_config = os.path.join(
       self.slap.instance_directory, self.__partition_reference__ + '1', 'etc',
-      'image-url-list.conf')
+      self.config_file_name)
     KVM1_config = os.path.join(
       self.slap.instance_directory, self.__partition_reference__ + '2', 'etc',
-      'image-url-list.conf')
+      self.config_file_name)
     with open(KVM0_config, 'r') as fh:
       self.assertEqual(
-        "%s#%s" % (self.fake_image, self.fake_image_md5sum),
-        fh.read()
+        self.input_value % (self.fake_image, self.fake_image_md5sum),
+        fh.read().strip()
       )
     with open(KVM1_config, 'r') as fh:
       self.assertEqual(
-        "%s#%s" % (self.fake_image2, self.fake_image2_md5sum),
-        fh.read()
+        self.input_value % (self.fake_image2, self.fake_image2_md5sum),
+        fh.read().strip()
       )
 
 
+@skipUnlessKvm
+class TestBootImageUrlSelectKvmCluster(TestBootImageUrlListKvmCluster):
+  __partition_reference__ = 'biuskc'
+
+  input_value = "[\"%s#%s\"]"
+  key = 'boot-image-url-select'
+  config_file_name = 'boot-image-url-select.json'
+
+
 @skipUnlessKvm
 class TestCpuMemMaxDynamic(InstanceTestCase):
   __partition_reference__ = 'cmm'

software/monitor/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = 1b5d5a72d0d0e3156770ce8ecc5d19ce
+md5sum = 9ddae686379e8d747410c1adf82b47d6
 
 [template-monitor]
 _update_hash_filename_ = instance-monitor.cfg.jinja2
@@ -30,7 +30,7 @@ md5sum = a57106ee88ff3295b9ffce84105da79b
 
 [template-monitor-edgebot]
 _update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
-md5sum = c535f4df6388cdc3f6c1df2a91d6dd53
+md5sum = c1885a42aadd45bab3185a53258d4ff4
 
 [network-bench-cfg]
 filename = network_bench.cfg.in

software/monitor/instance-edgetest-input-schema.json

@@ -13,6 +13,12 @@
       "description": "Default HTTP code to check against (default: 200).",
       "type": "string"
     },
+    "check-http-header-dict": {
+      "default": "{}",
+      "title": "HTTP header dict to check",
+      "description": "JSON dict of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}",
+      "type": "object"
+    },
     "check-frontend-ip": {
       "default": "",
       "title": "Default space separated list of Frontend IPs to check",

software/monitor/instance-edgetest-slave-input-schema.json

@@ -12,6 +12,12 @@
       "description": "HTTP code to check against (default: comes from master partition).",
       "type": "string"
     },
+    "check-http-header-dict": {
+      "default": "Master default",
+      "title": "HTTP header dict to check",
+      "description": "JSON dict of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}",
+      "type": "object"
+    },
     "check-frontend-ip": {
       "default": "Master default",
       "title": "Space separated list of Frontend IPs to check",

software/monitor/instance-monitor-edgebot.cfg.jinja2

@@ -13,6 +13,9 @@
 {%-   if 'check-status-code' not in slave %}
 {%-     do slave.__setitem__('check-status-code', CONFIGURATION['check-status-code']) %}
 {%-   endif %}
+{%-   if 'check-http-header-dict' not in slave %}
+{%-     do slave.__setitem__('check-http-header-dict', CONFIGURATION['check-http-header-dict']) %}
+{%-   endif %}
 {%-   if 'check-certificate-expiration-days' not in slave %}
 {%-     do slave.__setitem__('check-certificate-expiration-days', CONFIGURATION['check-certificate-expiration-days']) %}
 {%-   endif %}
@@ -48,6 +51,7 @@ name = {{ safe_name }}.py
 config-report = http_query
 config-url = {{ slave['url'] }}
 config-status-code = {{ slave['check-status-code'] }}
+config-http-header-dict = {{ slave['check-http-header-dict'] }}
 config-certificate-expiration-days = {{ slave['check-certificate-expiration-days'] }}
 config-failure-amount = {{ slave['failure-amount'] }}
 config-maximum-elapsed-time = {{ slave['check-maximum-elapsed-time'] }}

software/monitor/instance.cfg

@@ -76,6 +76,7 @@ key = $${slap-connection:key-file}
 cert = $${slap-connection:cert-file}
 # Defaults
 configuration.check-status-code = 200
+configuration.check-http-header-dict = {}
 configuration.nameserver =
 configuration.check-frontend-ip =
 configuration.check-certificate-expiration-days = 15

software/monitor/software.cfg

@@ -78,9 +78,9 @@ scripts =
 [versions]
 slapos.recipe.template = 4.4
 
-surykatka = 0.4.2
+surykatka = 0.5.0
 
-# For surykatka 0.4.2
+# For surykatka 0.5.0
 click = 7.0
 certifi = 2019.11.28
 chardet = 3.0.4

software/monitor/test/test.py

@@ -470,6 +470,90 @@ URL =
     )
 
 
+class TestEdgeCheckHTTPHeaderDict(EdgeSlaveMixin, SlapOSInstanceTestCase):
+  surykatka_dict = {
+    2: {'expected_ini': """[SURYKATKA]
+INTERVAL = 120
+TIMEOUT = 4
+SQLITE = %(db_file)s
+URL =
+  https://www.erp5.com/
+  https://www.erp5.org/"""}
+  }
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'check-http-header-dict':
+        '{"B": "BBB"}',
+    }
+
+  def assertSurykatkaPromises(self):
+    self.assertPromiseContent(
+      'http-query-backend-http-header-promise.py',
+      "'ip-list': ''")
+    self.assertPromiseContent(
+      'http-query-backend-http-header-promise.py',
+      "'report': 'http_query'")
+    self.assertPromiseContent(
+      'http-query-backend-http-header-promise.py',
+      "'status-code': '200'")
+    self.assertPromiseContent(
+      'http-query-backend-http-header-promise.py',
+      "'http-header-dict': '{\"A\": \"AAA\"}'")
+    self.assertPromiseContent(
+      'http-query-backend-http-header-promise.py',
+      "'certificate-expiration-days': '15'")
+    self.assertPromiseContent(
+      'http-query-backend-http-header-promise.py',
+      "'url': 'https://www.erp5.org/'")
+    self.assertPromiseContent(
+      'http-query-backend-http-header-promise.py',
+      "'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
+    )
+    self.assertPromiseContent(
+      'http-query-backend-http-header-promise.py',
+      "'failure-amount': '2'"
+    )
+
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'ip-list': ''")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'report': 'http_query'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'status-code': '200'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'http-header-dict': '{\"B\": \"BBB\"}'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'certificate-expiration-days': '15'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'url': 'https://www.erp5.com/'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
+    )
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'failure-amount': '2'"
+    )
+
+  def requestEdgetestSlaves(self):
+    self.requestEdgetestSlave(
+      'backend',
+      {'url': 'https://www.erp5.com/'},
+    )
+    self.requestEdgetestSlave(
+      'backend-http-header',
+      {'url': 'https://www.erp5.org/', 'check-http-header-dict': '{"A": "AAA"}'},
+    )
+
+
 class TestEdgeCheckCertificateExpirationDays(
   EdgeSlaveMixin, SlapOSInstanceTestCase):
   surykatka_dict = {

software/slapos-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = 47eb5f8e6f56729e2ce0e6213bccf4f7
+md5sum = a04806bdebd611c7f6698b4bef7af70d

software/slapos-testing/instance.cfg

@@ -88,7 +88,7 @@ command-line =
 #   guarantee about free ports on IPV4 and IPV6
 # * LOCAL_IPV4 is backward compatible, to be migrated, SLAPOS_TEST_IPV4
 environment =
-  PATH=${coreutils:location}/bin:${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:${socat:location}/bin:${lmsensors:location}/bin:${buildout:bin-directory}:/usr/bin/:/bin/
+  PATH=${coreutils:location}/bin:${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:${socat:location}/bin:${lmsensors:location}/bin:${rsync:location}/bin/:${buildout:bin-directory}:/usr/bin/:/bin/
   LOCAL_IPV4=$${slap-configuration:ipv4-random}
   SLAPOS_TEST_IPV4=$${slap-configuration:ipv4-random}
   SLAPOS_TEST_IPV6=$${slap-configuration:ipv6-random}

software/slapos-testing/software.cfg

@@ -12,6 +12,7 @@ extends =
   ../../component/coreutils/buildout.cfg
   ../../component/socat/buildout.cfg
   ../../component/lmsensors/buildout.cfg
+  ../../component/rsync/buildout.cfg
   ../../stack/slapos.cfg
   ./buildout.hash.cfg
 

software/theia/buildout.hash.cfg

@@ -15,12 +15,12 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = d8ad39bc93c492026a93c28b33a5dc3a
+md5sum = f95354d4af4a78ad7fd11ebc9281ed19
 
 [yarn.lock]
 filename = yarn.lock
-md5sum = 89d0a4d0c3ae90b9c5c0923b57766f0f
+md5sum = 07835b4acfbd8f4a96ac15ffecd238b0
 
 [python-language-server-requirements.txt]
 filename = python-language-server-requirements.txt
-md5sum = 2a296cba4c36d7a5fca5f4347a8c8469
+md5sum = d2ce161244ce9ebce5295302a1b2a7df

software/theia/instance.cfg.in

@@ -3,6 +3,7 @@ parts =
   monitor-base
   promises
   frontend-reload
+  tasks.json
   publish-connection-parameter
 
 extends = ${monitor-template:rendered}
@@ -58,6 +59,9 @@ template = inline:
     proxy /services $${theia-instance:base-url} {
       websocket
     }
+    proxy /file-upload $${theia-instance:base-url} {
+      websocket
+    }
     basicauth $${frontend-instance-password:username} $${frontend-instance-password:passwd} {
       realm "Theia"
       /
@@ -114,22 +118,101 @@ install =
 location = $${directory:frontend-static}/$${:filename}
 filename = $${:_buildout_section_name_}
 
-[user]
-recipe = slapos.cookbook:userinfo
+[tasks.json]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:dot-theia}/tasks.json
+template =
+  inline:
+  {
+    // See https://go.microsoft.com/fwlink/?LinkId=733558
+    // for the documentation about the tasks.json format
+    "version": "2.0.0",
+    "tasks": [
+      {
+        "label": "slapos node software",
+        "detail": "Build all software supplied to the node",
+        "type": "shell",
+        "command": "${buildout:bin-directory}/slapos",
+        "args": [
+          "node",
+          "software",
+          // debug mode can be enabled by commenting out this line:
+          // "--buildout-debug",
+          "--all"
+        ],
+        "options": {
+          "env": {
+            "SLAPOS_CONFIGURATION": "$${slapos-standalone-activate:slapos-configuration}",
+            "GIT_EXEC_PATH": ""
+          }
+        },
+        "group": {
+          "kind": "build",
+          "isDefault": true
+        },
+        "problemMatcher": []
+      },
+      {
+        "label": "slapos node instance",
+        "detail": "Create all instances requested on the node",
+        "type": "shell",
+        "command": "${buildout:bin-directory}/slapos",
+        "args": [
+          "node",
+          "instance",
+          // debug mode can be enabled by commenting out this line:
+          // "--buildout-debug",
+          "--all"
+        ],
+        "options": {
+          "env": {
+            "SLAPOS_CONFIGURATION": "$${slapos-standalone-activate:slapos-configuration}",
+            "GIT_EXEC_PATH": ""
+          }
+        },
+        "problemMatcher": [],
+        "group": {
+          "kind": "build",
+          "isDefault": true
+        }
+      }
+    ]
+  }
 
-[theia-instance]
-recipe = slapos.cookbook:wrapper
-wrapper-path = $${directory:services}/$${:_buildout_section_name_}
-command-line =
-  env LC_ALL=C.UTF-8 TMP=$${directory:tmp} THEIA_WEBVIEW_EXTERNAL_ENDPOINT='{{hostname}}' THEIA_SHELL=$${theia-shell:rendered} ${theia-wrapper:rendered} --hostname=$${:hostname} --port=$${:port} $${directory:project}
 
+[theia-service]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:bin}/$${:_buildout_section_name_}
+mode = 0700
+template =
+  inline:#!/bin/sh
+  {% raw %}
+  export THEIA_WEBVIEW_EXTERNAL_ENDPOINT='{{hostname}}'
+  {% endraw %}
+  export THEIA_OPEN_EDITOR_TOKEN=$(${openssl:location}/bin/openssl rand -hex 32)
+  export THEIA_URL=$${:base-url}
+  export THEIA_SHELL=$${theia-shell:rendered}
+  export HOME=$${buildout:directory}
+  export TMP=$${directory:tmp}
+  export TEMP=$TMP
+  export LC_ALL=C.UTF-8
+  export EDITOR="${python-language-server:location}/bin/python -m theia_open --wait"
+  exec ${theia-wrapper:rendered} $@
 ip =  $${instance-parameter:ipv4-random}
-hostname =  $${:ip}
 port = 3000
-base-url = http://$${:hostname}:$${:port}/
+base-url = http://$${:ip}:$${:port}/
+
+[theia-instance]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:services}/$${:_buildout_section_name_}
+command-line = $${theia-service:rendered}  --hostname=$${:hostname} --port=$${:port} $${directory:project}
 hash-existing-files =
   ${yarn.lock:output}
   ${theia-wrapper:rendered}
+ip =  $${instance-parameter:ipv4-random}
+hostname =  $${:ip}
+port = $${theia-service:port}
+base-url = $${theia-service:base-url}
 
 [theia-shell]
 recipe = slapos.recipe.template:jinja2
@@ -144,7 +227,6 @@ template = inline:
   if not args: args = ["-c", ". $${slapos-standalone-activate:rendered} && exec env GIT_EXEC_PATH= ${bash:location}/bin/bash", ]
   os.execv('${bash:location}/bin/bash', ['${bash:location}/bin/bash']  + args)
 
-
 [slapos-standalone-activate]
 recipe = slapos.recipe.template:jinja2
 rendered = $${directory:bin}/$${:_buildout_section_name_}
@@ -163,13 +245,13 @@ template =
       $${slap-connection:partition-id} \
       --key='$${slap-connection:key-file}' \
       --cert='$${slap-connection:cert-file}'
-  export SLAPOS_CONFIGURATION=$${directory:slapos}/etc/slapos.cfg
+  export SLAPOS_CONFIGURATION=$${:slapos-configuration}
   export SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION
 
 ipv4 = $${instance-parameter:ipv4-random}
 ipv6 = $${instance-parameter:ipv6-random}
 port = 4000
-
+slapos-configuration = $${directory:slapos}/etc/slapos.cfg
 
 [promises]
 recipe =
@@ -182,7 +264,7 @@ instance-promises =
 <= monitor-promise-base
 module = check_port_listening
 name = $${:_buildout_section_name_}.py
-config-hostname=  $${theia-instance:ip}
+config-hostname = $${theia-instance:ip}
 config-port = $${theia-instance:port}
 
 [frontend-listen-promise]
@@ -209,7 +291,7 @@ slave = true
 config-url = $${frontend-instance:url}
 config-https-only = true
 config-type = websocket
-config-websocket-path-list = /services
+config-websocket-path-list = /services /file-upload
 return = domain secure_access
 
 [publish-connection-parameter]
@@ -233,6 +315,7 @@ var = $${buildout:directory}/var
 srv = $${buildout:directory}/srv
 bin = $${buildout:directory}/bin
 tmp = $${buildout:directory}/tmp
+dot-theia = $${buildout:directory}/.theia/
 pidfiles = $${:var}/run
 
 services = $${:etc}/service

software/theia/python-language-server-requirements.txt

@@ -27,3 +27,4 @@ typing-extensions==3.7.4.2
 wrapt==1.11.2
 yapf==0.29.0
 zc.buildout.languageserver==0.2.1
+theia-open==0.1.2

software/theia/software.cfg

@@ -31,7 +31,7 @@ max_version = 0
 
 
 [nodejs]
-<= nodejs-10.19.0
+<= nodejs-12.18.3
 
 [yarn]
 # this could become a component, but it needs to be invoked from nodejs explicitly,
@@ -159,7 +159,7 @@ mode = 0644
 
 [package.json]
 recipe = slapos.recipe.template:jinja2
-# this comes from https://github.com/theia-ide/theia-apps/blob/3391dd07cba7ddc5cc833420349a27beb66a5433/theia-full-docker/latest.package.json
+# this comes from https://github.com/theia-ide/theia-apps/blob/a54aaa676e3db07d22ab75dde9f3447576135b4d/theia-full-docker/latest.package.json
 # but with a more recent version of vscode-java-redhat, where https://github.com/redhat-developer/vscode-java/issues/1301 was fixed
 template =
   inline:{
@@ -169,13 +169,15 @@ template =
               "config": {
                   "applicationName": "Theia SlapOS",
                   "preferences": {
+                      "application.confirmExit": "always",
                       "files.enableTrash": false,
                       "files.exclude": {
                           "**.pyc": true,
                           "**.egg-info": true,
                           "__pycache__": true,
                           ".git": true,
-                          ".env": true
+                          ".env": true,
+                          "**/node_modules/**": true
                       },
                       "files.watcherExclude": {
                           "**/.eggs/**": true,
@@ -185,8 +187,8 @@ template =
                       },
                       "editor.multiCursorModifier": "ctrlCmd",
                       "editor.tabSize": 2,
-                      "plantuml.monochrome": false,
-                      "plantuml.webservice": "//plantuml.host.vifib.net/svg/",
+                      "plantuml.server": "https://plantuml.host.vifib.net/svg/",
+                      "plantuml.render": "PlantUMLServer",
                       "gitlens.remotes": [{ "domain": "lab.nexedi.com", "type": "GitLab" }],
                       "java.home": "${java-jdk:location}"
                   }
@@ -205,9 +207,7 @@ template =
           "@theia/filesystem": "latest",
           "@theia/getting-started": "latest",
           "@theia/git": "latest",
-          "@theia/json": "latest",
           "@theia/keymaps": "latest",
-          "@theia/languages": "latest",
           "@theia/markers": "latest",
           "@theia/messages": "latest",
           "@theia/metrics": "latest",
@@ -216,14 +216,12 @@ template =
           "@theia/navigator": "latest",
           "@theia/outline-view": "latest",
           "@theia/output": "latest",
-          "@theia/plantuml": "latest",
           "@theia/plugin": "latest",
           "@theia/plugin-ext": "latest",
           "@theia/plugin-ext-vscode": "latest",
           "@theia/preferences": "latest",
           "@theia/preview": "latest",
           "@theia/process": "latest",
-          "@theia/rust": "latest",
           "@theia/scm": "latest",
           "@theia/search-in-workspace": "latest",
           "@theia/task": "latest",
@@ -232,14 +230,8 @@ template =
           "@theia/userstorage": "latest",
           "@theia/variable-resolver": "latest",
           "@theia/vsx-registry": "latest",
-          "@theia/workspace": "latest"
-      },
-      "resolutions": {
-          "vscode-json-languageserver": "1.2.2",
-          "vscode-languageserver-protocol": "3.15.0-next.9",
-          "vscode-languageserver-types": "3.15.0-next.5",
-          "**/vscode-json-languageserver/**/vscode-languageserver": "6.0.0-next.1",
-          "**/moment": "2.24.0"
+          "@theia/workspace": "latest",
+          "@perrinjerome/theia-open": "0.1.2"
       },
       "devDependencies": {
           "@theia/cli": "latest"
@@ -269,6 +261,7 @@ template =
           "vscode-builtin-java": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/java-1.39.1-prel.vsix",
           "vscode-builtin-javascript": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/javascript-1.39.1-prel.vsix",
           "vscode-builtin-json": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/json-1.39.1-prel.vsix",
+          "vscode-builtin-json-language-features": "https://open-vsx.org/api/vscode/json-language-features/1.46.1/file/vscode.json-language-features-1.46.1.vsix",
           "vscode-builtin-less": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/less-1.39.1-prel.vsix",
           "vscode-builtin-log": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/log-1.39.1-prel.vsix",
           "vscode-builtin-lua": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/lua-1.39.1-prel.vsix",
@@ -316,7 +309,8 @@ template =
           "vscode-java-test": "https://github.com/microsoft/vscode-java-test/releases/download/0.22.0/vscjava.vscode-java-test-0.22.0.vsix",
           "vscode-python": "https://github.com/microsoft/vscode-python/releases/download/2020.1.58038/ms-python-release.vsix",
           "vscode-ruby": "https://github.com/rubyide/vscode-ruby/releases/download/v0.25.0/ruby-0.25.0.vsix",
-          "vscode-zc-buildout": "https://github.com/perrinjerome/vscode-zc-buildout/releases/download/v0.2.0/vscode-zc-buildout-0.2.0.vsix"
+          "vscode-zc-buildout": "https://github.com/perrinjerome/vscode-zc-buildout/releases/download/v0.2.0/vscode-zc-buildout-0.2.0.vsix",
+          "plantuml": "https://open-vsx.org/api/jebbs/plantuml/2.13.12/file/jebbs.plantuml-2.13.12.vsix"
       }
     }
 rendered = ${buildout:directory}/${:_buildout_section_name_}
@@ -342,7 +336,7 @@ template =
   inline:
   #!/bin/bash
   . ${gowork:env.sh}
-  export PATH=${python-language-server:location}/bin/:${java-jdk:location}/bin/:${cli-utilities:PATH}:$PATH
+  export PATH=${python-language-server:location}/bin/:${java-jdk:location}/bin/:${cli-utilities:PATH}:$HOME/.cargo/bin:$PATH
   export THEIA_DEFAULT_PLUGINS="local-dir:${theia:THEIA_DEFAULT_PLUGINS}"
   # reset PS1 from gowork
   export PS1='$ '

stack/erp5/buildout.cfg

@@ -11,7 +11,6 @@ extends =
   ../../component/gzip/buildout.cfg
   ../../component/xz-utils/buildout.cfg
   ../../component/haproxy/buildout.cfg
-  ../../component/hookbox/buildout.cfg
   ../../component/findutils/buildout.cfg
   ../../component/librsvg/buildout.cfg
   ../../component/imagemagick/buildout.cfg
@@ -67,7 +66,6 @@ parts +=
   tesseract
   tesseract-eng-traineddata
   tesseract-osd-traineddata
-  hookbox
   zabbix-agent
 
 # Buildoutish
@@ -635,14 +633,10 @@ WSGIUtils = 0.7
 ZODB3 = 3.11.0
 # astroid 1.4.1 breaks testDynamicClassGeneration
 astroid = 1.3.8
-csp-eventlet = 0.7.0
 erp5diff = 0.8.1.7
-eventlet = 0.20.1
 five.formlib = 1.0.4
 five.localsitemanager = 2.0.5
 google-api-python-client = 1.6.1
-greenlet = 0.4.12
-http-parser = 0.8.3
 httplib2 = 0.10.3
 huBarcode = 1.0.0
 interval = 1.0.0
@@ -663,10 +657,7 @@ pyflakes = 1.5.0
 python-memcached = 1.58
 pytracemalloc = 1.2
 qrcode = 5.3
-restkit = 4.2.2
 rsa = 3.4.2
-rtjp-eventlet = 0.3.2
-socketpool = 0.5.3
 spyne = 2.12.14
 suds = 0.4
 facebook-sdk = 2.0.0
@@ -697,10 +688,6 @@ erp5.recipe.w3validator = 1.0.2
 # Products.CMFCore==2.2.10
 Products.ZSQLMethods = 2.13.5
 
-# Required by:
-# eventlet==0.20.1
-enum-compat = 0.0.2
-
 # Required by:
 # SOAPpy===0.12.0nxd001
 fpconst = 0.7.2

stack/slapos.cfg

@@ -136,7 +136,7 @@ eggs =
 [versions]
 setuptools = 44.0.0
 # Use SlapOS patched zc.buildout
-zc.buildout = 2.7.1+slapos008
+zc.buildout = 2.7.1+slapos009
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
 zc.recipe.egg = 2.0.3+slapos003
 # Use own version of h.r.download to be able to open .xz and .lz archives
@@ -173,14 +173,14 @@ pytz = 2016.10
 requests = 2.24.0
 six = 1.12.0
 slapos.cookbook = 1.0.152
-slapos.core = 1.6.1
+slapos.core = 1.6.2
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.20
 slapos.rebootstrap = 4.5
-slapos.recipe.build = 0.45
+slapos.recipe.build = 0.46
 slapos.recipe.cmmi = 0.16
-slapos.toolbox = 0.110
+slapos.toolbox = 0.111
 stevedore = 1.21.0
 subprocess32 = 3.5.3
 unicodecsv = 0.14.1