Merge branch 'master' into master+ZODB4-wc2

* master:
  stack/monitor: cleanup deprecated (and confusing) config
  Fix build for zodbtools/pygolang/nxdtest/... tests
  stack/lamp: remove old comment
  stack/monitor: change frontend url promise to use username/password
  upgrade to slapos.toolbox version 0.123

component/pytest/buildout.cfg

@@ -14,4 +14,4 @@ eggs    = ${pytest:eggs}
 
 
 [versions]
-pytest  = 4.6.11
+pytest  = 4.6.11:whl

software/caddy-frontend/buildout.hash.cfg

@@ -106,7 +106,7 @@ md5sum = 38792c2dceae38ab411592ec36fff6a8
 
 [profile-kedifa]
 filename = instance-kedifa.cfg.in
-md5sum = 16901e9eeb0d4f87e708ad91e7756f12
+md5sum = eab5ae579471ca86b40bd2da3b53fefa
 
 [template-backend-haproxy-rsyslogd-conf]
 _update_hash_filename_ = templates/backend-haproxy-rsyslogd.conf.in

software/caddy-frontend/instance-kedifa.cfg.in

@@ -253,7 +253,7 @@ extra-context =
 module = check_url_available
 name = kedifa-http-reply.py
 # Kedifa replies 400 on /, so use it to be sure that Kedifa replied
-config-http_code = 400
+config-http-code = 400
 config-url = https://[${kedifa-config:ip}]:${kedifa-config:port}
 config-ca-cert-file = ${kedifa-config:ca-certificate}
 

software/cloudooo/buildout.hash.cfg

@@ -18,4 +18,4 @@ md5sum = e986de01a57161b32425f1cd3ccac924
 
 [template-cloudooo-instance]
 filename = instance-cloudooo.cfg.in
-md5sum = 440f2b82b119cbfa6f8c7d27652c3170
+md5sum = 6e4bdb1df02aed5c96ccf7b9c3c71b89

software/cloudooo/instance-cloudooo.cfg.in

@@ -110,7 +110,7 @@ name = apache.py
 config-url = https://{{ ipv4 }}:{{ apache_dict.values()[0][0] }}
 # XXX cloudooo replies "400 Bad Request" for GET on / but what we want to check
 # is that we don't have a "503 Service Unavailable" from apache or haproxy.
-config-http_code = 400
+config-http-code = 400
 
 [apache-conf-ssl]
 cert = ${directory:apache-conf}/apache.crt

software/gitlab/buildout.hash.cfg

@@ -54,7 +54,7 @@ md5sum = 0f1ec4077dab586cc003ae13f689eda2
 
 [instance-gitlab.cfg.in]
 _update_hash_filename_ = instance-gitlab.cfg.in
-md5sum = dfeff229aa696bb6b6051a2aff4301fe
+md5sum = 64ec65b2daa0648453022f3afcbc4da3
 
 [instance-gitlab-export.cfg.in]
 _update_hash_filename_ = instance-gitlab-export.cfg.in

software/gitlab/instance-gitlab.cfg.in

@@ -444,7 +444,7 @@ tune-command =
 <= monitor-promise-base
 module = check_command_execute
 name   = ${:_buildout_section_name_}.py
-config-http_code   = 200
+config-http-code   = 200
 
 
 

software/html5as/buildout.hash.cfg

@@ -21,7 +21,7 @@ md5sum = 9e486efe4ab1aba8cb72b04f6c6da8ad
 
 [instance_html5as]
 _update_hash_filename_ = instance_html5as.cfg.in
-md5sum = 191ec2a8b967a3944971709365bdcd2d
+md5sum = 115918e0f8854e957ea8388bb064f457
 
 [template_nginx_conf]
 _update_hash_filename_ = templates/nginx_conf.in

software/html5as/instance_html5as.cfg.in

@@ -239,4 +239,4 @@ module = check_url_available
 name = html5as-http-frontend.py
 url = ${html5as-frontend:connection-secure_access}
 config-url = ${:url}
-config-check-secure = 1
+config-http-code = 401

software/jscrawler/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = 6c17361a49cfc47564063b867aab6e8c
 
 [template-jscrawler]
 filename = instance-jscrawler.cfg.jinja2.in
-md5sum = fece076231740b414612da5ef3a1685a
+md5sum = e76e35d9070bdeca014063e0a00879da
 
 [template-jscrawler-builder]
 filename = template-jscrawler.builder.sh.in

software/jscrawler/instance-jscrawler.cfg.jinja2.in

@@ -50,7 +50,7 @@ return = secure_access domain
 module = check_url_available
 name = jscrawler_frontend.py
 config-url = ${request-jscrawler-frontend:connection-secure_access}
-config-check-secure = 1
+config-http-code = 401
 
 [logrotate-entry-httpd]
 <= logrotate-entry-base

software/repman/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 8a08be95a04f1a47098c4fdef80bdfed
 
 [instance-repman.cfg]
 _update_hash_filename_ = instance-repman.cfg.jinja2.in
-md5sum = 0c173313b48d0005c46d968db1c8ab5f
+md5sum = b2273a47198d38f74c0bf84163b2ea1a
 
 [config-toml.in]
 _update_hash_filename_ = templates/config.toml.in

software/repman/instance-repman.cfg.jinja2.in

@@ -511,14 +511,14 @@ return = domain secure_access
 module = check_url_available
 name = check_repman_frontend.py
 config-url = https://${repman-frontend:connection-domain}
-config-check-secure = 1
+config-http-code = 401
 
 [repman-backend-promise]
 <= monitor-promise-base
 module = check_url_available
 name = check_repman_backend.py
 config-url = ${nginx-parameter:backend-ssl-url}
-config-check-secure = 1
+config-http-code = 401
 
 [template-proxysql-need-stop-start]
 recipe = slapos.recipe.template:jinja2

software/restic-rest-server/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [instance.cfg.in]
 filename = instance.cfg.in
-md5sum = fee2097f3d12fd4bcfbc2698ecf49afc
+md5sum = 0e5521cbbf78be884241e8cf585646ae

software/restic-rest-server/instance.cfg.in

@@ -185,7 +185,7 @@ return = domain secure_access
 [frontend-available-promise]
 <= check-url-available-promise
 url = ${frontend:connection-secure_access}
-check-secure = 1
+config-http-code = 401
 
 
 [promises]

software/slaprunner/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 8d6878ff1d2e75010c50a1a2b0c13b24
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = ebdecea5c1653ed752e06fbc8be7e6b2
+md5sum = 60b4d2025eace8da7de14b5bae3772d9
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2

software/slaprunner/instance-runner.cfg

@@ -91,7 +91,7 @@ module = check_url_available
 name = custom_frontend_promise.py
 config-url = https://$${request-custom-frontend:connection-domain}
 {% if slapparameter_dict.get('custom-frontend-basic-auth') -%}
-config-check-secure = 1
+config-http-code = 401
 {% endif -%}
 
 [custom-frontend-url-ready-promise-bin]
@@ -440,7 +440,7 @@ module = check_url_available
 name = $${:filename}.py
 filename = apache-httpd-listening-on-tcp
 config-url = $${apache-httpd:access-url}
-config-check-secure = 1
+config-http-code = 401
 
 [slaprunner-httpd-cors]
 recipe = plone.recipe.command
@@ -541,7 +541,7 @@ return = site_url domain
 module = check_url_available
 name = slaprunner_frontend.py
 config-url = https://$${request-frontend:connection-domain}/login
-config-check-secure = 1
+config-http-code = 401
 
 [request-httpd-frontend]
 <= slap-connection
@@ -561,7 +561,7 @@ return = secure_access domain
 module = check_url_available
 name = slaprunner-apache-http-frontend.py
 config-url = $${request-httpd-frontend:connection-secure_access}
-config-check-secure = 1
+config-http-code = 401
 
 {% endif %}
 

software/theia/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance-theia]
 _update_hash_filename_ = instance-theia.cfg.jinja.in
-md5sum = 557dec61b7bf58601051575234e7fd05
+md5sum = 11d347dd2bf762902341746a388673a0
 
 [instance]
 _update_hash_filename_ = instance.cfg.in

software/theia/instance-theia.cfg.jinja.in

@@ -94,7 +94,7 @@ config-port = $${frontend-instance:port}
 module = check_url_available
 name = $${:_buildout_section_name_}.py
 config-url = $${remote-frontend:connection-secure_access}
-config-check-secure = 1
+config-http-code = 401
 
 {% if additional_frontend %}
 [remote-additional-frontend-url-available-promise]
@@ -102,7 +102,7 @@ config-check-secure = 1
 module = check_url_available
 name = $${:_buildout_section_name_}.py
 config-url = $${remote-additional-frontend:connection-secure_access}
-config-check-secure = 1
+config-http-code = 401
 {% endif %}
 
 [slapos-standalone-listen-promise]

stack/lamp/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [instance]
 filename = instance.cfg.in
-md5sum = 5c953c0f5d3376718eb9c4030288647a
+md5sum = e4e070f93adaf917f9427ae9f35573d9
 
 [instance-apache-php]
 filename = instance-apache-php.cfg.in
@@ -22,7 +22,7 @@ md5sum = 4afee4377fa9cbc1e4ff80647b2f279c
 
 [instance-lamp]
 filename = instance-lamp.cfg.jinja2.in
-md5sum = 52b76d3c8aa23f467db33e32a2b50ed6
+md5sum = 437ec82ac37b28626091000ba7a991a6
 
 [template-apache.conf]
 filename = apache.conf.in

stack/lamp/instance-lamp.cfg.jinja2.in

@@ -82,7 +82,7 @@ name = lamp-http-frontend.py
 url = ${request-frontend:connection-secure_access}
 config-url = ${:url}
 config-custom-domain = {{ slapparameter_dict.get('custom-domain', '') }}
-config-check-secure = 1
+config-http-code = 401
 
 {% do publish_dict.__setitem__('url', '${lamp-frontend-promise:url}') -%}
 

stack/lamp/instance.cfg.in

@@ -8,7 +8,6 @@ develop-eggs-directory = {{ buildout_develop_directory }}
 offline = true
 
 [switch_softwaretype]
-#recipe = slapos.cookbook:softwaretype
 recipe = slapos.cookbook:switch-softwaretype
 default = dynamic-template-lamp:rendered
 RootSoftwareInstance = ${:default}

stack/monitor/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [monitor2-template]
 filename = instance-monitor.cfg.jinja2.in
-md5sum = 200bb126dbfc33e5c5c165ae17bab64b
+md5sum = fce342a392141f91c2e1a7b40efa9896
 
 [monitor-httpd-conf]
 _update_hash_filename_ = templates/monitor-httpd.conf.in

stack/monitor/instance-monitor.cfg.jinja2.in

@@ -97,7 +97,6 @@ parameter-file-path = ${monitor-instance-parameter:configuration-file-path}
 parameter-list =
   raw monitor-user ${monitor-instance-parameter:username}
   htpasswd monitor-password ${httpd-monitor-htpasswd:password-file} ${monitor-instance-parameter:username} ${httpd-monitor-htpasswd:htpasswd-path}
-  file min-free-disk-MB ${promise-check-free-disk-space:config-threshold-file}
   ${monitor-instance-parameter:instance-configuration}
 # htpasswd entry:  htpasswd key password-file username htpasswd-file
 
@@ -127,9 +126,9 @@ recipe = slapos.recipe.template:jinja2
 template = {{ monitor_conf_template }}
 rendered = ${directory:etc}/${:filename}
 filename = monitor.conf
-context = section parameter_dict           monitor-conf-parameters
+context = section parameter_dict          monitor-conf-parameters
           section promise_parameter_dict  monitor-promise-conf
-          section monitor_base_urls        monitor-base-url-dict
+          section monitor_base_urls       monitor-base-url-dict
 
 [start-monitor]
 recipe = slapos.cookbook:wrapper
@@ -312,7 +311,7 @@ output = ${directory:plugins}/${:name}
 module = check_url_available
 name = monitor-httpd-listening-on-tcp.py
 config-url = ${monitor-httpd-conf-parameter:url}
-config-check-secure = 1
+config-http-code = 401
 
 [monitor-publish-parameters]
 # XXX depends on monitor-base section
@@ -326,7 +325,7 @@ monitor-title = ${slap-configuration:instance-title}
 monitor-httpd-ipv6 = ${slap-configuration:ipv6-random}
 monitor-httpd-port = 8196
 # XXX - Set monitor-base-url = ${monitor-httpd-conf-parameter:url} => https://[ipv6]:port
-monitor-base-url = ${monitor-frontend-promise:url}
+monitor-base-url = ${monitor-frontend:connection-secure_access}
 #monitor-base-url = ${monitor-httpd-conf-parameter:url}
 root-instance-title = ${slap-configuration:root-instance-title}
 monitor-url-list =
@@ -356,13 +355,26 @@ config-https-only = true
 #software-type = custom-personal
 return = domain secure_access
 
+# Requests to the frontend URL should succeed with the correct
+# credentials.
+[check-monitor-password-promise]
+<= monitor-promise-base
+module = check_url_available
+name = check-monitor-frontend-password.py
+url = ${monitor-frontend:connection-secure_access}
+config-url = ${:url}
+config-username = ${monitor-instance-parameter:username}
+config-password = ${monitor-instance-parameter:username}
+
+# Requests to the frontend URL should fail when no credentials are
+# supplied.
 [monitor-frontend-promise]
 <= monitor-promise-base
 module = check_url_available
 name = monitor-http-frontend.py
 url = ${monitor-frontend:connection-secure_access}
 config-url = ${:url}
-config-check-secure = 1
+config-http-code = 401
 
 [monitor-bootstrap-promise]
 <= monitor-promise-base
@@ -383,7 +395,6 @@ config-monitor-url = ${monitor-instance-parameter:monitor-base-url}
 module = check_free_disk_space
 name = check-free-disk-space.py
 config-collectordb = ${monitor-instance-parameter:collector-db}
-config-threshold-file = ${directory:etc}/min-free-disk-size
 
 [monitor-base]
 # create dependencies between required monitor parts
@@ -402,6 +413,8 @@ depends =
   ${start-monitor:wrapper-path}
   ${ca-monitor-httpd-service:wrapper-path}
   ${monitor-httpd-promise:name}
+  ${monitor-frontend-promise:name}
+  ${check-monitor-password-promise:name}
   ${monitor-bootstrap-promise:name}
   ${monitor-symlink:recipe}
   ${promise-check-slapgrid:recipe}

stack/slapos.cfg

@@ -155,7 +155,7 @@ cliff = 2.8.3:whl
 cmd2 = 0.7.0
 collective.recipe.shelloutput = 0.1
 collective.recipe.template = 2.0
-configparser = 4.0.2
+configparser = 4.0.2:whl
 contextlib2 = 0.6.0.post1
 cryptography = 3.3.1
 dateparser = 0.7.6
@@ -165,7 +165,7 @@ gevent = 20.9.0
 geventmp = 0.0.1
 greenlet = 0.4.17
 idna = 2.9
-importlib-metadata = 1.7.0
+importlib-metadata = 1.7.0:whl
 inotify-simple = 1.1.1
 itsdangerous = 0.24
 lock-file = 2.0
@@ -178,8 +178,8 @@ pbr = 2.0.0
 plone.recipe.command = 1.1
 prettytable = 0.7.2
 psutil = 5.8.0
-pluggy = 0.13.1
-py = 1.9.0
+pluggy = 0.13.1:whl
+py = 1.9.0:whl
 pyOpenSSL = 19.1.0
 pyparsing = 2.2.0
 pytz = 2016.10
@@ -198,7 +198,7 @@ slapos.rebootstrap = 4.5
 slapos.recipe.build = 0.47
 slapos.recipe.cmmi = 0.17
 slapos.recipe.template = 4.6
-slapos.toolbox = 0.122
+slapos.toolbox = 0.123
 stevedore = 1.21.0:whl
 subprocess32 = 3.5.4
 unicodecsv = 0.14.1
@@ -207,7 +207,7 @@ wheel = 0.35.1:whl
 xml-marshaller = 1.0.2
 zc.lockfile = 1.0.2
 zdaemon = 4.2.0
-zipp = 1.2.0
+zipp = 1.2.0:whl
 zodburi = 2.4.0
 zope.event = 3.5.2
 paramiko = 2.1.3