Merge remote-tracking branch 'origin/master' into t

component/mariadb/buildout.cfg

@@ -30,8 +30,8 @@ parts =
 recipe = slapos.recipe.cmmi
 shared = true
 url = https://archive.mariadb.org//mariadb-${:version}/source/mariadb-${:version}.tar.gz
-version = 10.4.16
-md5sum = e7e5071cc879de5358f60789f53df99e
+version = 10.4.17
+md5sum = e8193b9cd008b6d7f177f5a5c44c7a9f
 location = @@LOCATION@@
 pre-configure =
   set '\bSET(PLUGIN_AUTH_PAM YES CACHE BOOL "")' cmake/build_configurations/mysql_release.cmake
@@ -131,8 +131,8 @@ environment =
 ### (we just override here for easier revert)
 [mariadb-10.3]
 <= mariadb-10.4
-version = 10.3.26
-md5sum = 7a3eda171db192c15ec31ac2520551ad
+version = 10.3.27
+md5sum = 6ab2934a671191d8ca8730e9a626c5c9
 post-install =
   ldd=`ldd ${:location}/lib/plugin/ha_rocksdb.so`
   for x in ${lz4:location} ${snappy:location} ${zstd:location}

software/caddy-frontend/README.caddy_frontend.rst

@@ -245,6 +245,15 @@ Necessary to activate cache.
 
 ``enable_cache`` is an optional parameter.
 
+backend-active-check-*
+~~~~~~~~~~~~~~~~~~~~~~
+
+This set of parameters is used to control the way how the backend checks will be done. Such active checks can be really useful for `stale-if-error` caching technique and especially in case if backend is very slow to reply or to connect to.
+
+`backend-active-check-http-method` can be used to configure the HTTP method used to check the backend. Special method `CONNECT` can be used to check only for connection attempt.
+
+Please be aware that the `backend-active-check-timeout` is really short by default, so in case if `/` of the backend is slow to reply configure proper path with `backend-active-check-http-path` to not mark such backend down too fast, before increasing the check timeout.
+
 Examples
 ========
 

software/caddy-frontend/buildout.hash.cfg

@@ -26,11 +26,11 @@ md5sum = e7d7e1448b6420657e953026573311ca
 
 [profile-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = 2d421ce6def12f7796cfa28f59eef0df
+md5sum = c713aa837c5d0af8b59ed65e17c36c3a
 
 [profile-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = ab143bfa2e20725aa35940c9033fa0ee
+md5sum = 8f187ebb6807732c9e40bdf03e6a8113
 
 [profile-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
@@ -50,7 +50,7 @@ md5sum = 266f175dbdfc588af7a86b0b1884fe73
 
 [template-backend-haproxy-configuration]
 _update_hash_filename_ = templates/backend-haproxy.cfg.in
-md5sum = bf40f8d0a049a8dd924ccc731956c87e
+md5sum = 7c171979aa1d5c88faafe4993f087edd
 
 [template-log-access]
 _update_hash_filename_ = templates/template-log-access.conf.in

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -124,6 +124,35 @@ context =
 {%   elif slave_type not in [None, '', 'default', 'zope', 'redirect', 'notebook', 'websocket'] %}
 {%     do slave_error_list.append('type:%s is not supported' % (slave_type,)) %}
 {%   endif %}
+{#   Check backend-active-check-* #}
+{%   set backend_active_check = (str(slave.get('backend-active-check', False)) or 'false').lower() %}
+{%   if backend_active_check in TRUE_VALUES %}
+{%     set backend_active_check_http_method = slave.get('backend-active-check-http-method') or 'GET' %}
+{%     if backend_active_check_http_method not in ['GET', 'OPTIONS', 'CONNECT', 'POST'] %}
+{%       do slave_error_list.append('Wrong backend-active-check-http-method %s' % (backend_active_check_http_method,)) %}
+{%     endif %}
+{%     set backend_active_check_http_path = slave.get('backend-active-check-http-path') or '/' %}
+{%     set backend_active_check_http_version = slave.get('backend-active-check-http-version') or 'HTTP/1.1' %}
+{%     if backend_active_check_http_version not in ['HTTP/1.1', 'HTTP/1.0'] %}
+{%       do slave_error_list.append('Wrong backend-active-check-http-version %s' % (backend_active_check_http_version,)) %}
+{%     endif %}
+{%     set backend_active_check_timeout = (slave.get('backend-active-check-timeout') or '2') | int(False) %}
+{%     if backend_active_check_timeout in [False] or backend_active_check_timeout <= 0 %}
+{%       do slave_error_list.append('Wrong backend-active-check-timeout %s' % (slave.get('backend-active-check-timeout'),)) %}
+{%     endif %}
+{%     set backend_active_check_interval = (slave.get('backend-active-check-interval') or '5') | int(False) %}
+{%     if backend_active_check_interval in [False] or backend_active_check_interval <= 0 %}
+{%       do slave_error_list.append('Wrong backend-active-check-interval %s' % (slave.get('backend-active-check-interval'),)) %}
+{%     endif %}
+{%     set backend_active_check_rise = (slave.get('backend-active-check-rise') or '1') | int(False) %}
+{%     if backend_active_check_rise in [False] or backend_active_check_rise <= 0 %}
+{%       do slave_error_list.append('Wrong backend-active-check-rise %s' % (slave.get('backend-active-check-rise'),)) %}
+{%     endif %}
+{%     set backend_active_check_fall = (slave.get('backend-active-check-fall') or '1') | int(False) %}
+{%     if backend_active_check_fall in [False] or backend_active_check_fall <= 0 %}
+{%       do slave_error_list.append('Wrong backend-active-check-fall %s' % (slave.get('backend-active-check-fall'),)) %}
+{%     endif %}
+{%   endif %}
 {#   Check ciphers #}
 {%   set slave_cipher_list = slave.get('ciphers', '').strip().split() %}
 {%   if slave_cipher_list %}

software/caddy-frontend/instance-slave-caddy-input-schema.json

@@ -223,6 +223,68 @@
       ],
       "title": "Authenticate to backend",
       "type": "string"
+    },
+    "backend-active-check": {
+      "title": "Backend Active Check",
+      "description": "Enables active checks of the backend. For HTTP level checks the HTTP code shall be 2xx or 3xx, otherwise backend will be considered down.",
+      "enum": [
+        "false",
+        "true"
+      ],
+      "default": "false",
+      "type": "string"
+    },
+    "backend-active-check-http-method": {
+      "title": "Backend Active Check HTTP Metod",
+      "description": "Selects method to do the active check. CONNECT means that connection will be enough for the check, otherwise it's HTTP method.",
+      "enum": [
+        "GET",
+        "OPTIONS",
+        "POST",
+        "CONNECT"
+      ],
+      "default": "GET",
+      "type": "string"
+    },
+    "backend-active-check-http-path": {
+      "title": "Backend Active Check HTTP Path",
+      "description": "A path on which do the active check, unused in case of CONNECT.",
+      "default": "/",
+      "type": "string"
+    },
+    "backend-active-check-http-version": {
+      "title": "Backend Active Check HTTP Version",
+      "description": "A HTTP version to use to check the backend, unused in case of CONNECT.",
+      "enum": [
+        "HTTP/1.1",
+        "HTTP/1.0"
+      ],
+      "default": "HTTP/1.1",
+      "type": "string"
+    },
+    "backend-active-check-timeout": {
+      "title": "Backend Active Check Timeout (seconds)",
+      "description": "A timeout to for the request to be fulfilled, after connection happen.",
+      "default": "2",
+      "type": "integer"
+    },
+    "backend-active-check-interval": {
+      "title": "Backend Active Check Interval (seconds)",
+      "description": "An interval of backend active check.",
+      "default": "5",
+      "type": "integer"
+    },
+    "backend-active-check-rise": {
+      "title": "Backend Active Check Rise",
+      "description": "Amount of correct responses from the backend to consider it up.",
+      "default": "1",
+      "type": "integer"
+    },
+    "backend-active-check-fall": {
+      "title": "Backend Active Check Fall",
+      "description": "Amount of bad responses from the backend to consider it down.",
+      "default": "1",
+      "type": "integer"
     }
   },
   "title": "Input Parameters",

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -113,6 +113,33 @@ context =
 {%-     endif %}
 {%-   endfor %}
 {%-   do slave_instance.__setitem__('authenticate-to-backend', ('' ~ slave_instance.get('authenticate-to-backend', '')).lower() in TRUE_VALUES) %}
+{#-   Setup active check #}
+{%-   do slave_instance.__setitem__('backend-active-check',  ('' ~ slave_instance.get('backend-active-check', '')).lower() in TRUE_VALUES) %}
+{%-   if slave_instance['backend-active-check']  %}
+{%-     if 'backend-active-check-http-method' not in slave_instance %}
+{%-       do slave_instance.__setitem__('backend-active-check-http-method', 'GET') %}
+{%-     endif %}
+{%-     if 'backend-active-check-http-version' not in slave_instance %}
+{%-       do slave_instance.__setitem__('backend-active-check-http-version', 'HTTP/1.1') %}
+{%-     endif %}
+{%-     if 'backend-active-check-interval' not in slave_instance %}
+{%-       do slave_instance.__setitem__('backend-active-check-interval', '5') %}
+{%-     endif %}
+{%-     if 'backend-active-check-rise' not in slave_instance %}
+{%-       do slave_instance.__setitem__('backend-active-check-rise', '1') %}
+{%-     endif %}
+{%-     if 'backend-active-check-fall' not in slave_instance %}
+{%-       do slave_instance.__setitem__('backend-active-check-fall', '2') %}
+{%-     endif %}
+{%-     if 'backend-active-check-timeout' not in slave_instance %}
+{%-       do slave_instance.__setitem__('backend-active-check-timeout', '2') %}
+{%-     endif %}
+{%-     do slave_instance.__setitem__('backend-active-check-http-path', slave_instance.get('backend-active-check-http-path') or '/') %}
+{%-   else %}
+{%-     do slave_instance.__setitem__('backend-active-check-http-method', '') %}
+{%-     do slave_instance.__setitem__('backend-active-check-http-version', '') %}
+{%-     do slave_instance.__setitem__('backend-active-check-http-path', '') %}
+{%-   endif %} {# if backend_active_check #}
 {#-   Set Up log files #}
 {%-   do slave_parameter_dict.__setitem__('access_log', '/'.join([caddy_log_directory, '%s_access_log' % slave_reference])) %}
 {%-   do slave_parameter_dict.__setitem__('error_log', '/'.join([caddy_log_directory, '%s_error_log' % slave_reference])) %}

software/caddy-frontend/templates/backend-haproxy.cfg.in

@@ -100,7 +100,22 @@ backend {{ slave_instance['slave_reference'] }}-{{ scheme }}
   timeout server {{ slave_instance['request-timeout'] }}s
   timeout connect {{ slave_instance['backend-connect-timeout'] }}s
   retries {{ slave_instance['backend-connect-retries'] }}
-  server {{ slave_instance['slave_reference'] }}-backend {{ hostname }}:{{ port }} {{ ' '.join(ssl_list) }}
+{%-         set active_check_list = [] %}
+{%-         set active_check_option_list = [] %}
+{%-         if slave_instance['backend-active-check'] %}
+{%-           do active_check_list.append('check') %}
+{%-           do active_check_list.append('inter %ss' % (slave_instance['backend-active-check-interval'])) %}
+{%-           do active_check_list.append('rise %s' % (slave_instance['backend-active-check-rise'])) %}
+{%-           do active_check_list.append('fall %s' % (slave_instance['backend-active-check-fall'])) %}
+{%-           if slave_instance['backend-active-check-http-method'] != 'CONNECT' %}
+{%-             do active_check_option_list.append('option httpchk %s %s %s' % (slave_instance['backend-active-check-http-method'], slave_instance['backend-active-check-http-path'] | urlencode, slave_instance['backend-active-check-http-version'])) %}
+{%-           endif %}
+{%-           do active_check_option_list.append('timeout check %ss' % (slave_instance['backend-active-check-timeout'])) %}
+{%-         endif %}
+  server {{ slave_instance['slave_reference'] }}-backend {{ hostname }}:{{ port }} {{ ' '.join(ssl_list) }} {{ ' ' + ' '.join(active_check_list)}}
+{%-         for active_check_option in active_check_option_list %}
+  {{ active_check_option }}
+{%-         endfor %}
 {%-         if path %}
   http-request set-path {{ path }}%[path]
 {%-         endif %}

software/caddy-frontend/test/test.py

@@ -48,6 +48,7 @@ from slapos.recipe.librecipe import generateHashFromFiles
 import xml.etree.ElementTree as ET
 import urlparse
 import socket
+import sys
 
 
 try:
@@ -430,6 +431,9 @@ def fakeHTTPResult(domain, real_ip, path, port=HTTP_PORT,
 class TestHandler(BaseHTTPRequestHandler):
   identification = None
 
+  def do_POST(self):
+    return self.do_GET()
+
   def do_GET(self):
     timeout = int(self.headers.dict.get('timeout', '0'))
     compress = int(self.headers.dict.get('compress', '0'))
@@ -487,12 +491,6 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
   # minimise partition path
   __partition_reference__ = 'T-'
 
-  @classmethod
-  def getInstanceSoftwareType(cls):
-    # Because of unknown problem yet, the root instance software type changes
-    # from RootSoftwareInstance to '', so always request it with given type
-    return "RootSoftwareInstance"
-
   @classmethod
   def prepareCertificate(cls):
     cls.another_server_ca = CertificateAuthority("Another Server Root CA")
@@ -920,18 +918,32 @@ class SlaveHttpFrontendTestCase(HttpFrontendTestCase):
     return default_instance
 
   @classmethod
-  def requestSlaves(cls):
+  def requestSlaveInstance(cls, partition_reference, partition_parameter_kw):
     software_url = cls.getSoftwareURL()
+    software_type = cls.getInstanceSoftwareType()
+    cls.logger.debug(
+      'requesting slave "%s" type: %r software:%s parameters:%s',
+      partition_reference, software_type, software_url, partition_parameter_kw)
+    return cls.slap.request(
+      software_release=software_url,
+      software_type=software_type,
+      partition_reference=partition_reference,
+      partition_parameter_kw=partition_parameter_kw,
+      shared=True
+    )
+
+  @classmethod
+  def requestSlaves(cls):
     for slave_reference, partition_parameter_kw in cls\
             .getSlaveParameterDictDict().items():
+      software_url = cls.getSoftwareURL()
+      software_type = cls.getInstanceSoftwareType()
       cls.logger.debug(
-        'requesting slave "%s" software:%s parameters:%s',
-        slave_reference, software_url, partition_parameter_kw)
-      cls.slap.request(
-        software_release=software_url,
+        'requesting slave "%s" type: %r software:%s parameters:%s',
+        slave_reference, software_type, software_url, partition_parameter_kw)
+      cls.requestSlaveInstance(
         partition_reference=slave_reference,
         partition_parameter_kw=partition_parameter_kw,
-        shared=True
       )
 
   @classmethod
@@ -971,11 +983,9 @@ class SlaveHttpFrontendTestCase(HttpFrontendTestCase):
 
     for slave_reference, partition_parameter_kw in cls\
             .getSlaveParameterDictDict().items():
-      parameter_dict_list.append(cls.slap.request(
-        software_release=cls.getSoftwareURL(),
+      parameter_dict_list.append(cls.requestSlaveInstance(
         partition_reference=slave_reference,
         partition_parameter_kw=partition_parameter_kw,
-        shared=True
       ).getConnectionParameterDict())
     return parameter_dict_list
 
@@ -1009,14 +1019,11 @@ class SlaveHttpFrontendTestCase(HttpFrontendTestCase):
   def updateSlaveConnectionParameterDictDict(cls):
     cls.slave_connection_parameter_dict_dict = {}
     # run partition for slaves to be setup
-    request = cls.slap.request
     for slave_reference, partition_parameter_kw in cls\
             .getSlaveParameterDictDict().items():
-      slave_instance = request(
-        software_release=cls.getSoftwareURL(),
+      slave_instance = cls.requestSlaveInstance(
         partition_reference=slave_reference,
         partition_parameter_kw=partition_parameter_kw,
-        shared=True
       )
       cls.slave_connection_parameter_dict_dict[slave_reference] = \
           slave_instance.getConnectionParameterDict()
@@ -5837,11 +5844,9 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
       ssl_ca_crt=ca.certificate_pem,
     )
 
-    self.slap.request(
-        software_release=self.getSoftwareURL(),
+    self.requestSlaveInstance(
         partition_reference='custom_domain_ssl_crt_ssl_key_ssl_ca_crt',
         partition_parameter_kw=slave_parameter_dict,
-        shared=True
     )
 
     self.slap.waitForInstance()
@@ -6289,6 +6294,46 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
         'ssl_key': '${section:option}ssl_keyunsafe\nunsafe',
         'ssl_crt': '${section:option}ssl_crtunsafe\nunsafe',
       },
+      'backend-active-check-http-method': {
+        'backend-active-check': True,
+        'backend-active-check-http-method': 'WRONG',
+      },
+      'backend-active-check-http-version': {
+        'backend-active-check': True,
+        'backend-active-check-http-version': 'WRONG/1.1',
+      },
+      'backend-active-check-timeout': {
+        'backend-active-check': True,
+        'backend-active-check-timeout': 'WRONG',
+      },
+      'backend-active-check-timeout-negative': {
+        'backend-active-check': True,
+        'backend-active-check-timeout': '-2',
+      },
+      'backend-active-check-interval': {
+        'backend-active-check': True,
+        'backend-active-check-interval': 'WRONG',
+      },
+      'backend-active-check-interval-negative': {
+        'backend-active-check': True,
+        'backend-active-check-interval': '-2',
+      },
+      'backend-active-check-rise': {
+        'backend-active-check': True,
+        'backend-active-check-rise': 'WRONG',
+      },
+      'backend-active-check-rise-negative': {
+        'backend-active-check': True,
+        'backend-active-check-rise': '-2',
+      },
+      'backend-active-check-fall': {
+        'backend-active-check': True,
+        'backend-active-check-fall': 'WRONG',
+      },
+      'backend-active-check-fall-negative': {
+        'backend-active-check': True,
+        'backend-active-check-fall': '-2',
+      }
     }
 
   def test_master_partition_state(self):
@@ -6303,8 +6348,8 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
       'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
       'domain': 'example.com',
       'accepted-slave-amount': '7',
-      'rejected-slave-amount': '14',
-      'slave-amount': '21',
+      'rejected-slave-amount': '24',
+      'slave-amount': '31',
       'rejected-slave-dict': {
         '_HTTPS-URL': ['slave https-url "https://[fd46::c2ae]:!py!u\'123123\'"'
                        ' invalid'],
@@ -6339,6 +6384,26 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
         '_EMPTY-BACKEND': [
           "slave https-url '' invalid",
           "slave url '' invalid"],
+        '_backend-active-check-fall': [
+          'Wrong backend-active-check-fall WRONG'],
+        '_backend-active-check-fall-negative': [
+          'Wrong backend-active-check-fall -2'],
+        '_backend-active-check-http-method': [
+          'Wrong backend-active-check-http-method WRONG'],
+        '_backend-active-check-http-version': [
+          'Wrong backend-active-check-http-version WRONG/1.1'],
+        '_backend-active-check-interval': [
+          'Wrong backend-active-check-interval WRONG'],
+        '_backend-active-check-interval-negative': [
+          'Wrong backend-active-check-interval -2'],
+        '_backend-active-check-rise': [
+          'Wrong backend-active-check-rise WRONG'],
+        '_backend-active-check-rise-negative': [
+          'Wrong backend-active-check-rise -2'],
+        '_backend-active-check-timeout': [
+          'Wrong backend-active-check-timeout WRONG'],
+        '_backend-active-check-timeout-negative': [
+          'Wrong backend-active-check-timeout -2'],
       },
       'warning-slave-dict': {
         '_SSL_CA_CRT_ONLY': [
@@ -7079,3 +7144,139 @@ class TestPassedRequestParameter(HttpFrontendTestCase):
       expected_partition_parameter_dict_dict,
       partition_parameter_dict_dict
     )
+
+
+class TestSlaveBackendActiveCheck(SlaveHttpFrontendTestCase, TestDataMixin):
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'domain': 'example.com',
+      'public-ipv4': cls._ipv4_address,
+      'port': HTTPS_PORT,
+      'plain_http_port': HTTP_PORT,
+      'kedifa_port': KEDIFA_PORT,
+      'caucase_port': CAUCASE_PORT,
+      'mpm-graceful-shutdown-timeout': 2,
+      'request-timeout': '12',
+    }
+
+  @classmethod
+  def getSlaveParameterDictDict(cls):
+    cls.setUpAssertionDict()
+    return {
+      'backend-active-check-disabled': {
+        'url': cls.backend_url,
+      },
+      'backend-active-check-default': {
+        'url': cls.backend_url,
+        'backend-active-check': True,
+      },
+      'backend-active-check-connect': {
+        'url': cls.backend_url,
+        'backend-active-check': True,
+        'backend-active-check-http-method': 'CONNECT',
+      },
+      'backend-active-check-custom': {
+        'url': cls.backend_url,
+        'backend-active-check': True,
+        'backend-active-check-http-method': 'POST',
+        'backend-active-check-http-path': '/POST-path to be encoded',
+        'backend-active-check-http-version': 'HTTP/1.0',
+        'backend-active-check-timeout': '7',
+        'backend-active-check-interval': '15',
+        'backend-active-check-rise': '3',
+        'backend-active-check-fall': '7',
+      },
+    }
+
+  @classmethod
+  def setUpAssertionDict(cls):
+    backend = urlparse.urlparse(cls.backend_url).netloc
+    cls.assertion_dict = {
+      'backend-active-check-disabled': """\
+backend _backend-active-check-disabled-http
+  timeout server 12s
+  timeout connect 5s
+  retries 3
+  server _backend-active-check-disabled-backend %s""" % (backend,),
+      'backend-active-check-connect': """\
+backend _backend-active-check-connect-http
+  timeout server 12s
+  timeout connect 5s
+  retries 3
+  server _backend-active-check-connect-backend %s   check inter 5s"""
+      """ rise 1 fall 2
+  timeout check 2s""" % (backend,),
+      'backend-active-check-custom': """\
+backend _backend-active-check-custom-http
+  timeout server 12s
+  timeout connect 5s
+  retries 3
+  server _backend-active-check-custom-backend %s   check inter 15s"""
+      """ rise 3 fall 7
+  option httpchk POST /POST-path%%20to%%20be%%20encoded HTTP/1.0
+  timeout check 7s""" % (backend,),
+      'backend-active-check-default': """\
+backend _backend-active-check-default-http
+  timeout server 12s
+  timeout connect 5s
+  retries 3
+  server _backend-active-check-default-backend %s   check inter 5s"""
+      """ rise 1 fall 2
+  option httpchk GET / HTTP/1.1
+  timeout check 2s""" % (backend, )
+    }
+
+  def _get_backend_haproxy_configuration(self):
+    backend_configuration_file = glob.glob(os.path.join(
+      self.instance_path, '*', 'etc', 'backend-haproxy.cfg'))[0]
+    with open(backend_configuration_file) as fh:
+      return fh.read()
+
+  def _test(self, key):
+    parameter_dict = self.assertSlaveBase(key)
+    self.assertIn(
+      self.assertion_dict[key],
+      self._get_backend_haproxy_configuration()
+    )
+    result = fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'],
+      'test-path/deep/.././deeper',
+      headers={
+        'Timeout': '10',  # more than default backend-connect-timeout == 5
+        'Accept-Encoding': 'gzip',
+      }
+    )
+    self.assertEqual(
+      self.certificate_pem,
+      der2pem(result.peercert))
+
+    self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
+
+  def test_backend_active_check_disabled(self):
+    self._test('backend-active-check-disabled')
+
+  def test_backend_active_check_default(self):
+    self._test('backend-active-check-default')
+
+  def test_backend_active_check_connect(self):
+    self._test('backend-active-check-connect')
+
+  def test_backend_active_check_custom(self):
+    self._test('backend-active-check-custom')
+
+
+if __name__ == '__main__':
+  class HTTP6Server(HTTPServer):
+    address_family = socket.AF_INET6
+  ip, port = sys.argv[1], int(sys.argv[2])
+  if ':' in ip:
+    klass = HTTP6Server
+    url_template = 'http://[%s]:%s/'
+  else:
+    klass = HTTPServer
+    url_template = 'http://%s:%s/'
+
+  server = klass((ip, port), TestHandler)
+  print url_template % server.server_address[:2]
+  server.serve_forever()

software/caddy-frontend/test/test_data/test.TestSlaveBackendActiveCheck.test_file_list_etc_cron_d-CADDY.txt

+T-0/etc/cron.d/logrotate
+T-0/etc/cron.d/monitor-configurator
+T-0/etc/cron.d/monitor-globalstate
+T-0/etc/cron.d/monitor_collect
+T-1/etc/cron.d/logrotate
+T-1/etc/cron.d/monitor-configurator
+T-1/etc/cron.d/monitor-globalstate
+T-1/etc/cron.d/monitor_collect
+T-2/etc/cron.d/logrotate
+T-2/etc/cron.d/monitor-configurator
+T-2/etc/cron.d/monitor-globalstate
+T-2/etc/cron.d/monitor_collect
+T-2/etc/cron.d/trafficserver-logrotate

software/caddy-frontend/test/test_data/test.TestSlaveBackendActiveCheck.test_file_list_log-CADDY.txt

+T-0/var/log/monitor-httpd-access.log
+T-0/var/log/monitor-httpd-error.log
+T-0/var/log/slapgrid-T-0-error.log
+T-1/var/log/expose-csr_id.log
+T-1/var/log/kedifa.log
+T-1/var/log/monitor-httpd-access.log
+T-1/var/log/monitor-httpd-error.log
+T-2/var/log/backend-haproxy.log
+T-2/var/log/frontend-access.log
+T-2/var/log/frontend-error.log
+T-2/var/log/httpd-csr_id/expose-csr_id.log
+T-2/var/log/httpd/_backend-active-check-connect_access_log
+T-2/var/log/httpd/_backend-active-check-connect_backend_log
+T-2/var/log/httpd/_backend-active-check-connect_error_log
+T-2/var/log/httpd/_backend-active-check-custom_access_log
+T-2/var/log/httpd/_backend-active-check-custom_backend_log
+T-2/var/log/httpd/_backend-active-check-custom_error_log
+T-2/var/log/httpd/_backend-active-check-default_access_log
+T-2/var/log/httpd/_backend-active-check-default_backend_log
+T-2/var/log/httpd/_backend-active-check-default_error_log
+T-2/var/log/httpd/_backend-active-check-disabled_access_log
+T-2/var/log/httpd/_backend-active-check-disabled_backend_log
+T-2/var/log/httpd/_backend-active-check-disabled_error_log
+T-2/var/log/monitor-httpd-access.log
+T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
+T-2/var/log/trafficserver/manager.log

software/caddy-frontend/test/test_data/test.TestSlaveBackendActiveCheck.test_file_list_plugin-CADDY.txt

+T-0/etc/plugin/__init__.py
+T-0/etc/plugin/aibcc-user-caucase-updater.py
+T-0/etc/plugin/aikc-user-caucase-updater.py
+T-0/etc/plugin/buildout-T-0-status.py
+T-0/etc/plugin/caucased-backend-client.py
+T-0/etc/plugin/check-free-disk-space.py
+T-0/etc/plugin/monitor-bootstrap-status.py
+T-0/etc/plugin/monitor-http-frontend.py
+T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
+T-0/etc/plugin/rejected-slave.py
+T-1/etc/plugin/__init__.py
+T-1/etc/plugin/buildout-T-1-status.py
+T-1/etc/plugin/caucased.py
+T-1/etc/plugin/check-free-disk-space.py
+T-1/etc/plugin/expose-csr_id-ip-port-listening.py
+T-1/etc/plugin/kedifa-http-reply.py
+T-1/etc/plugin/monitor-bootstrap-status.py
+T-1/etc/plugin/monitor-http-frontend.py
+T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-1/etc/plugin/promise-logrotate-setup.py
+T-2/etc/plugin/__init__.py
+T-2/etc/plugin/backend-client-caucase-updater.py
+T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend_haproxy_http.py
+T-2/etc/plugin/backend_haproxy_https.py
+T-2/etc/plugin/buildout-T-2-status.py
+T-2/etc/plugin/caddy_frontend_ipv4_http.py
+T-2/etc/plugin/caddy_frontend_ipv4_https.py
+T-2/etc/plugin/caddy_frontend_ipv6_http.py
+T-2/etc/plugin/caddy_frontend_ipv6_https.py
+T-2/etc/plugin/caucase-updater.py
+T-2/etc/plugin/check-free-disk-space.py
+T-2/etc/plugin/expose-csr_id-ip-port-listening.py
+T-2/etc/plugin/frontend-caddy-configuration-promise.py
+T-2/etc/plugin/monitor-bootstrap-status.py
+T-2/etc/plugin/monitor-http-frontend.py
+T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/promise-logrotate-setup.py
+T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
+T-2/etc/plugin/trafficserver-cache-availability.py
+T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestSlaveBackendActiveCheck.test_file_list_run-CADDY.txt

+T-0/var/run/monitor-httpd.pid
+T-1/var/run/kedifa.pid
+T-1/var/run/monitor-httpd.pid
+T-2/var/run/backend-haproxy-rsyslogd.pid
+T-2/var/run/backend-haproxy.pid
+T-2/var/run/backend_haproxy_configuration_last_state
+T-2/var/run/backend_haproxy_graceful_configuration_state_signature
+T-2/var/run/bhlog.sck
+T-2/var/run/graceful_configuration_state_signature
+T-2/var/run/httpd.pid
+T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestSlaveBackendActiveCheck.test_supervisor_state-CADDY.txt

+T-0:aibcc-user-caucase-updater-on-watch RUNNING
+T-0:aikc-user-caucase-updater-on-watch RUNNING
+T-0:bootstrap-monitor EXITED
+T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
+T-0:certificate_authority-{hash-generic}-on-watch RUNNING
+T-0:crond-{hash-generic}-on-watch RUNNING
+T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-0:monitor-httpd-graceful EXITED
+T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
+T-1:bootstrap-monitor EXITED
+T-1:caucase-updater-on-watch RUNNING
+T-1:caucased-{hash-generic}-on-watch RUNNING
+T-1:certificate_authority-{hash-generic}-on-watch RUNNING
+T-1:crond-{hash-generic}-on-watch RUNNING
+T-1:expose-csr_id-{hash-generic}-on-watch RUNNING
+T-1:kedifa-{hash-generic}-on-watch RUNNING
+T-1:kedifa-reloader EXITED
+T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-1:monitor-httpd-graceful EXITED
+T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
+T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
+T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
+T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
+T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
+T-2:backend-haproxy-safe-graceful EXITED
+T-2:bootstrap-monitor EXITED
+T-2:certificate_authority-{hash-generic}-on-watch RUNNING
+T-2:crond-{hash-generic}-on-watch RUNNING
+T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
+T-2:frontend-caddy-safe-graceful EXITED
+T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
+T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
+T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
+T-2:trafficserver-{hash-generic}-on-watch RUNNING
+T-2:trafficserver-reload EXITED

software/erp5/test/test/test_balancer.py

@@ -5,15 +5,13 @@ import logging
 import os
 import re
 import shutil
-import socket
 import subprocess
 import tempfile
 import time
 import urlparse
 from BaseHTTPServer import BaseHTTPRequestHandler
-from typing import Any, Dict, Optional
+from typing import Dict
 
-import idna
 import mock
 import OpenSSL.SSL
 import pexpect
@@ -106,20 +104,6 @@ class CaucaseService(ManagedResource):
     self._caucased_process.wait()
     shutil.rmtree(self.directory)
 
-  @property
-  def ca_crt_path(self):
-    # type: () -> str
-    """Path of the CA certificate from this caucase.
-    """
-    ca_crt_path = os.path.join(self.directory, 'ca.crt.pem')
-    if not os.path.exists(ca_crt_path):
-      with open(ca_crt_path, 'w') as f:
-        f.write(
-            requests.get(urlparse.urljoin(
-                self.url,
-                '/cas/crt/ca.crt.pem',
-            )).text)
-    return ca_crt_path
 
 class BalancerTestCase(ERP5InstanceTestCase):
 
@@ -387,85 +371,6 @@ class TestHTTP(BalancerTestCase):
     ])
 
 
-class TestTLS(BalancerTestCase):
-  """Check TLS
-  """
-  __partition_reference__ = 's'
-
-  def _getServerCertificate(self, hostname, port):
-    # type: (Optional[str], Optional[int]) -> Any
-    hostname_idna = idna.encode(hostname)
-    sock = socket.socket()
-
-    sock.connect((hostname, port))
-    ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
-    ctx.check_hostname = False
-    ctx.verify_mode = OpenSSL.SSL.VERIFY_NONE
-
-    sock_ssl = OpenSSL.SSL.Connection(ctx, sock)
-    sock_ssl.set_connect_state()
-    sock_ssl.set_tlsext_host_name(hostname_idna)
-    sock_ssl.do_handshake()
-    cert = sock_ssl.get_peer_certificate()
-    crypto_cert = cert.to_cryptography()
-    sock_ssl.close()
-    sock.close()
-    return crypto_cert
-
-  def test_certificate_validates_with_caucase_ca(self):
-    # type: () -> None
-    caucase = self.getManagedResource("caucase", CaucaseService)
-    requests.get(self.default_balancer_url, verify=caucase.ca_crt_path)
-
-  def test_certificate_renewal(self):
-    # type: () -> None
-    caucase = self.getManagedResource("caucase", CaucaseService)
-    balancer_parsed_url = urlparse.urlparse(self.default_balancer_url)
-    certificate_before_renewal = self._getServerCertificate(
-        balancer_parsed_url.hostname,
-        balancer_parsed_url.port)
-
-    # run caucase updater 90 days in the future, so that certificate is
-    # renewed.
-    caucase_updater = os.path.join(
-        self.computer_partition_root_path,
-        'etc',
-        'service',
-        'caucase-updater',
-    )
-    process = pexpect.spawnu(
-       "faketime +90days %s" % caucase_updater,
-        env=dict(os.environ, PYTHONPATH=''),
-    )
-    logger = self.logger
-    class DebugLogFile:
-      def write(self, msg):
-        logger.info("output from caucase_updater: %s", msg)
-      def flush(self):
-        pass
-    process.logfile = DebugLogFile()
-    process.expect(u"Renewing .*\nNext wake-up.*")
-    process.terminate()
-    process.wait()
-
-    # wait for server to use new certificate
-    for _ in range(30):
-      certificate_after_renewal = self._getServerCertificate(
-          balancer_parsed_url.hostname,
-          balancer_parsed_url.port)
-      if certificate_after_renewal.not_valid_before > certificate_before_renewal.not_valid_before:
-        break
-      time.sleep(.5)
-
-    self.assertGreater(
-        certificate_after_renewal.not_valid_before,
-        certificate_before_renewal.not_valid_before,
-    )
-
-    # requests are served properly after cert renewal
-    requests.get(self.default_balancer_url, verify=caucase.ca_crt_path).raise_for_status()
-
-
 class ContentTypeHTTPServer(ManagedHTTPServer):
   """An HTTP Server which reply with content type from path.
 

software/erp5/test/test/test_erp5.py

@@ -31,7 +31,6 @@ import glob
 import urlparse
 import socket
 import time
-import tempfile
 
 import psutil
 import requests
@@ -44,7 +43,7 @@ setUpModule # pyflakes
 class TestPublishedURLIsReachableMixin(object):
   """Mixin that checks that default page of ERP5 is reachable.
   """
-  def _checkERP5IsReachable(self, url, verify):
+  def _checkERP5IsReachable(self, url):
     # What happens is that instanciation just create the services, but does not
     # wait for ERP5 to be initialized. When this test run ERP5 instance is
     # instanciated, but zope is still busy creating the site and haproxy replies
@@ -52,7 +51,7 @@ class TestPublishedURLIsReachableMixin(object):
     # erp5 site is not created, with 500 when mysql is not yet reachable, so we
     # retry in a loop until we get a succesful response.
     for i in range(1, 60):
-      r = requests.get(url, verify=verify)
+      r = requests.get(url, verify=False)  # XXX can we get CA from caucase already ?
       if r.status_code != requests.codes.ok:
         delay = i * 2
         self.logger.warn("ERP5 was not available, sleeping for %ds and retrying", delay)
@@ -63,36 +62,19 @@ class TestPublishedURLIsReachableMixin(object):
 
     self.assertIn("ERP5", r.text)
 
-  def _getCaucaseServiceCACertificate(self):
-    ca_cert = tempfile.NamedTemporaryFile(
-        prefix="ca.crt.pem",
-        mode="w",
-        delete=False,
-    )
-    ca_cert.write(
-        requests.get(
-            urlparse.urljoin(
-                self.getRootPartitionConnectionParameterDict()['caucase-http-url'],
-                '/cas/crt/ca.crt.pem',
-            )).text)
-    self.addCleanup(os.unlink, ca_cert.name)
-    return ca_cert.name
-
   def test_published_family_default_v6_is_reachable(self):
     """Tests the IPv6 URL published by the root partition is reachable.
     """
     param_dict = self.getRootPartitionConnectionParameterDict()
     self._checkERP5IsReachable(
-      urlparse.urljoin(param_dict['family-default-v6'], param_dict['site-id']),
-      self._getCaucaseServiceCACertificate())
+      urlparse.urljoin(param_dict['family-default-v6'], param_dict['site-id']))
 
   def test_published_family_default_v4_is_reachable(self):
     """Tests the IPv4 URL published by the root partition is reachable.
     """
     param_dict = self.getRootPartitionConnectionParameterDict()
     self._checkERP5IsReachable(
-      urlparse.urljoin(param_dict['family-default'], param_dict['site-id']),
-      self._getCaucaseServiceCACertificate())
+      urlparse.urljoin(param_dict['family-default'], param_dict['site-id']))
 
 
 class TestDefaultParameters(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):

software/theia/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = 397fcb3279029af3055b23525d147445
+md5sum = 2ceb9389281c00261abd864fc8ed566f
 
 [yarn.lock]
 filename = yarn.lock

software/theia/instance.cfg.in

@@ -98,8 +98,12 @@ stop-on-error = true
 [frontend-instance-logo]
 recipe = plone.recipe.command
 filename = logo.png
+full-path = $${directory:frontend-static}/$${:filename}
 command =
-  ln -s ${logo.png:output} $${directory:frontend-static}/$${:filename}
+  if [ ! -e $${:full-path} ]
+  then
+    ln -s ${logo.png:output} $${:full-path}
+  fi
 stop-on-error = true
 
 [frontend-instance-slapos.css]

stack/erp5/buildout.hash.cfg

@@ -90,7 +90,7 @@ md5sum = 2f3ddd328ac1c375e483ecb2ef5ffb57
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = ecf119142e6b5cd85a2ba397552d2142
+md5sum = 4ba93d28d93bd066d5d19f4f74fc13d7
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -18,56 +18,25 @@ per partition. No more (undefined result), no less (IndexError).
 recipe = slapos.recipe.template:jinja2
 mode = 644
 
-[balancer-csr-request-config]
-< = jinja2-template-base
-template = inline:
-    [req]
-    prompt = no
-    req_extensions = req_ext
-    distinguished_name = dn
-    [ dn ]
-    CN = example.com
-    [ req_ext ]
-    subjectAltName = @alt_names
-    [ alt_names ]
-    IP.1 =  {{ ipv4 }}
-    {% if ipv6_set -%}
-    IP.2 =  {{ ipv6 }}
-    {% endif %}
-rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:_buildout_section_name_}.txt
-
-[balancer-csr-request]
-recipe = plone.recipe.command
-command = {{ parameter_dict["openssl"] }}/bin/openssl req \
-  -newkey rsa:2048 \
-  -batch \
-  -new \
-  -nodes \
-  -keyout '${apache-conf-ssl:key}' \
-  -config '${balancer-csr-request-config:rendered}' \
-  -out '${:csr}'
-stop-on-error = true
-csr = ${directory:etc}/${:_buildout_section_name_}.csr.pem
-
-
 {{ caucase.updater(
      prefix='caucase-updater',
      buildout_bin_directory=parameter_dict['bin-directory'],
      updater_path='${directory:services-on-watch}/caucase-updater',
      url=ssl_parameter_dict['caucase-url'],
      data_dir='${directory:srv}/caucase-updater',
-     crt_path='${apache-conf-ssl:cert}',
+     crt_path='${apache-conf-ssl:caucase-cert}',
      ca_path='${directory:srv}/caucase-updater/ca.crt',
      crl_path='${directory:srv}/caucase-updater/crl.pem',
-     key_path='${apache-conf-ssl:key}',
+     key_path='${apache-conf-ssl:caucase-key}',
      on_renew='${apache-graceful:output}',
      max_sleep=ssl_parameter_dict.get('max-crl-update-delay', 1.0),
      template_csr_pem=ssl_parameter_dict.get('csr'),
-     template_csr=None if ssl_parameter_dict.get('csr') else '${balancer-csr-request:csr}',
      openssl=parameter_dict['openssl'] ~ '/bin/openssl',
 )}}
+{# XXX we don't use caucase yet.
 {% do section('caucase-updater') -%}
 {% do section('caucase-updater-promise') -%}
+#}
 
 {% set frontend_caucase_url_hash_list = [] -%}
 {% for frontend_caucase_url in frontend_caucase_url_list -%}
@@ -209,6 +178,10 @@ hash-files = ${haproxy-cfg:rendered}
 [apache-conf-ssl]
 cert = ${directory:apache-conf}/apache.crt
 key = ${directory:apache-conf}/apache.pem
+# XXX caucase is/was buggy and this certificate does not match key for instances
+# that were updated, so don't use it yet.
+caucase-cert = ${directory:apache-conf}/apache-caucase.crt
+caucase-key = ${directory:apache-conf}/apache-caucase.pem
 {% if frontend_caucase_url_list -%}
 depends = ${caucase-updater-housekeeper-run:recipe}
 ca-cert-dir = ${directory:apache-ca-cert-dir}
@@ -231,6 +204,19 @@ context = key content {{content_section_name}}:content
 mode = {{ mode }}
 {%- endmacro %}
 
+[apache-ssl]
+{% if ssl_parameter_dict.get('key') -%}
+key = ${apache-ssl-key:rendered}
+cert = ${apache-ssl-cert:rendered}
+{{ simplefile('apache-ssl-key', '${apache-conf-ssl:key}', ssl_parameter_dict['key']) }}
+{{ simplefile('apache-ssl-cert', '${apache-conf-ssl:cert}', ssl_parameter_dict['cert']) }}
+{% else %}
+recipe = plone.recipe.command
+command = "{{ parameter_dict['openssl'] }}/bin/openssl" req -newkey rsa -batch -new -x509 -days 3650 -nodes -keyout "${:key}" -out "${:cert}"
+key = ${apache-conf-ssl:key}
+cert = ${apache-conf-ssl:cert}
+{%- endif %}
+
 [apache-conf-parameter-dict]
 backend-list = {{ dumps(apache_dict.values()) }}
 zope-virtualhost-monster-backend-dict = {{ dumps(zope_virtualhost_monster_backend_dict) }}
@@ -242,8 +228,8 @@ access-log = ${directory:log}/apache-access.log
 # Apache 2.4's default value (60 seconds) can be a bit too short
 timeout = 300
 # Basic SSL server configuration
-cert = ${apache-conf-ssl:cert}
-key = ${apache-conf-ssl:key}
+cert = ${apache-ssl:cert}
+key = ${apache-ssl:key}
 cipher =
 ssl-session-cache = ${directory:log}/apache-ssl-session-cache
 {% if frontend_caucase_url_list -%}