stack/erp5: use caucase managed certificate for balancer

TODO:
 - split code move in preparatory commit

This reverts commit 620c9332 (stack/erp5: stop using caucase managed
certificate for balancer, 2020-11-10) with an updated design. We add a
caucase service for balancer in the balancer partition. The caucase
service from the root partition (that was not used) is removed.

The underlying idea is that the default configuration should use multiple
caucases with limited scope, here we have one caucase to manage the
certificate used by haproxy server in the balancer partition, so we put
one caucase to manage this certificate and the caucase is configured to
auto-accept one certificate only. The plan is that when we will add a
certificate for mariadb server, we'll add another caucase inside this
mariadb server.

For more advanced usage and also to support the cases where a new
certificate needs to be re-emitted for some reason (data loss,
partition IP change, etc), users can request with an existing caucase
URL. In that case, they will have to accept the certificate requests.

Notable changes:

balancer/ssl/caucase-url is no longer documented in parameters, this is
an internal parameter, users can pass one global caucase service to
manage all partition

CAUCASE environment variable is no longer set when running zope. There
was no identified use case and with this new approach of multiple
caucases, the term "caucase" alone became ambiguous.

software/erp5/README.rst

@@ -90,7 +90,6 @@ This software release assigns the following port ranges by default:
   balancer              2150-2199
   zope                  2200-*
   jupyter               8888
-  caucase               8890,8891
   ====================  ==========
 
 Non-zope partitions are unique in an ERP5 cluster, so you shouldn't have to

software/erp5/instance-erp5-input-schema.json

@@ -513,7 +513,7 @@
           "properties": {
             "url": {
               "title": "Caucase URL",
-              "description": "URL of existing caucase instance to use. If empty, a new caucase instance will be deployed. If not empty, other properties in this section will be ignored.",
+              "description": "URL of existing caucase instance to use. If empty, caucase instances will be deployed inside partitions.",
               "default": "",
               "type": "string",
               "format": "uri"
@@ -722,13 +722,13 @@
             },
             "caucase-url": {
               "title": "Caucase URL",
-              "description": "URL of caucase service to use. If not set, global setting will be used.",
+              "description": "URL of caucase service to use. If not set, a caucase service will be instantiated.",
               "type": "string",
               "format": "uri"
             },
             "csr": {
               "title": "csr",
-              "description": "PEM-encoded certificate signature request to request server certificate with. If not provided, HTTPS will be disabled.",
+              "description": "PEM-encoded certificate signature request to request server certificate with.",
               "type": "string"
             },
             "max-crl-update-delay": {

software/erp5/instance-erp5-output-schema.json

@@ -69,7 +69,7 @@
       "type": "string"
     },
     "caucase-http-url": {
-      "description": "Caucase url on HTTP. For HTTPS URL, uses https scheme, if port is explicitely specified in http URL, take that port and add 1 and use it as https port. If it is not specified.",
+      "description": "Caucase url on HTTP. For HTTPS URL, uses https scheme, if port is explicitely specified in http URL, take that port and add 1 and use it as https port.",
       "pattern": "^http://",
       "type": "string"
     }

software/erp5/test/test/__init__.py

@@ -24,13 +24,26 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-
+import hashlib
 import itertools
 import json
 import os
+import shutil
+import subprocess
 import sys
+import tempfile
+import time
+import urllib
+
+import requests
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.x509.oid import NameOID
 
-from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+from slapos.testing.testcase import ManagedResource, makeModuleSetUpAndTestCaseClass
+from slapos.testing.utils import findFreeTCPPort
 
 ERP5PY3 = os.environ['SLAPOS_SR_TEST_NAME'] == 'erp5-py3'
 
@@ -205,3 +218,175 @@ class ERP5InstanceTestCase(SlapOSInstanceTestCase, metaclass=ERP5InstanceTestMet
   def getComputerPartitionPath(cls, partition_reference):
     partition_id = cls.getComputerPartition(partition_reference).getId()
     return os.path.join(cls.slap._instance_root, partition_id)
+
+
+class CaucaseService(ManagedResource):
+  """A caucase service.
+  """
+  url: str = None
+  directory: str = None
+  _caucased_process: subprocess.Popen = None
+
+  def open(self) -> None:
+    # start a caucased and server certificate.
+    software_release_root_path = os.path.join(
+        self._cls.slap._software_root,
+        hashlib.md5(self._cls.getSoftwareURL().encode()).hexdigest(),
+    )
+    caucased_path = os.path.join(software_release_root_path, 'bin', 'caucased')
+
+    self.directory = tempfile.mkdtemp()
+    caucased_dir = os.path.join(self.directory, 'caucased')
+    os.mkdir(caucased_dir)
+    os.mkdir(os.path.join(caucased_dir, 'user'))
+    os.mkdir(os.path.join(caucased_dir, 'service'))
+
+    backend_caucased_netloc = f'{self._cls._ipv4_address}:{findFreeTCPPort(self._cls._ipv4_address)}'
+    self.url = 'http://' + backend_caucased_netloc
+    self._caucased_process = subprocess.Popen(
+        [
+            caucased_path,
+            '--db', os.path.join(caucased_dir, 'caucase.sqlite'),
+            '--server-key', os.path.join(caucased_dir, 'server.key.pem'),
+            '--netloc', backend_caucased_netloc,
+            '--service-auto-approve-count', '1',
+        ],
+        # capture subprocess output not to pollute test's own stdout
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    )
+    for _ in range(30):
+      try:
+        if requests.get(self.url).status_code == 200:
+          break
+      except Exception:
+        pass
+      time.sleep(1)
+    else:
+      raise RuntimeError('caucased failed to start.')
+
+  def close(self) -> None:
+    self._caucased_process.terminate()
+    self._caucased_process.wait()
+    self._caucased_process.stdout.close()
+    shutil.rmtree(self.directory)
+
+  @property
+  def ca_crt_path(self) -> str:
+    """Path of the CA certificate from this caucase.
+    """
+    ca_crt_path = os.path.join(self.directory, 'ca.crt.pem')
+    if not os.path.exists(ca_crt_path):
+      with open(ca_crt_path, 'w') as f:
+        f.write(
+            requests.get(urllib.parse.urljoin(
+                self.url,
+                '/cas/crt/ca.crt.pem',
+            )).text)
+    return ca_crt_path
+
+
+
+class CaucaseCertificate(ManagedResource):
+  """A certificate signed by a caucase service.
+  """
+
+  ca_crt_file: str = None
+  crl_file: str = None
+  csr_file: str = None
+  cert_file: str = None
+  key_file: str = None
+
+  def open(self) -> None:
+    self.tmpdir = tempfile.mkdtemp()
+    self.ca_crt_file = os.path.join(self.tmpdir, 'ca-crt.pem')
+    self.crl_file = os.path.join(self.tmpdir, 'ca-crl.pem')
+    self.csr_file = os.path.join(self.tmpdir, 'csr.pem')
+    self.cert_file = os.path.join(self.tmpdir, 'crt.pem')
+    self.key_file = os.path.join(self.tmpdir, 'key.pem')
+
+  def close(self) -> None:
+    shutil.rmtree(self.tmpdir)
+
+  @property
+  def _caucase_path(self) -> str:
+    """path of caucase executable.
+    """
+    software_release_root_path = os.path.join(
+        self._cls.slap._software_root,
+        hashlib.md5(self._cls.getSoftwareURL().encode()).hexdigest(),
+    )
+    return os.path.join(software_release_root_path, 'bin', 'caucase')
+
+  def request(self, common_name: str, caucase: CaucaseService) -> None:
+    """Generate certificate and request signature to the caucase service.
+
+    This overwrite any previously requested certificate for this instance.
+    """
+    cas_args = [
+        self._caucase_path,
+        '--ca-url', caucase.url,
+        '--ca-crt', self.ca_crt_file,
+        '--crl', self.crl_file,
+    ]
+
+    key = rsa.generate_private_key(
+        public_exponent=65537,
+        key_size=2048,
+        backend=default_backend()
+    )
+    with open(self.key_file, 'wb') as f:
+      f.write(
+          key.private_bytes(
+              encoding=serialization.Encoding.PEM,
+              format=serialization.PrivateFormat.TraditionalOpenSSL,
+              encryption_algorithm=serialization.NoEncryption(),
+          ))
+
+    csr = x509.CertificateSigningRequestBuilder().subject_name(
+        x509.Name([
+            x509.NameAttribute(
+                NameOID.COMMON_NAME,
+                common_name,
+            ),
+        ])).sign(
+            key,
+            hashes.SHA256(),
+            default_backend(),
+        )
+    with open(self.csr_file, 'wb') as f:
+      f.write(csr.public_bytes(serialization.Encoding.PEM))
+
+    csr_id = subprocess.check_output(
+      cas_args + [
+          '--send-csr', self.csr_file,
+      ],
+    ).split()[0].decode()
+    assert csr_id
+
+    for _ in range(30):
+      if not subprocess.call(
+        cas_args + [
+            '--get-crt', csr_id, self.cert_file,
+        ],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+      ) == 0:
+        break
+      else:
+        time.sleep(1)
+    else:
+      raise RuntimeError('getting service certificate failed.')
+    with open(self.cert_file) as cert_file:
+      assert 'BEGIN CERTIFICATE' in cert_file.read()
+
+  def revoke(self, caucase: CaucaseService) -> None:
+    """Revoke the client certificate on this caucase instance.
+    """
+    subprocess.check_call([
+        self._caucase_path,
+        '--ca-url', caucase.url,
+        '--ca-crt', self.ca_crt_file,
+        '--crl', self.crl_file,
+        '--revoke-crt', self.cert_file, self.key_file,
+    ])

software/erp5/test/test/test_erp5.py

@@ -50,7 +50,7 @@ import urllib3
 from slapos.testing.utils import CrontabMixin
 import zc.buildout.configparser
 
-from . import ERP5InstanceTestCase, default, matrix, neo, setUpModule, ERP5PY3
+from . import CaucaseService, ERP5InstanceTestCase, default, matrix, neo, setUpModule, ERP5PY3
 
 setUpModule # pyflakes
 
@@ -98,6 +98,22 @@ class TestPublishedURLIsReachableMixin:
       self.assertEqual(r.status_code, requests.codes.ok)
       self.assertIn("ERP5", r.text)
 
+  def _getCaucaseServiceCACertificate(self):
+    ca_cert = tempfile.NamedTemporaryFile(
+      prefix="ca.crt.pem",
+      mode="w",
+      delete=False,
+    )
+    ca_cert.write(
+      requests.get(
+        urllib.parse.urljoin(
+          self.getRootPartitionConnectionParameterDict()['caucase-http-url'],
+          '/cas/crt/ca.crt.pem',
+        )).text)
+    ca_cert.flush()
+    self.addCleanup(os.unlink, ca_cert.name)
+    return ca_cert.name
+
   def test_published_family_default_v6_is_reachable(self):
     """Tests the IPv6 URL published by the root partition is reachable.
     """
@@ -105,7 +121,7 @@ class TestPublishedURLIsReachableMixin:
     self._checkERP5IsReachable(
       param_dict['family-default-v6'],
       param_dict['site-id'],
-      verify=False,
+      self._getCaucaseServiceCACertificate(),
     )
 
   def test_published_family_default_v4_is_reachable(self):
@@ -115,7 +131,7 @@ class TestPublishedURLIsReachableMixin:
     self._checkERP5IsReachable(
       param_dict['family-default'],
       param_dict['site-id'],
-      verify=False,
+      self._getCaucaseServiceCACertificate(),
     )
 
   def test_published_frontend_default_is_reachable(self):
@@ -125,7 +141,7 @@ class TestPublishedURLIsReachableMixin:
     self._checkERP5IsReachable(
       param_dict['url-frontend-default'],
       param_dict['site-id'],
-      verify=False,
+      self._getCaucaseServiceCACertificate(),
     )
 
 
@@ -158,6 +174,66 @@ class TestDefaultParameters(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
       self.getRootPartitionConnectionParameterDict()['url-frontend-default'])
 
 
+class TestExternalCaucase(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+  """Test providing the URL of an external caucase in parameters.
+  """
+  __partition_reference__ = 'ec'
+
+  @classmethod
+  def getInstanceParameterDict(cls) -> dict:
+    caucase_url = cls.getManagedResource("caucase", CaucaseService).url
+    return {'_': json.dumps({'caucase': {'url': caucase_url}})}
+
+  def test_published_caucase_http_url_parameter(self) -> None:
+    self.assertEqual(
+      self.getRootPartitionConnectionParameterDict()['caucase-http-url'],
+      self.getManagedResource("caucase", CaucaseService).url,
+    )
+
+
+class TestReinstantiateWithExternalCaucase(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+  """Test providing the URL of an external caucase in parameters after
+  the initial instantiation.
+  """
+  __partition_reference__ = 'sc'
+
+  def test_switch_to_external_caucase(self) -> None:
+    external_caucase_url = self.getManagedResource("caucase", CaucaseService).url
+    partition_parameter_kw = {
+      '_': json.dumps({'caucase': {
+        'url': external_caucase_url
+      }})
+    }
+    def rerequest():
+      return self.slap.request(
+        software_release=self.getSoftwareURL(),
+        software_type=self.getInstanceSoftwareType(),
+        partition_reference=self.default_partition_reference,
+        partition_parameter_kw=partition_parameter_kw,
+        state='started')
+
+    rerequest()
+    breakpoint()
+    self.slap.waitForInstance(max_retry=10)
+
+    self.assertEqual(
+      json.loads(rerequest().getConnectionParameterDict()['_'])['caucase-http-url'],
+      external_caucase_url)
+
+    with tempfile.NamedTemporaryFile(mode="w") as ca_cert:
+      ca_cert.write(
+        requests.get(
+          urllib.parse.urljoin(
+            external_caucase_url,
+            '/cas/crt/ca.crt.pem',
+          )).text)
+      ca_cert.flush()
+
+      requests.get(
+        self.getRootPartitionConnectionParameterDict()['url-frontend-default'],
+        verify=ca_cert.name).raise_for_status()
+
+
 class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   """Test ERP5 Jupyter notebook
   """

stack/erp5/buildout.hash.cfg

@@ -74,7 +74,7 @@ md5sum = 55232eae0bcdb68a7cb2598d2ba9d60c
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = a85867f7d05c13e576b5453c49bd8997
+md5sum = 87368e3c6a881066fcf64a355c619884
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -86,11 +86,11 @@ md5sum = 0ac4b74436f554cd677f19275d18d880
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = 4ab71c2cb50310b948828a07c521e0b7
+md5sum = f9a68cd6f12bc2d576d50662d83ee480
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = b0751d3d12cfcc8934cb1027190f5e5e
+md5sum = acf3a0b398c8d830f26222254009a4aa
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -2,6 +2,7 @@
 {% set part_list = [] -%}
 {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
 {% set ssl_parameter_dict = slapparameter_dict['ssl'] -%}
+{% set caucase_url = ssl_parameter_dict.get('caucase-url') %}
 {% set frontend_caucase_url_list = ssl_parameter_dict.get('frontend-caucase-url-list', []) -%}
 {#
 XXX: This template only supports exactly one IPv4 and (if ipv6 is used) one IPv6
@@ -15,25 +16,114 @@ per partition. No more (undefined result), no less (IndexError).
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
 
+[simplefile]
+< = jinja2-template-base
+inline = {{ '{{ content }}' }}
+
+{% macro simplefile(section_name, file_path, content, mode='') -%}
+{%   set content_section_name = section_name ~ '-content' -%}
+[{{  content_section_name }}]
+content = {{ dumps(content) }}
+
+[{{  section(section_name) }}]
+< = simplefile
+output = {{ file_path }}
+context = key content {{content_section_name}}:content
+mode = {{ mode }}
+{%- endmacro %}
+
+{% if not caucase_url -%}
+{%   if ipv6_set -%}
+{%     set caucase_host = '[' ~ (ipv6_set | list)[0] ~ ']' %}
+{%-  else -%}
+{%     set caucase_host = (ipv4_set | list)[0] %}
+{%-  endif %}
+{%   set caucase_http_port = 2198 -%}
+{%   set caucase_netloc = caucase_host ~ ':' ~ caucase_http_port -%}
+{%   set caucase_url = 'http://' ~ caucase_netloc %}
+{{   caucase.caucased(
+       prefix='caucased-haproxy-certificate',
+       buildout_bin_directory=bin_directory,
+       caucased_path='${directory:services-on-watch}/caucased',
+       backup_dir='${directory:backup-caucased}',
+       data_dir='${directory:srv}/caucased',
+       netloc=caucase_netloc,
+       tmp='${directory:tmp}',
+       service_auto_approve_count=ssl_parameter_dict.get('service-auto-approve-amount', 1),
+       user_auto_approve_count=ssl_parameter_dict.get('user-auto-approve-amount', 0),
+       key_len=ssl_parameter_dict.get('key-length', 2048),
+)}}
+{%   set caucase_haproxy_certificate_hash = '' %}
+{%   do section('caucased-haproxy-certificate') -%}
+{%   do section('caucased-haproxy-certificate-promise') -%}
+{% else %}
+{%   set caucase_haproxy_certificate_hash = '-' + hashlib.md5(six.ensure_binary(caucase_url)).hexdigest()[:6] -%}
+{% endif -%}
+
+[haproxy-certificate]
+cert-and-key-file = ${directory:etc}/${:_buildout_section_name_}{{ caucase_haproxy_certificate_hash }}-cert-and-key.pem
+ca-file = ${directory:etc}/${:_buildout_section_name_}{{ caucase_haproxy_certificate_hash }}.ca.crt
+crl-file = ${directory:etc}/${:_buildout_section_name_}{{ caucase_haproxy_certificate_hash }}.crl
+
+[haproxy-certificate-csr-config]
+recipe = slapos.recipe.template
+inline =
+  [ req ]
+  prompt = no
+  req_extensions = req_ext
+  distinguished_name = dn
+  [ dn ]
+  CN = haproxy
+  [ req_ext ]
+  subjectAltName = @alt_names
+  [ alt_names ]
+  IP.1 = {{ ipv4 }}
+{% if ipv6_set %}
+  IP.2 = {{ ipv6 }}
+{% endif %}
+output = ${buildout:parts-directory}/${:_buildout_section_name_}/${:_buildout_section_name_}.txt
+
+[haproxy-certificate-csr]
+recipe = plone.recipe.command
+command =
+  if [ ! -f '${:csr}' ] ; then
+   {{ parameter_dict['openssl'] }}/bin/openssl req \
+      -newkey rsa \
+      -batch \
+      -new \
+      -nodes \
+      -keyout /dev/null \
+      -config '${haproxy-certificate-csr-config:output}' \
+      -out '${:csr}'
+  fi
+stop-on-error = true
+csr = ${directory:srv}/${:_buildout_section_name_}{{ caucase_haproxy_certificate_hash }}.csr.pem
+
+{% if ssl_parameter_dict.get('csr') %}
+{{ simplefile(
+    'haproxy-certificate-csr-from-parameters',
+    '${directory:etc}/haproxy-certificate-csr-from-parameters.pem',
+    ssl_parameter_dict['csr']) }}
+[haproxy-certificate-csr]
+csr = ${haproxy-certificate-csr-from-parameters:output}
+{% endif %}
+
 {{ caucase.updater(
-     prefix='caucase-updater',
+     prefix='caucase-updater-haproxy-certificate',
      buildout_bin_directory=parameter_dict['bin-directory'],
-     updater_path='${directory:services-on-watch}/caucase-updater',
-     url=ssl_parameter_dict['caucase-url'],
-     data_dir='${directory:srv}/caucase-updater',
-     crt_path='${apache-conf-ssl:caucase-cert}',
-     ca_path='${directory:srv}/caucase-updater/ca.crt',
-     crl_path='${directory:srv}/caucase-updater/crl.pem',
-     key_path='${apache-conf-ssl:caucase-key}',
+     updater_path='${directory:services-on-watch}/caucase-updater-haproxy-certificate' + caucase_haproxy_certificate_hash,
+     url=caucase_url,
+     data_dir='${directory:caucase-updater-haproxy-certificate}',
+     crt_path='${haproxy-certificate:cert-and-key-file}',
+     ca_path='${haproxy-certificate:ca-file}',
+     crl_path='${haproxy-certificate:crl-file}',
+     key_path='${haproxy-certificate:cert-and-key-file}',
      on_renew='${haproxy-reload:output}',
      max_sleep=ssl_parameter_dict.get('max-crl-update-delay', 1.0),
-     template_csr_pem=ssl_parameter_dict.get('csr'),
+     template_csr='${haproxy-certificate-csr:csr}',
      openssl=parameter_dict['openssl'] ~ '/bin/openssl',
 )}}
-{# XXX we don't use caucase yet.
-{% do section('caucase-updater') -%}
-{% do section('caucase-updater-promise') -%}
-#}
+{% do section('caucase-updater-haproxy-certificate') -%}
 
 {% set frontend_caucase_url_hash_list = [] -%}
 {% for frontend_caucase_url in frontend_caucase_url_list -%}
@@ -228,22 +318,6 @@ hash-files = ${haproxy-cfg:output}
 caucase-cert = ${directory:apache-conf}/apache-caucase.crt
 caucase-key = ${directory:apache-conf}/apache-caucase.pem
 
-[simplefile]
-< = jinja2-template-base
-inline = {{ '{{ content }}' }}
-
-{% macro simplefile(section_name, file_path, content, mode='') -%}
-{%   set content_section_name = section_name ~ '-content' -%}
-[{{  content_section_name }}]
-content = {{ dumps(content) }}
-
-[{{  section(section_name) }}]
-< = simplefile
-output = {{ file_path }}
-context = key content {{content_section_name}}:content
-mode = {{ mode }}
-{%- endmacro %}
-
 [{{ section('haproxy-socat-stats')}}]
 recipe = slapos.cookbook:wrapper
 wrapper-path = ${directory:bin}/${:_buildout_section_name_}
@@ -277,7 +351,7 @@ config-command = test -S ${rsyslogd-cfg-parameter-dict:log-socket}
 
 
 [haproxy-conf-ssl]
-certificate = ${build-certificate-and-key:certificate-and-key}
+certificate = ${haproxy-certificate:cert-and-key-file}
 {% if frontend_caucase_url_list -%}
 ca-cert = ${directory:etc}/frontend-ca.pem
 ca-cert-dir = ${directory:ca-cert}
@@ -286,18 +360,15 @@ crl-dir = ${directory:crl}
 depends = ${caucase-updater-housekeeper-run:recipe}
 {%- endif %}
 
-[build-certificate-and-key]
+# BBB cert and key as ssl parameters. Use caucase instead.
 {% if ssl_parameter_dict.get('key') -%}
-certificate-and-key = ${tls-certificate-and-key-from-parameters:output}
 {{ simplefile(
-    'tls-certificate-and-key-from-parameters',
+    'haproxy-conf-ssl-certificate-and-key-from-parameters',
     '${directory:etc}/certificate-and-key-from-parameters.pem',
     ssl_parameter_dict['cert'] ~ "\n" ~ ssl_parameter_dict['key']) }}
-{% else %}
-recipe = plone.recipe.command
-command = "{{ parameter_dict['openssl'] }}/bin/openssl" req -newkey rsa -batch -new -x509 -days 3650 -nodes -keyout "${:certificate-and-key}" -out "${:certificate-and-key}"
-certificate-and-key = ${directory:etc}/certificate-and-key-generated.pem
-{%- endif %}
+[haproxy-conf-ssl]
+certificate = ${haproxy-conf-ssl-certificate-and-key-from-parameters:output}
+{% endif %}
 
 [{{ section('haproxy-promise') }}]
 <= monitor-promise-base
@@ -316,6 +387,7 @@ recipe = slapos.cookbook:publish.serialised
 {% for family_name, test_runner_url_list in test_runner_url_dict.items() -%}
 {{    family_name ~ '-test-runner-url-list' }} = {{ dumps(test_runner_url_list) }}
 {% endfor -%}
+caucase-http-url = {{ caucase_url }}
 monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
 
 [{{ section('logrotate-rsyslogd') }}]
@@ -335,6 +407,9 @@ var = ${buildout:directory}/var
 run = ${:var}/run
 log = ${:var}/log
 srv = ${buildout:directory}/srv
+backup-caucased = ${:srv}/backup/caucased/
+caucase-updater-haproxy-certificate = ${:srv}/caucase-updater-haproxy-certificate
+tmp = ${buildout:directory}/tmp
 apachedex = ${monitor-directory:private}/apachedex
 rsyslogd-spool = ${:run}/rsyslogd-spool
 {% if frontend_caucase_url_list -%}

stack/erp5/instance-erp5.cfg.in

@@ -79,35 +79,30 @@ bin = ${buildout:directory}/bin
 service-on-watch = ${buildout:directory}/etc/service
 srv = ${buildout:directory}/srv
 tmp = ${buildout:directory}/tmp
-backup-caucased = ${:srv}/backup/caucased
 
-{% if not caucase_url -%}
-{%   if use_ipv6 -%}
-{%     set caucase_host = '[' ~ (ipv6_set | list)[0] ~ ']' %}
-{%-  else -%}
-{%     set caucase_host = (ipv4_set | list)[0] %}
-{%-  endif %}
-{%   set caucase_port = caucase_dict.get('base-port', 8890) -%}
-{%   set caucase_netloc = caucase_host ~ ':' ~ caucase_port -%}
-{%   set caucase_url = 'http://' ~ caucase_netloc -%}
-{{   caucase.caucased(
-       prefix='caucased',
-       buildout_bin_directory=bin_directory,
-       caucased_path='${directory:service-on-watch}/caucased',
-       backup_dir='${directory:backup-caucased}',
-       data_dir='${directory:srv}/caucased',
-       netloc=caucase_netloc,
-       tmp='${directory:tmp}',
-       service_auto_approve_count=caucase_dict.get('service-auto-approve-amount', 1),
-       user_auto_approve_count=caucase_dict.get('user-auto-approve-amount', 0),
-       key_len=caucase_dict.get('key-length', 2048),
-)}}
-{% do root_common.section('caucased') -%}
-{% do root_common.section('caucased-promise') -%}
-{% endif -%}
-{% do publish_dict.__setitem__('caucase-http-url', caucase_url) -%}
 {% set balancer_dict = slapparameter_dict.setdefault('balancer', {}) -%}
-{% do balancer_dict.setdefault('ssl', {}).setdefault('caucase-url', caucase_url) -%}
+{% do balancer_dict.setdefault('ssl', {}) %}
+{% if caucase_url %}
+{%   do balancer_dict['ssl'].setdefault('caucase-url', caucase_url) -%}
+
+[balancer-updated-caucase-url]
+# Promise to wait for balancer partition to use the provided URL, for cases where the
+# caucase URL is changed
+recipe = slapos.cookbook:check_parameter
+value = ${publish:caucase-http-url}
+expected-not-value =
+expected-value = {{ caucase_url }}
+path = ${directory:bin}/${:_buildout_section_name_}
+
+[balancer-updated-caucase-url-promise]
+<= monitor-promise-base
+promise = check_command_execute
+name = ${:_buildout_section_name_}.py
+config-command = ${balancer-updated-caucase-url:path}
+{% do root_common.section("balancer-updated-caucase-url-promise") -%}
+
+{% endif %}
+
 {% do balancer_dict.setdefault('tcpv4-port', 2150) -%}
 {% do balancer_dict.__setitem__('haproxy-server-check-path', balancer_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id}) -%}
 {% set routing_path_template_field_dict = {"site-id": site_id} -%}
@@ -241,7 +236,6 @@ return =
 config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }}
 config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
 config-cloudooo-url-list = {{ dumps(slapparameter_dict.get('cloudooo-url-list', default_cloudooo_url_list)) }}
-config-caucase-url = {{ dumps(caucase_url) }}
 config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
 config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
 config-selenium-server-configuration-dict = {{ dumps(slapparameter_dict.get('selenium-server-configuration-dict', {})) }}
@@ -447,7 +441,7 @@ config-allow-redirects = 0
 
 {%- endif %}
 
-{% set balancer_ret_dict = {'monitor-base-url': False} -%}
+{% set balancer_ret_dict = {'monitor-base-url': False, 'caucase-http-url': False} -%}
 {% for family in zope_family_dict -%}
 {%   do balancer_ret_dict.__setitem__(family, False) -%}
 {%   do balancer_ret_dict.__setitem__(family + '-v6', False) -%}
@@ -496,7 +490,7 @@ config-allow-redirects = 0
   ret=balancer_ret_dict,
   key_config=balancer_key_config_dict,
 ) }}
-
+{% do publish_dict.__setitem__('caucase-http-url', '${request-balancer:connection-caucase-http-url}' ) -%}
 
 {% endif -%}{# if zope_partition_dict -#}
 

stack/erp5/instance-zope.cfg.in

@@ -81,7 +81,6 @@ environment +=
   MATPLOTLIBRC={{ parameter_dict['matplotlibrc'] }}
   PYTHONUNBUFFERED=1
   INSTANCE_HOME=${:instance-home}
-  CAUCASE={{ slapparameter_dict['caucase-url'] }}
   FONTCONFIG_FILE=${fontconfig-conf:output}
   JUPYTER_PATH=${directory:jupyter-dir}
   JUPYTER_CONFIG_DIR=${directory:jupyter-config-dir}