XXX rapid-cdn: Implement simple and working frontend support

Checkpoint: front-haproxy serves some backend, but it still speaks directly to the backend, and shall use back-haproxy for backend intelligence.

software/rapid-cdn/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+[template]
+_update_hash_filename_ = instance.cfg.jinja2
+md5sum = 32e0289fa8e4b8b7baf6b0454fe9b7e2
+
+[template-common]
+_update_hash_filename_ = instance-common.cfg.jinja2
+md5sum = c3a54ac864508ecb693c16cdec06671e
+
+[template-rapid-cdn-human]
+_update_hash_filename_ = instance-human.cfg.jinja2
+md5sum = 6a57e7727c8d258f5ae81bdbb86328cb
+
+[template-rapid-cdn-pop]
+_update_hash_filename_ = instance-pop.cfg.jinja2
+md5sum = 151e07e7ced18c631c11344bf4d7cf51
+
+[template-rapid-cdn-kedifa]
+_update_hash_filename_ = instance-kedifa.cfg.jinja2
+md5sum = 13c92f006a0c06789b691ca6b21a8195
+
+[template-rapid-cdn-log-aggregator]
+_update_hash_filename_ = instance-log-aggregator.cfg.jinja2
+md5sum = 6a57e7727c8d258f5ae81bdbb86328cb
+
+[template-rapid-cdn-frontend]
+_update_hash_filename_ = instance-frontend.cfg.jinja2
+md5sum = 281e99430d9ca523ae8a015550edffb3
+
+[template-front-haproxy]
+_update_hash_filename_ = template/front-haproxy.cfg.jinja2
+md5sum = 9a7529e2633b59a132f30395b6755ac2
+
+[template-front-rsyslogd]
+_update_hash_filename_ = template/front-rsyslogd.conf.jinja2
+md5sum = e975b5cb71915eb9743928a5fee5acaf

software/rapid-cdn/instance-common.cfg.jinja2

+[buildout]
+eggs-directory = {{ software_parameter_dict['eggs-directory'] }}
+develop-eggs-directory = {{ software_parameter_dict['develop-eggs-directory'] }}
+offline = true
+
+[kedifa-config]
+# shared kedifa configuration parameters for the whole cluster
+marker = __NotReadyYet__
+
+[csr]
+recipe = plone.recipe.command
+organization = ${slap-configuration:root-instance-title}
+command =
+  if [ ! -f ${:template-csr} ] && [ ! -f ${:key} ]  ; then
+    /bin/bash -c '{{ software_parameter_dict['openssl'] }} req -new -sha256 \
+      -newkey rsa:2048 -nodes -keyout ${:key} \
+      -subj "/O=${:organization}/OU=${:organizational_unit}" \
+      -reqexts SAN \
+      -config <(cat {{ software_parameter_dict['openssl-cnf'] }} \
+      <(printf "\n[SAN]\nsubjectAltName=IP:${:ip}")) \
+      -out ${:template-csr}'
+  fi
+update-command = ${:command}
+{#- Can be stopped on error, as does not rely on self provided service #}
+stop-on-error = True
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+
+bin = ${buildout:directory}/bin/
+etc = ${buildout:directory}/etc/
+srv = ${buildout:directory}/srv/
+var = ${buildout:directory}/var/
+tmp = ${buildout:directory}/tmp/
+
+backup = ${:srv}/backup
+log = ${:var}/log
+run = ${:var}/run
+service = ${:etc}/service
+etc-run = ${:etc}/run
+
+[slap_connection]
+# Kept for backward compatibility
+computer_id = ${slap-connection:computer-id}
+partition_id = ${slap-connection:partition-id}
+server_url = ${slap-connection:server-url}
+software_release_url = ${slap-connection:software-release-url}
+key_file = ${slap-connection:key-file}
+cert_file = ${slap-connection:cert-file}

software/rapid-cdn/instance-frontend.cfg.jinja2

+[buildout]
+extends =
+  {{ software_parameter_dict['template-common'] }}
+  {{ software_parameter_dict['template-monitor2'] }}
+
+{#- Manage frontends #}
+{%- set DEFAULT_SCHEME_PORT_MAPPING = {'http': 80, 'https': 443} %}
+{%- set FRONTEND_PARAMETER_KEY_LIST = ['url', 'domain', 'domain-icp-validation'] %} {#- matches schema-rapid-cdn-frontend-shared.input.json #}
+{%- set BACKWARD_COMPATIBLE_FRONTEND_KEY_LIST = ['url'] %} {#- based on old JSON and also found important keys #}
+{%- set FRONTEND_TRANSMIT_LIST = [] %} {#- List of frontend parameters to be passed to kedifa and exposed nodes #}
+{%- set FRONTEND_LIST = [] %} {#- List of frontend parameters used internally for publishing and processing #}
+{%- for slave in instance_parameter_dict['slave-instance-list'] %}
+{%-   set frontend = {} %}
+{%-   set frontend_transmit = {} %}
+{%-   do frontend.__setitem__('title', slave['slave_title']) %}
+{%-   do frontend.__setitem__('reference', slave['slave_reference']) %}
+{%-   do frontend.__setitem__('hostname', '%s.%s' % (frontend['reference'].replace("-", "").replace("_", "").lower(), slapparameter_dict['autogeneration-domain'])) %}
+{%-   do frontend_transmit.__setitem__('title', slave['slave_title']) %}
+{%-   do frontend_transmit.__setitem__('reference', slave['slave_reference']) %}
+{%-   do frontend_transmit.__setitem__('hostname', '%s.%s' % (frontend['reference'].replace("-", "").replace("_", "").lower(), slapparameter_dict['autogeneration-domain'])) %}
+{%-   do frontend.__setitem__('rapid-https','https://%s/' % (frontend['hostname'],)) %}
+{%-   do frontend.__setitem__('rapid-http','http://%s/' % (frontend['hostname'],)) %}
+{%-   do frontend.__setitem__('section-title', 'publish-frontend-' ~ hashlib.md5(frontend['title'].encode('utf-8')).hexdigest()) %}
+{%-   set frontend_dict = {} %}
+{%-   if '_' in slave %}
+{%-     do frontend_dict.update(json.loads(slave['_'])) %}
+{%-   endif %}
+{%-   do frontend.__setitem__('__style__', 'modern') %}
+{%-   for key in BACKWARD_COMPATIBLE_FRONTEND_KEY_LIST %}
+{%-     if key in slave %}
+{%-       do frontend.__setitem__('__style__', 'obsolete') %}
+{%-       do frontend_dict.__setitem__(key, slave[key]) %}
+{%-     endif %}
+{%-   endfor %}
+{%-   do frontend.__setitem__('_', frontend_dict) %}
+{%-   for key in FRONTEND_PARAMETER_KEY_LIST %}
+{%-     if key in frontend %}
+{%-       do frontend_transmit.__setitem__(key, frontend[key]) %}
+{%-     endif %}
+{%-   endfor %}
+{%-   set url_parsed = urllib_parse.urlparse(frontend['_']['url']) %}
+{%-   do frontend_transmit.__setitem__('backend-url', {
+  'scheme': url_parsed.scheme,
+  'hostname': url_parsed.hostname,
+  'port': url_parsed.port or DEFAULT_SCHEME_PORT_MAPPING[url_parsed.scheme],
+  'path': url_parsed.path,
+  'fragment': url_parsed.fragment,
+  'query': url_parsed.query}) %}
+{%-   do FRONTEND_TRANSMIT_LIST.append(frontend_transmit) %}
+{%-   do FRONTEND_LIST.append(frontend) %}
+{%- endfor %}
+
+{%- set PART_LIST = [] %}
+
+{%- for frontend in FRONTEND_LIST | sort(attribute='title') %}
+{%-   do PART_LIST.append(frontend['section-title']) %}
+[{{ frontend['section-title'] }}]
+-slave-reference = {{ frontend['reference'] }}
+{%-   if frontend.pop('__style__') == 'modern' %}
+recipe = slapos.cookbook:publish.serialised
+rapid-url = {{ frontend['rapid-https'] }}
+{%-   else %}
+recipe = slapos.cookbook:publish
+01-ATTENTION = Backward compatbility and obsolete frontend definition detected.
+02-ATTENTION = The instance parameters are hidden in the UI, you see everything empty.
+03-ATTENTION = Do not update parameters in the UI without checking the previous ones, as otherwise they will be lost.
+04-ATTENTION = You can access your parameters by switching to XML view with "Show Parameter XML".
+05-ATTENTION = Then you can wisely migrate them to the new way.
+url = {{ frontend['rapid-http'] }}
+secure_access = {{ frontend['rapid-https'] }}
+site_url = {{ frontend['rapid-http'] }}
+domain = {{ frontend['hostname'] }}
+log-access-url = XXX-TODO
+backend-client-caucase-url = XXX-TODO
+key-generate-auth-url = XXX-TODO
+replication_number = XXX-TODO
+{%-   endif %}
+{%- endfor %}
+
+{#- Request the tree #}
+{%- macro sla(sla_key) %}
+{%-   set sla_key_length = sla_key | length %}
+{%-   for key in slapparameter_dict.keys() %}
+{%-     if key.startswith(sla_key) %}
+sla-{{ key[sla_key_length:] }} = {{ slapparameter_dict[key] }}
+{%-     endif %}
+{%-   endfor %}
+{%- endmacro %}
+
+{%- for name in ['kedifa', 'human', 'log-aggregator'] %}
+{%-   set sla_key = '-sla-%s-computer_guid' % (name,) %}
+{%-   set software_type = 'rapid-cdn-' ~ name %}
+{%-   if not sla_key in slapparameter_dict %}
+{%-     do slapparameter_dict.__setitem__(sla_key, '${slap-connection:computer-id}') %}
+{%-   endif %}
+{%-   set software_key = '-%s-software-release-url' % (name,) %}
+{%-   if not software_key in slapparameter_dict %}
+{%-     do slapparameter_dict.__setitem__(software_key, '${slap-connection:software-release-url}') %}
+{%-   endif %}
+[request-{{ name }}]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional.serialised
+name = {{ name }}
+software-type = {{ software_type }}
+software-url = {{ slapparameter_dict['-%s-software-release-url' % (name,)] }}
+{{ sla('-sla-%s-' % (name,)) }}
+{%- endfor %}
+
+{%- set NODE_PART_LIST = [] %}
+{%- for node, node_dict in slapparameter_dict.get('cluster-dict', {}).items() %}
+{%-   set node_name = 'node-' ~ node %}
+{%-   set part_name = 'request-' ~ node %}
+{%-   do NODE_PART_LIST.append(part_name) %}
+{%-   set sla_dict = node_dict.get('sla', {'computer_guid': '${slap-connection:computer-id}'}) %}
+{%-   set software_release_url = node_dict.pop('-software-release-url', '${slap-connection:software-release-url}') %}
+[{{part_name }}]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional.serialised
+name = {{ node_name }}
+software-type = rapid-cdn-pop
+software-url = {{ software_release_url }}
+config-node-name = {{ node_name }}
+config-frontend-list = {{ dumps(FRONTEND_TRANSMIT_LIST | sort(attribute='title')) }}
+config-kedifa-caucase-url = ${kedifa-caucase-promise:config-url}
+{%-   for sla_key, sla_value in sla_dict.items() %}
+sla-{{ sla_key }} = {{ sla_value }}
+{%-   endfor %}
+{%- endfor %}
+
+[request-kedifa]
+config-kedifa-port = {{ dumps(slapparameter_dict.get('-kedifa-port', 7890)) }}
+config-caucase-port = {{ dumps(slapparameter_dict.get('-kedifa-caucase-port', 8090)) }}
+config-frontend-list = {{ dumps(FRONTEND_TRANSMIT_LIST | sort(attribute='title')) }}
+return =
+  caucase-url
+  master-key-generate-auth-url
+  master-key-upload-url
+
+[kedifa-caucase-promise]
+<= monitor-promise-base
+promise = check_url_available
+name = ${:_buildout_section_name_}.py
+config-http-code = 200
+config-url = ${request-kedifa:connection-caucase-url}
+
+[directory]
+url-done = ${:var}/url-done
+
+[url-ready]
+recipe = slapos.recipe.build
+depot = ${directory:url-done}
+file = ${directory:url-done}/${:_buildout_section_name_}
+marker = ${kedifa-config:marker}
+init =
+  import os
+  if options['marker'] in options['url']:
+    if os.path.exists(options['file']):
+      os.unlink(options['file'])
+  else:
+    try:
+      with open(options['file'], 'w') as fh:
+        fh.write('')
+    except FileNotFoundError:
+      # directories are created during install, but promise runs during init
+      # so do nothing in such case
+      pass
+
+[kedifa-master-key-generate-auth-url]
+<= url-ready
+url = ${request-kedifa:connection-master-key-generate-auth-url}
+
+[kedifa-master-key-upload-url]
+<= url-ready
+url = ${request-kedifa:connection-master-key-upload-url}
+
+[kedifa-master-key-promise]
+<= monitor-promise-base
+promise = check_file_state
+name = ${:_buildout_section_name_}.py
+config-state = present
+
+[kedifa-master-key-generate-auth-url-promise]
+<= kedifa-master-key-promise
+config-filename = ${kedifa-master-key-generate-auth-url:file}
+url = ${kedifa-master-key-generate-auth-url:url}
+
+[kedifa-master-key-upload-url-promise]
+<= kedifa-master-key-promise
+config-filename = ${kedifa-master-key-upload-url:file}
+url = ${kedifa-master-key-upload-url:url}
+
+[publish-information]
+<= monitor-publish
+recipe = slapos.cookbook:publish.serialised
+xxx-replace-with-information-fetch-depends =
+  ${request-human:name}
+  ${request-log-aggregator:name}
+{%- for node_part in NODE_PART_LIST %}
+  {{ '${' ~ node_part}}:name}
+{%- endfor %}
+
+kedifa-caucase-url = ${kedifa-caucase-promise:config-url}
+key-generate-auth-url = ${kedifa-master-key-generate-auth-url-promise:url}
+key-upload-url = ${kedifa-master-key-upload-url-promise:url}
+
+[buildout]
+parts +=
+  publish-information
+{% for part in PART_LIST %}
+{{ '  %s' % part }}
+{% endfor %}

software/rapid-cdn/instance-human.cfg.jinja2

+[buildout]
+extends =
+  {{ software_parameter_dict['template-common'] }}
+  {{ software_parameter_dict['template-monitor2'] }}

software/rapid-cdn/instance-kedifa.cfg.jinja2

+{%- import "caucase" as caucase with context %}
+[buildout]
+extends =
+  {{ software_parameter_dict['template-common'] }}
+  {{ software_parameter_dict['template-monitor2'] }}
+
+parts +=
+  kedifa
+  caucased
+  caucase-updater
+  publish-information
+
+[caucased]
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+{% set caucase_host = '[' ~ instance_parameter_dict['ipv6-random'] ~ ']' %}
+{%- set caucase_netloc = caucase_host ~ ':' ~ slapparameter_dict['caucase-port'] -%}
+{%- set caucase_url = 'http://' ~ caucase_netloc -%}
+{{-  caucase.caucased(
+      prefix='caucased',
+      buildout_bin_directory=software_parameter_dict['bin-directory'],
+      caucased_path='${directory:service}/caucased',
+      backup_dir='${directory:backup-caucased}',
+      data_dir='${directory:caucased}',
+      netloc=caucase_netloc,
+      tmp='${directory:tmp}',
+      service_auto_approve_count=0,
+      user_auto_approve_count=1,
+      key_len=2048
+)}}
+
+[caucase-updater]
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+{{ caucase.updater(
+     prefix='caucase-updater',
+     buildout_bin_directory=software_parameter_dict['bin-directory'],
+     updater_path='${directory:service}/caucase-updater',
+     url=caucase_url,
+     data_dir='${directory:srv}/caucase-updater',
+     crt_path='${kedifa-config:certificate}',
+     ca_path='${kedifa-config:ca-certificate}',
+     crl_path='${kedifa-config:crl}',
+     key_path='${kedifa-csr:key}',
+     on_renew='${kedifa-reloader:rendered}',
+     template_csr='${kedifa-csr:template-csr}'
+)}}
+
+
+[kedifa-csr]
+<= csr
+organizational_unit = Kedifa Partition
+template-csr = ${kedifa-config:template-csr}
+key = ${kedifa-config:key}
+ip = ${kedifa-config:ip}
+
+[kedifa-reloader]
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:etc-run}/kedifa-reloader
+mode = 0700
+
+template = inline:#!{{ software_parameter_dict['dash'] }}
+{%- raw %}
+  {{ content }}
+{%- endraw %}
+
+command =
+  kill -HUP `cat ${kedifa-config:pidfile}`
+context =
+  key content :command
+
+[directory]
+# KeDiFa directories
+srv-kedifa = ${:srv}/kedifa
+
+# CAUCASE directories
+caucased = ${:srv}/caucased
+backup-caucased = ${:backup}/caucased
+
+# reservation
+reservation = ${:srv}/reservation
+
+[kedifa-config]
+ip = {{ instance_parameter_dict['ipv6-random'] }}
+port = {{ slapparameter_dict['kedifa-port'] }}
+db = ${directory:srv-kedifa}/kedifa.sqlite
+certificate = ${directory:srv-kedifa}/certificate.pem
+key = ${:certificate}
+ca-certificate = ${directory:srv-kedifa}/ca-certificate.pem
+crl = ${directory:srv-kedifa}/crl.pem
+template-csr = ${directory:srv-kedifa}/template-csr.pem
+pidfile = ${directory:run}/kedifa.pid
+logfile = ${directory:log}/kedifa.log
+
+[kedifa]
+recipe = slapos.cookbook:wrapper
+command-line = {{ software_parameter_dict['kedifa'] }}
+  --ip ${kedifa-config:ip}
+  --port ${kedifa-config:port}
+  --db ${kedifa-config:db}
+  --certificate ${kedifa-config:certificate}
+  --ca-certificate ${kedifa-config:ca-certificate}
+  --crl ${kedifa-config:crl}
+  --pidfile ${kedifa-config:pidfile}
+  --logfile ${kedifa-config:logfile}
+
+wrapper-path = ${directory:service}/kedifa
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+
+[master-auth-random-config]
+file = ${directory:reservation}/${:_buildout_section_name_}
+
+[master-auth-random-promise]
+<= monitor-promise-base
+promise = check_file_state
+name = ${:_buildout_section_name_}.py
+config-filename = ${master-auth-random-config:file}
+config-state = present
+
+[master-auth-random]
+recipe = slapos.recipe.build
+kedifa-reserve = https://[${kedifa-config:ip}]:${kedifa-config:port}/reserve-id
+cert = ${kedifa-config:certificate}
+cacert = ${kedifa-config:ca-certificate}
+result = ${kedifa-config:marker}
+file = ${master-auth-random-promise:config-filename}
+# The reservation has to happen during init, this slapos.recipe.build is used
+# with python's requests instead of curl
+init =
+  import errno
+  import os
+  import requests
+  try:
+    with open(options['file']) as fh:
+      options['result'] = fh.read()
+  except FileNotFoundError:
+    try:
+      result = requests.post(
+        options['kedifa-reserve'],
+        cert=options['cert'],
+        verify=options['cacert']
+      )
+      status_code = result.status_code
+    except Exception:
+      status_code = 500
+    if status_code == 201:
+      with open(options['file'], 'w') as fh:
+        fh.write(result.text)
+      options['result'] = result.text
+
+[promise-kedifa-http-reply]
+<= monitor-promise-base
+promise = check_url_available
+name = kedifa-http-reply.py
+# Kedifa replies 400 on /, so use it to be sure that Kedifa replied
+config-http-code = 400
+config-url = https://[${kedifa-config:ip}]:${kedifa-config:port}
+config-ca-cert-file = ${kedifa-config:ca-certificate}
+
+[publish-information]
+recipe = slapos.cookbook:publish.serialised
+caucase-url = {{ caucase_url }}
+master-key-generate-auth-url = ${promise-kedifa-http-reply:config-url}/${master-auth-random:result}/generateauth
+master-key-upload-url = ${promise-kedifa-http-reply:config-url}/${master-auth-random:result}?auth=
+master-key-download-url = ${promise-kedifa-http-reply:config-url}/${master-auth-random:result}

software/rapid-cdn/instance-log-aggregator.cfg.jinja2

+[buildout]
+extends =
+  {{ software_parameter_dict['template-common'] }}
+  {{ software_parameter_dict['template-monitor2'] }}

software/rapid-cdn/instance-pop.cfg.jinja2

+{%- import "caucase" as caucase with context %}
+[buildout]
+extends =
+  {{ software_parameter_dict['template-common'] }}
+  {{ software_parameter_dict['template-monitor2'] }}
+
+parts +=
+  kedifa-caucase-updater
+  front-haproxy
+
+[directory]
+kedifa-ssl = ${:srv}/kedifa-ssl
+kedifa-caucase-updater = ${:srv}/kedifa-caucase-updater
+
+[kedifa-config]
+certificate = ${directory:kedifa-ssl}/certificate.pem
+key = ${:certificate}
+ca-certificate = ${directory:kedifa-ssl}/ca-certificate.pem
+crl = ${directory:kedifa-ssl}/crl.pem
+template-csr = ${directory:kedifa-ssl}/template-csr.pem
+
+[kedifa-csr]
+<= csr
+organizational_unit = {{ slapparameter_dict['node-name'] }}
+template-csr = ${kedifa-config:template-csr}
+key = ${kedifa-config:key}
+ip = {{ instance_parameter_dict['ipv6-random'] }}
+
+[kedifa-caucase-updater]
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+{{ caucase.updater(
+     prefix='kedifa-caucase-updater',
+     buildout_bin_directory=software_parameter_dict['bin-directory'],
+     updater_path='${directory:service}/kedifa-caucase-updater',
+     url=slapparameter_dict['kedifa-caucase-url'],
+     data_dir='${directory:kedifa-caucase-updater}',
+     crt_path='${kedifa-config:certificate}',
+     ca_path='${kedifa-config:ca-certificate}',
+     crl_path='${kedifa-config:crl}',
+     key_path='${kedifa-csr:key}',
+     template_csr='${kedifa-csr:template-csr}'
+)}}
+
+[directory]
+backend-haproxy-rsyslogd-spool = ${:run}/backend-haproxy-rsyslogd-spool
+
+[jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+extensions = jinja2.ext.do
+extra-context =
+context =
+    ${:extra-context}
+
+[front-rsyslogd-configuration-section]
+log-socket = ${directory:run}/frlog.sck
+log-file = ${directory:log}/front-haproxy.log
+pid-file = ${directory:run}/front-rsyslogd.pid
+spool-directory = ${directory:backend-haproxy-rsyslogd-spool}
+
+[front-rsyslogd-configuration]
+<= jinja2-template-base
+template = {{ software_parameter_dict['template-front-rsyslogd'] }}
+rendered = ${directory:etc}/front-rsyslogd.conf
+extra-context =
+  section configuration front-rsyslogd-configuration-section
+
+[front-rsyslogd]
+recipe = slapos.cookbook:wrapper
+command-line = {{ software_parameter_dict['rsyslogd'] }} -n -i ${front-rsyslogd-configuration-section:pid-file} -f ${front-rsyslogd-configuration:rendered}
+wrapper-path = ${directory:service}/${:_buildout_section_name_}
+log-socket = ${front-rsyslogd-configuration-section:log-socket}
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+
+[front-configuration-section]
+log-socket = ${front-rsyslogd:log-socket}
+frontend-list = {{ dumps(slapparameter_dict['frontend-list']) }}
+pid-file = ${directory:run}/front-haproxy.pid
+# XXX!
+request-timeout = 100
+backend-connect-timeout = 100
+backend-connect-retries = 3
+http-port = 8080
+https-port = 4443
+ipv4 = {{ dumps(instance_parameter_dict['ipv4-random']) }}
+ipv6 = {{ dumps(instance_parameter_dict['ipv6-random']) }}
+statistic-port = 9696
+statistic-certificate = /dev/null
+statistic-identification = oink
+statistic-username = statu
+statistic-password = statp
+certificate = /dev/null
+
+
+[front-configuration]
+<= jinja2-template-base
+binary = {{ software_parameter_dict['haproxy'] }}
+template = {{ software_parameter_dict['template-front-haproxy'] }}
+rendered = ${directory:etc}/front-haproxy.cfg
+extra-context =
+  section configuration front-configuration-section
+
+[front-haproxy]
+recipe = slapos.cookbook:wrapper
+command-line = ${front-configuration:binary} -f ${front-configuration:rendered}
+wrapper-path = ${directory:service}/${:_buildout_section_name_}
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg

software/rapid-cdn/instance.cfg.jinja2

+[buildout]
+extends = {{ software_parameter_dict['template-common'] }}
+
+parts =
+  switch-softwaretype
+
+[switch-softwaretype]
+recipe = slapos.cookbook:switch-softwaretype
+{#- Cluster itself #}
+rapid-cdn-frontend = template-rapid-cdn-frontend:rendered
+rapid-cdn-pop = template-rapid-cdn-pop:rendered
+rapid-cdn-human = template-rapid-cdn-human:rendered
+rapid-cdn-kedifa = template-rapid-cdn-kedifa:rendered
+rapid-cdn-log-aggregator = template-rapid-cdn-log-aggregator:rendered
+
+{#- Default #}
+default = ${:rapid-cdn-frontend}
+{#- Fallback #}
+RootSoftwareInstance = ${:rapid-cdn-frontend}
+
+[jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/${:filename}
+extensions = jinja2.ext.do
+extra-context =
+context =
+    import json json
+    import hashlib hashlib
+    key slapparameter_dict slap-configuration:configuration
+    section instance_parameter_dict slap-configuration
+    section software_parameter_dict software-parameter-section
+    ${:extra-context}
+import-list =
+  file caucase software-parameter-section:caucase-jinja2-library
+
+[template-base]
+<= jinja2-template-base
+filename = ${:_buildout_section_name_}.cfg
+
+{%- for template in software_parameter_dict.keys() %}
+{%-   if template.startswith('template-rapid-cdn-') %}
+[{{ template }}]
+<= template-base
+template = {{ software_parameter_dict[template] }}
+{%-   endif %}
+{%- endfor %}
+
+[template-rapid-cdn-frontend]
+extra-context =
+  import urllib_parse urllib.parse
+
+[software-parameter-section]
+{%- for key, value in software_parameter_dict.items() %}
+{{ key }} = {{ dumps(value) }}
+{% endfor -%}
+
+[slap-configuration]
+# Fetches parameters defined in SlapOS Master for this instance.
+# Always the same.
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}

software/rapid-cdn/software.cfg

 [buildout]
 extends =
+  buildout.hash.cfg
+  ../../stack/caucase/buildout.cfg
   ../../stack/slapos.cfg
+  ../../component/haproxy/buildout.cfg
+  ../../component/nginx/buildout.cfg
+  ../../component/python3/buildout.cfg
+  ../../component/rsyslogd/buildout.cfg
+  ../../component/trafficserver/buildout.cfg
+
+parts +=
+  template
+
+[python]
+part = python3
+
+[template-render-base]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/${:_update_hash_filename_}
+rendered = ${buildout:directory}/${:_buildout_section_name_}.cfg
+mode = 0644
+context =
+  section software_parameter_dict software-parameter-section
+  ${:extra-context}
+extra-context =
+
+[template]
+<= template-render-base
+
+[template-common]
+<= template-render-base
+
+[template-download-base]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+
+[template-rapid-cdn-frontend]
+<= template-download-base
+
+[template-rapid-cdn-human]
+<= template-download-base
+
+[template-rapid-cdn-pop]
+<= template-download-base
+
+[template-rapid-cdn-kedifa]
+<= template-download-base
+
+[template-rapid-cdn-log-aggregator]
+<= template-download-base
+
+[template-front-haproxy]
+<= template-download-base
+
+[template-front-rsyslogd]
+<= template-download-base
+
+[kedifa]
+recipe = zc.recipe.egg
+eggs =
+  ${python-cryptography:egg}
+  kedifa
+
+[software-parameter-section]
+# This section passes binaries and other software provided outputs to all
+# profiles. In the same time allows to gather all parameters in one place.
+
+# Buildout parameters
+bin-directory = ${buildout:bin-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+eggs-directory = ${buildout:eggs-directory}
+
+# Software
+caucase = ${caucase-eggs:bin-directory}/caucase
+caucase-probe = ${caucase-eggs:bin-directory}/caucase-probe
+caucase-updater = ${caucase-eggs:bin-directory}/caucase-updater
+caucased = ${caucase-eggs:bin-directory}/caucased
+dash = ${dash-output:dash}
+haproxy = ${haproxy:location}/sbin/haproxy
+kedifa = ${kedifa:bin-directory}/kedifa
+kedifa-updater = ${kedifa:bin-directory}/kedifa-updater
+nginx = ${nginx-output:nginx}
+nginx_mime = ${nginx-output:mime}
+openssl = ${openssl:location}/bin/openssl
+openssl-cnf = ${openssl:location}/etc/ssl/openssl.cnf
+rsyslogd = ${rsyslogd:location}/sbin/rsyslogd
+trafficserver = ${trafficserver:location} {#- XXX: Fix to expose exact binaries #}
+
+# Libraries
+caucase-jinja2-library = ${caucase-jinja2-library:target}
+
+# Templates
+template-common = ${template-common:rendered}
+template-front-haproxy = ${template-front-haproxy:target}
+template-front-rsyslogd = ${template-front-rsyslogd:target}
+template-monitor2 = ${monitor2-template:rendered}
+template-rapid-cdn-frontend = ${template-rapid-cdn-frontend:target}
+template-rapid-cdn-human = ${template-rapid-cdn-human:target}
+template-rapid-cdn-kedifa = ${template-rapid-cdn-kedifa:target}
+template-rapid-cdn-log-aggregator = ${template-rapid-cdn-log-aggregator:target}
+template-rapid-cdn-pop = ${template-rapid-cdn-pop:target}
+
+[versions]
+kedifa = 0.0.6
+# modernish zc.lockfile is required by kedifa
+zc.lockfile = 1.4

software/rapid-cdn/template/front-haproxy.cfg.jinja2

+{%- macro apache_log_format() %}
+  capture request header Referer len 512
+  capture request header User-Agent len 512
+  log-format "%{+Q}o %{-Q}ci - - [%trg] %r %ST %B %{+Q}[capture.req.hdr(0)] %{+Q}[capture.req.hdr(1)] %Tt"
+{%- endmacro %}
+
+global
+  pidfile {{ configuration['pid-file'] }}
+  # master-worker is compatible with foreground with process management
+  master-worker
+
+log {{ configuration['log-socket'] }} local0
+
+defaults
+  log global
+  mode http
+  option httplog
+  timeout queue 60s
+  timeout server {{ configuration['request-timeout'] }}s
+  timeout client {{ configuration['request-timeout'] }}s
+  timeout connect {{ configuration['backend-connect-timeout'] }}s
+  retries {{ configuration['backend-connect-retries'] }}
+  {#- Allow to start with not resolved yet servers #}
+  default-server init-addr last,libc,none
+
+# statistic
+frontend statistic
+  bind {{ configuration['ipv6']}}:{{ configuration['statistic-port'] }} {#- ssl crt-list /proper-file #}
+  stats enable
+  stats uri /
+  stats show-desc {{ configuration['statistic-identification'] }}
+  stats auth {{ configuration['statistic-username'] }}:{{ configuration['statistic-password'] }}
+  stats realm {{ configuration['statistic-identification'] }}
+{{- apache_log_format() }}
+
+frontend http
+  bind {{ configuration['ipv4'] }}:{{ configuration['http-port'] }}
+  bind {{ configuration['ipv6'] }}:{{ configuration['http-port'] }}
+{{- apache_log_format() }}
+{%- for frontend in configuration['frontend-list'] %}
+  acl is_{{ frontend['reference'] }}_http hdr_reg(host) -i ^{{ frontend['hostname'] }}($|:.*)
+  use_backend {{ frontend['reference'] }}-http if is_{{ frontend['reference'] }}_http
+{%- endfor %}
+  default_backend INSTANCE_NOT_READY_BACKEND
+
+frontend https
+  bind {{ configuration['ipv4'] }}:{{ configuration['https-port'] }}
+  bind {{ configuration['ipv6'] }}:{{ configuration['https-port'] }}
+{{- apache_log_format() }}
+  default_backend INSTANCE_NOT_READY_BACKEND
+
+{%- for frontend in configuration['frontend-list'] %}
+{%-   set ssl_list = [] %}
+{%-   if frontend['backend-url']['scheme'] == 'https' %}
+{%-     do ssl_list.append('ssl verify none') %}
+{%-   endif %}
+backend {{ frontend['reference'] }}-http
+  http-request set-path {{ frontend['backend-url']['path'] }}%[path]
+  server {{ frontend['reference'] }}-backend-http {{ frontend['backend-url']['hostname'] }}:{{ frontend['backend-url']['port'] }} {{ ' '.join(ssl_list) }}
+{%- endfor %}
+
+backend INSTANCE_NOT_READY_BACKEND
+  server INSTANCE_NOT_READY_BACKEND 2001:67c:1254:53:b362:dd91:0:1:8080
+  http-request set-path STUB_FOR_NOT_READY_BACKEND%[path]
+
+{#- keep the empty line below #}
+

software/rapid-cdn/template/front-rsyslogd.conf.jinja2

+module(
+  load="imuxsock"
+  SysSock.Name="{{ configuration['log-socket'] }}")
+
+# Just simply output the raw line without any additional information, as
+# haproxy emits enough information by itself
+# Also cut out first empty space in msg, which is related to rsyslogd
+# internal and end up cutting on 8k, as it's default of $MaxMessageSize
+template(name="rawoutput" type="string" string="%msg:2:8192%\n")
+$ActionFileDefaultTemplate rawoutput
+
+$FileCreateMode 0600
+$DirCreateMode 0700
+$Umask 0022
+
+$WorkDirectory {{ configuration['spool-directory'] }}
+
+## Setup logging per slave, by extracting the slave name from the log stream
+{%- raw %}
+#{%- set regex = ".*-backend (.*)-http.{0,1}(|-failover)/" %}
+#template(name="extract_slave_name" type="string" string="%msg:R,ERE,1,FIELD:{{ regex }}--end%")
+#set $!slave_name = exec_template("extract_slave_name");
+#template(name="slave_output" type="string" string="{{ configuration['caddy-log-directory'] }}/%$!slave_name%_backend_log")
+#if (re_match($msg, "{{ regex }}")) then {
+# action(type="omfile" dynaFile="slave_output")
+# stop
+#}
+{%- endraw %}
+
+{#- emit all not catched messages to full log file #}
+*.* {{ configuration['log-file'] }}

software/rapid-cdn/test/setup.py

@@ -43,6 +43,8 @@ setup(
     install_requires=[
         'slapos.core',
         'slapos.libnetworkcache',
+        'requests',
+        'caucase',
     ],
     zip_safe=True,
     test_suite='test',

software/rapid-cdn/test/test.py

@@ -25,7 +25,12 @@
 #
 ##############################################################################
 
+import json
 import os
+import shutil
+import subprocess
+import tempfile
+import slapos.slap.standalone
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 
@@ -34,6 +39,109 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
         os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
 
 
+class CaucaseBinaryClient(object):
+  # Class to mimic user, which is using caucase binaries and some directories
+  # to be a client, thus real binaries are used with executable call examples
+  def __init__(self, path, url):
+    self.path = path
+    os.mkdir(self.path)
+    self.url = url
+
+    def makepath(p):
+      return os.path.join(self.path, p)
+    self.client_csr = makepath('client.csr.pem')
+    self.client = makepath('client.pem')
+    self.ca_crt = makepath('ca-crt.pem')
+    self.user_ca_crt = makepath('user-ca-crt.pem')
+    self.crl = makepath('crl.pem')
+    self.user_crl = makepath('user-crl.pem')
+    subprocess.check_call([
+      "openssl", "req", "-out", self.client_csr, "-new", "-newkey", "rsa:2048",
+      "-nodes", "-keyout", self.client, "-subj", "/CN=user"])
+    self.cau_list = [
+      "caucase", "--ca-url", self.url, "--ca-crt", self.ca_crt,
+      "--user-ca-crt", self.user_ca_crt, "--crl", self.crl, "--user-crl",
+      self.user_crl]
+    self.user_cau_list = self.cau_list + ["--mode", "user"]
+    self.service_cau_list = self.cau_list + [
+      "--user-key", self.client, "--mode", "service"]
+    output = subprocess.check_output(
+      self.user_cau_list + ["--send-csr", self.client_csr])
+    subprocess.check_call(
+      self.user_cau_list + ["--get-crt", output.split()[0], self.client])
+
+  def signServiceCsr(self, ou):
+    for line in subprocess.check_output(
+      self.service_cau_list + ["--list-csr"]).splitlines():
+      splitted = line.decode().split('|')
+      if len(splitted) == 2:
+        if 'OU=%s)' % (ou,) in splitted[1]:
+          csr_id = splitted[0].strip()
+          subprocess.check_call(self.service_cau_list + ["--sign-csr", csr_id])
+          break
+
+
 class Test(SlapOSInstanceTestCase):
-  def test(self):
-    self.fail('TODO')
+  # full diffs are very informative for those tests
+  maxDiff = None
+  # as instantiation is split in chunks with cluster administrator operations,
+  # let it run a bit rarely
+  instance_max_retry = 5
+
+  @classmethod
+  def _setUpClass(cls):
+    cls.work_dir = tempfile.mkdtemp()
+    try:
+      super()._setUpClass()
+    except slapos.slap.standalone.SlapOSNodeInstanceError:
+      pass
+
+    # Cluster is initially setup, now it's required to allow joining nodes,
+    # which is usually cluster administrator responsibility
+    connection_parameter_dict = json.loads(
+      cls.requestDefaultInstance().getConnectionParameterDict()['_'])
+
+    cls.kedifa_caucase_client = CaucaseBinaryClient(
+      os.path.join(cls.work_dir, 'kedifa_caucase'),
+      connection_parameter_dict['kedifa-caucase-url'])
+    cls.kedifa_caucase_client.signServiceCsr("Kedifa Partition")
+
+    # Time travel to the future: restart all caucase updaters, so they will
+    # pick up just signed service CSR; note that in reality the cluster
+    # administrator can wait for the caucase updater to kick in
+    with cls.slap.instance_supervisor_rpc as instance_supervisor_rpc:
+      for process in instance_supervisor_rpc.getAllProcessInfo():
+        if 'caucase-updater' in process['name']:
+          process_id = '%(group)s:%(name)s' % process
+          instance_supervisor_rpc.stopProcess(process_id)
+          instance_supervisor_rpc.startProcess(process_id)
+
+    cls.waitForInstance()
+
+  @classmethod
+  def tearDownClass(cls):
+    super().tearDownClass()
+    shutil.rmtree(cls.work_dir)
+
+  def assertMonitorSetupUrl(self, value):
+    # TODO: check that monitor can correctly acces whole cluster
+    pass
+
+  def test_connection_parameter(self):
+    connection_parameter_dict = self.requestDefaultInstance(
+      ).getConnectionParameterDict()
+    self.assertIn('_', connection_parameter_dict)
+    json_parmeter_dict = json.loads(connection_parameter_dict.pop('_'))
+    self.assertEqual({}, connection_parameter_dict)
+    self.assertNotIn(
+      '__NotReadyYet__', json_parmeter_dict.pop('key-generate-auth-url'))
+    self.assertNotIn(
+      '__NotReadyYet__', json_parmeter_dict.pop('key-upload-url'))
+    self.assertMonitorSetupUrl(json_parmeter_dict.pop('monitor-setup-url'))
+    self.assertEqual(
+      {
+        'kedifa-caucase-url': 'http://[%s]:8090' % (self._ipv6_address,),
+        'monitor-base-url': 'https://[%s]:8196' % (self._ipv6_address,),
+        'xxx-replace-with-information-fetch-depends': 'human\nlog-aggregator'
+      },
+      json_parmeter_dict)