caddy-frontend: Implement netloc-list

JSON schema uses simple string and internal process, and caddy instances are
not kept simple and are not considered future-proof.

software/caddy-frontend/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg.in
-md5sum = 1dfbd20c77fb3c1f01005a8a920d2ed9
+md5sum = fb3a20e7f555a9ce7fe1ec547d0fcdfc
 
 [profile-common]
 filename = instance-common.cfg.in
@@ -26,11 +26,11 @@ md5sum = 0950e09ad1f03f0789308f5f7a7eb1b8
 
 [profile-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = 7c2e52b76c42bed95702763c344e41dd
+md5sum = c5d1e235959a877b4f3157369c6f5e10
 
 [profile-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = 313671d343ceccfca5af1baa642132c5
+md5sum = c67e172c0c6eca955b18962404056a33
 
 [profile-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
@@ -50,7 +50,7 @@ md5sum = 37475d79f28c5f126bc1947fdb938fdb
 
 [template-backend-haproxy-configuration]
 _update_hash_filename_ = templates/backend-haproxy.cfg.in
-md5sum = d2851c7ebd2c9baa2edecb3ca3485511
+md5sum = ae4c9ce775ea003aa51eda5ecbbeec73
 
 [template-empty]
 _update_hash_filename_ = templates/empty.in
@@ -98,7 +98,7 @@ md5sum = 8e1c6c06c09beb921965b3ce98c67c9e
 
 [caddyprofiledeps-dummy]
 filename = caddyprofiledummy.py
-md5sum = 38792c2dceae38ab411592ec36fff6a8
+md5sum = 59cb33f11272ee09eccea74981d2304a
 
 [profile-kedifa]
 filename = instance-kedifa.cfg.in

software/caddy-frontend/caddyprofiledummy.py

+import urlparse
+
 class Recipe(object):
   def __init__(self, *args, **kwargs):
     pass
@@ -7,3 +9,13 @@ class Recipe(object):
 
   def update(self):
     return self.install()
+
+def validate_netloc(netloc):
+  # a bit crazy way to validate that the passed parameter is haproxy
+  # compatible server netloc
+  parsed = urlparse.urlparse('scheme://'+netloc)
+  if ':' in parsed.hostname:
+    hostname = '[%s]' % parsed.hostname
+  else:
+    hostname = parsed.hostname
+  return netloc == '%s:%s' % (hostname, parsed.port)

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -207,6 +207,15 @@ context =
 {%       endif %}
 {%     endif %}
 {%   endfor %}
+{%   for url_key in ['url-netloc-list', 'https-url-netloc-list', 'health-check-failover-url-netloc-list'] %}
+{%     if url_key in slave %}
+{%       for netloc in slave[url_key].split() %}
+{%         if not caddyprofiledummy.validate_netloc(netloc) %}
+{%           do slave_error_list.append('slave %s %r invalid' % (url_key, netloc)) %}
+{%         endif %}
+{%       endfor %}
+{%     endif %}
+{%   endfor %}
 {%   for k in ['ssl_proxy_ca_crt', 'health-check-failover-ssl-proxy-ca-crt'] %}
 {%     if k in slave %}
 {%       set crt = slave.get(k, '') %}
@@ -909,4 +918,4 @@ parts =
   caucased-backend-client-promise
 {% for part in part_list %}
 {{ '  %s' % part }}
-{% endfor %}
\ No newline at end of file
+{% endfor %}

software/caddy-frontend/instance-slave-caddy-input-schema.json

@@ -348,6 +348,26 @@
       ],
       "type": "string",
       "default": "false"
+    },
+    "url-netloc-list": {
+      "type": "string",
+      "title": "[EXPERT] List of netlocs for \"Backend URL\"",
+      "description": "Space separated list of netlocs (ip and port) of backend to connect to. They will share the scheme and path of the original URL and additional backend parameters (like \"SSL Backend Authority's Certificate\"). Each of them will be used, and at least one is enough for the connectivity to work, and the best results are with \"Health Check\" feature enabled. Port is mandatory, so hostnames shall be provided as hostname:port (eg. example.com:80), IPv4 - as ipv4:port (eg. 127.0.0.1:80), IPv6 - as ipv6:port (eg. ::1:80). Simply this parameters only overrides netloc (network location) of the original URL."
+    },
+    "https-url-netloc-list": {
+      "type": "string",
+      "title": "[EXPERT] List of netlocs for \"HTTPS Backend URL\"",
+      "description": "See \"[EXPERT] List of netlocs for \"Backend URL\"\" description."
+    },
+    "health-check-failover-url-netloc-list": {
+      "type": "string",
+      "title": "[EXPERT] List of netlocs for \"Failover backend URL\"",
+      "description": "See \"[EXPERT] List of netlocs for \"Backend URL\"\" description."
+    },
+    "health-check-failover-https-url-netloc-list": {
+      "type": "string",
+      "title": "[EXPERT] List of netlocs for \"Failover HTTPS Backend URL\"",
+      "description": "See \"[EXPERT] List of netlocs for \"Backend URL\"\" description."
     }
   },
   "title": "Input Parameters",

software/caddy-frontend/instance.cfg.in

@@ -55,6 +55,7 @@ extra-context =
     import subprocess_module subprocess
     import functools_module functools
     import validators validators
+    import caddyprofiledummy caddyprofiledummy
 # Must match the key id in [switch-softwaretype] which uses this section.
     raw software_type RootSoftwareInstance-default-custom-personal-replicate
 

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -53,7 +53,7 @@ context =
 {#-     * stabilise values for backend #}
 {%-   for key, prefix in [('url', 'http_backend'), ('https-url', 'https_backend')] %}
 {%-     set parsed = urlparse_module.urlparse(slave_instance.get(key, '').strip()) %}
-{%-     set info_dict = {'scheme': parsed.scheme, 'hostname': parsed.hostname, 'port': parsed.port or DEFAULT_PORT[parsed.scheme], 'path': parsed.path, 'fragment': parsed.fragment, 'query': parsed.query} %}
+{%-     set info_dict = {'scheme': parsed.scheme, 'hostname': parsed.hostname, 'port': parsed.port or DEFAULT_PORT[parsed.scheme], 'path': parsed.path, 'fragment': parsed.fragment, 'query': parsed.query, 'netloc-list': slave_instance.get(key + '-netloc-list', '').split() } %}
 {%-     do slave_instance.__setitem__(prefix, info_dict) %}
 {%-   endfor %}
 {%-   do slave_instance.__setitem__('ssl_proxy_verify', ('' ~ slave_instance.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES) %}
@@ -66,6 +66,7 @@ context =
 {%-     do info_dict.__setitem__('health-check-failover-path', parsed.path) %}
 {%-     do info_dict.__setitem__('health-check-failover-query', parsed.query) %}
 {%-     do info_dict.__setitem__('health-check-failover-fragment', parsed.fragment) %}
+{%-     do info_dict.__setitem__('health-check-netloc-list', slave_instance.get('health-check-failover-url-netloc-list', '').split()) %}
 {%-     do slave_instance.__setitem__(prefix, info_dict) %}
 {%-   endfor %}
 {%-   do slave_instance.__setitem__('health-check-failover-ssl-proxy-verify', ('' ~ slave_instance.get('health-check-failover-ssl-proxy-verify', '')).lower() in TRUE_VALUES) %}
@@ -658,4 +659,4 @@ commands =
 promise = check_command_execute
 name = ${:_buildout_section_name_}.py
 config-command =
-  ${logrotate:wrapper-path} -d
\ No newline at end of file
+  ${logrotate:wrapper-path} -d

software/caddy-frontend/templates/backend-haproxy.cfg.in

@@ -106,7 +106,7 @@ backend {{ slave_instance['slave_reference'] }}-{{ scheme }}
 {%-         do path_list.append(query) %}
 {%-       endif %}
 {%-       set path = '?'.join(path_list) %}
-{%-       if hostname and port %}
+{%-       if hostname and port or len(info_dict['netloc-list']) > 0 %}
   timeout server {{ slave_instance['request-timeout'] }}s
   timeout connect {{ slave_instance['backend-connect-timeout'] }}s
   retries {{ slave_instance['backend-connect-retries'] }}
@@ -122,7 +122,15 @@ backend {{ slave_instance['slave_reference'] }}-{{ scheme }}
 {%-           endif %}
 {%-           do active_check_option_list.append('timeout check %ss' % (slave_instance['health-check-timeout'])) %}
 {%-         endif %}
+{%-         if len(info_dict['netloc-list']) > 0 %}
+{%-           set counter = {'count': 1} %}
+{%-           for netloc in info_dict['netloc-list'] %}
+  server {{ slave_instance['slave_reference'] }}-backend-{{ scheme }}-{{ counter['count'] }} {{ netloc }} {{ ' '.join(ssl_list) }} {{ ' ' + ' '.join(active_check_list)}}
+{%-             do counter.__setitem__('count', counter['count'] + 1) %}
+{%-           endfor %}
+{%-         else %}
   server {{ slave_instance['slave_reference'] }}-backend-{{ scheme }} {{ hostname }}:{{ port }} {{ ' '.join(ssl_list) }} {{ ' ' + ' '.join(active_check_list)}}
+{%-         endif %}
 {%-         for active_check_option in active_check_option_list %}
   {{ active_check_option }}
 {%-         endfor %}
@@ -161,10 +169,18 @@ backend {{ slave_instance['slave_reference'] }}-{{ scheme }}-failover
 {%-       endif %}
 {%-       set path = '?'.join(path_list) %}
 {%-       if hostname and port %}
-  timeout server {{ slave_instance['request-timeout'] }}s
+{%-         if len(info_dict['health-check-netloc-list']) > 0 %}
+{%-           set counter = {'count': 1} %}
+{%-           for netloc in info_dict['health-check-netloc-list'] %}
+  server {{ slave_instance['slave_reference'] }}-backend-{{ scheme }}-{{ counter['count'] }} {{ netloc }} {{ ' '.join(ssl_list) }}
+{%-             do counter.__setitem__('count', counter['count'] + 1) %}
+{%-           endfor %}
+{%-         else %}
+  server {{ slave_instance['slave_reference'] }}-backend-{{ scheme }} {{ hostname }}:{{ port }} {{ ' '.join(ssl_list) }}
+{%-         endif %}
   timeout connect {{ slave_instance['backend-connect-timeout'] }}s
+  timeout server {{ slave_instance['request-timeout'] }}s
   retries {{ slave_instance['backend-connect-retries'] }}
-  server {{ slave_instance['slave_reference'] }}-backend-{{ scheme }} {{ hostname }}:{{ port }} {{ ' '.join(ssl_list) }}
 {%-         if path %}
   http-request set-path {{ path }}%[path]
 {%-         endif %}

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-CADDY.txt

@@ -73,6 +73,9 @@ T-2/var/log/httpd/_enable_cache_server_alias_error_log
 T-2/var/log/httpd/_https-only_access_log
 T-2/var/log/httpd/_https-only_backend_log
 T-2/var/log/httpd/_https-only_error_log
+T-2/var/log/httpd/_https-url-netloc-list_access_log
+T-2/var/log/httpd/_https-url-netloc-list_backend_log
+T-2/var/log/httpd/_https-url-netloc-list_error_log
 T-2/var/log/httpd/_monitor-ipv4-test_access_log
 T-2/var/log/httpd/_monitor-ipv4-test_error_log
 T-2/var/log/httpd/_monitor-ipv6-test_access_log
@@ -156,6 +159,9 @@ T-2/var/log/httpd/_type-zope-virtualhostroot-https-port_error_log
 T-2/var/log/httpd/_type-zope_access_log
 T-2/var/log/httpd/_type-zope_backend_log
 T-2/var/log/httpd/_type-zope_error_log
+T-2/var/log/httpd/_url-netloc-list_access_log
+T-2/var/log/httpd/_url-netloc-list_backend_log
+T-2/var/log/httpd/_url-netloc-list_error_log
 T-2/var/log/httpd/_url_https-url_access_log
 T-2/var/log/httpd/_url_https-url_backend_log
 T-2/var/log/httpd/_url_https-url_error_log

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -73,6 +73,9 @@ T-2/var/log/httpd/_enable_cache_server_alias_error_log
 T-2/var/log/httpd/_https-only_access_log
 T-2/var/log/httpd/_https-only_backend_log
 T-2/var/log/httpd/_https-only_error_log
+T-2/var/log/httpd/_https-url-netloc-list_access_log
+T-2/var/log/httpd/_https-url-netloc-list_backend_log
+T-2/var/log/httpd/_https-url-netloc-list_error_log
 T-2/var/log/httpd/_monitor-ipv4-test_access_log
 T-2/var/log/httpd/_monitor-ipv4-test_error_log
 T-2/var/log/httpd/_monitor-ipv6-test_access_log
@@ -156,6 +159,9 @@ T-2/var/log/httpd/_type-zope-virtualhostroot-https-port_error_log
 T-2/var/log/httpd/_type-zope_access_log
 T-2/var/log/httpd/_type-zope_backend_log
 T-2/var/log/httpd/_type-zope_error_log
+T-2/var/log/httpd/_url-netloc-list_access_log
+T-2/var/log/httpd/_url-netloc-list_backend_log
+T-2/var/log/httpd/_url-netloc-list_error_log
 T-2/var/log/httpd/_url_https-url_access_log
 T-2/var/log/httpd/_url_https-url_backend_log
 T-2/var/log/httpd/_url_https-url_error_log

software/caddy-frontend/test/test_data/test.TestSlaveHealthCheck.test_file_list_log-CADDY.txt

@@ -24,6 +24,9 @@ T-2/var/log/httpd/_health-check-disabled_error_log
 T-2/var/log/httpd/_health-check-failover-url-auth-to-backend_access_log
 T-2/var/log/httpd/_health-check-failover-url-auth-to-backend_backend_log
 T-2/var/log/httpd/_health-check-failover-url-auth-to-backend_error_log
+T-2/var/log/httpd/_health-check-failover-url-netloc-list_access_log
+T-2/var/log/httpd/_health-check-failover-url-netloc-list_backend_log
+T-2/var/log/httpd/_health-check-failover-url-netloc-list_error_log
 T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verified_access_log
 T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verified_backend_log
 T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verified_error_log