caddy-frontend: Improve generated files

Features:

 * amend configuration with comments
 * drop obsolete comments from Apache copy
 * remove not needed whitespaces
 * use indentation for conditionals in Jinja2

software/caddy-frontend/TODO.rst

@@ -15,7 +15,6 @@ Generally things to be done with ``caddy-frontend``:
  * ``ssl_proxy_ca_crt`` for ``ssl_proxy_verify``, this is related to bug https://github.com/mholt/caddy/issues/1550, proposed solution `just adding your CA to the system's trust store`
  * ``check-error-on-caddy-log`` like ``check-error-on-apache-log``
  * cover test suite like resilient tests for KVM and prove it works the same way as Caddy
- * make beautiful (eg. with whitespaces and nice comments) generated files (mostly Jinja2)
  * have ``caddy-frontend`` specific parameters, with backward compatibility to ``apache-frontend`` ones (like ``apache_custom_http`` --> ``caddy_custom_http``)
  * change ``switch-softwaretype`` to way how ``software/erp5`` does, which will help with dropping jinja2 template for ``caddy-wrapper``, which is workaround for current situation https://lab.nexedi.com/nexedi/slapos/merge_requests/312#note_62678
  * use `slapos!326 <https://lab.nexedi.com/nexedi/slapos/merge_requests/326>`_ instead of self-developed graceful restart scripts

software/caddy-frontend/buildout.hash.cfg

@@ -38,7 +38,7 @@ md5sum = 8d318af17da5631d4242c0d6d1531066
 
 [template-caddy-frontend-configuration]
 filename = templates/Caddyfile.in
-md5sum = 924d3bb528f590916552534934c604a2
+md5sum = 9404959e500a868aab1a217503117047
 
 [template-custom-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
@@ -50,11 +50,11 @@ md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
 
 [template-default-slave-virtualhost]
 filename = templates/default-virtualhost.conf.in
-md5sum = b524304177e7854232aa43bed98ddbfd
+md5sum = fa7dc8481f0c3066045c1dd5a8a3191a
 
 [template-cached-slave-virtualhost]
 filename = templates/cached-virtualhost.conf.in
-md5sum = 5aab4c15189a39837f56d4f442b233c6
+md5sum = bfcc2bcfe9151b9d3f25c4616e2c4f4f
 
 [template-log-access]
 filename = templates/template-log-access.conf.in
@@ -82,7 +82,7 @@ md5sum = 117238225b3fc3c5b5be381815f44c67
 
 [template-nginx-configuration]
 filename = templates/nginx.cfg.in
-md5sum = b1d6bac767db77ad1662edd06aabdf49
+md5sum = fadb2fcaf0f2b4fe735617fac222f7ed
 
 [template-nginx-eventsource-slave-virtualhost]
 filename = templates/nginx-eventsource-slave.conf.in
@@ -90,7 +90,7 @@ md5sum = 69d65e461cd7cd5ef5b1ccd0098b50c8
 
 [template-nginx-notebook-slave-virtualhost]
 filename = templates/nginx-notebook-slave.conf.in
-md5sum = 753e87647d1ed4655432393bba062d3f
+md5sum = b97ec5b84d5e0d3a76871c15b5bcce2e
 
 [template-apache-lazy-script-call]
 filename = templates/apache-lazy-script-call.sh.in

software/caddy-frontend/templates/Caddyfile.in

-# Automatically generated
+# Main caddy configuration file
 
 import {{frontend_configuration.get('log-access-configuration')}}
 import {{ slave_configuration_directory }}/*.conf
 import {{ slave_with_cache_configuration_directory }}/*.conf
 
+# Catch-all and 404 for not configured instances
 :{{ https_port }} {
   tls {{ login_certificate }} {{ login_key }}
   bind {{ local_ipv4 }}
@@ -14,6 +15,16 @@ import {{ slave_with_cache_configuration_directory }}/*.conf
   }
 }
 
+:{{ http_port }} {
+  bind {{ local_ipv4 }}
+  status 404 /
+  log / {{ access_log }} "{remote} {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
+  errors {{ error_log }} {
+    * {{ not_found_file }}
+  }
+}
+
+# Access to server-status Caddy-style
 https://[{{ global_ipv6 }}]:{{ https_port }}/server-status, https://{{ local_ipv4 }}:{{ https_port }}/server-status {
   tls {{ login_certificate }} {{ login_key }}
   bind {{ local_ipv4 }}
@@ -28,12 +39,3 @@ https://[{{ global_ipv6 }}]:{{ https_port }}/server-status, https://{{ local_ipv
     * {{ not_found_file }}
   }
 }
-
-:{{ http_port }} {
-  bind {{ local_ipv4 }}
-  status 404 /
-  log / {{ access_log }} "{remote} {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
-  errors {{ error_log }} {
-    * {{ not_found_file }}
-  }
-}

software/caddy-frontend/templates/cached-virtualhost.conf.in

-{% set TRUE_VALUES = ['y', 'yes', '1', 'true'] %}
-{% set server_alias_list =  slave_parameter.get('server-alias', '').split() %}
-{% set ssl_proxy_verify = ('' ~ slave_parameter.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES %}
-{% set host_list = [slave_parameter.get('custom_domain')] + server_alias_list %}
-{% set http_backend_host_list = [] %}
-{% set https_backend_host_list = [] %}
-{% for host in host_list %}
-{%   do http_backend_host_list.append('http://%s:%s' % (host, cached_port)) %}
-{%   do https_backend_host_list.append('http://%s:%s' % (host, ssl_cached_port)) %}
-{% endfor %}
+{%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] %}
+{%- set server_alias_list =  slave_parameter.get('server-alias', '').split() %}
+{%- set ssl_proxy_verify = ('' ~ slave_parameter.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES %}
+{%- set host_list = [slave_parameter.get('custom_domain')] + server_alias_list %}
+{%- set http_backend_host_list = [] %}
+{%- set https_backend_host_list = [] %}
+{%- for host in host_list %}
+{%-   do http_backend_host_list.append('http://%s:%s' % (host, cached_port)) %}
+{%-   do https_backend_host_list.append('http://%s:%s' % (host, ssl_cached_port)) %}
+{%- endfor %}
 
-# Only accept generic (i.e not Zope) backends on http
+# SSL-disabled backends
 {{ http_backend_host_list|join(', ') }} {
   bind {{ local_ipv4 }}
-{% if ssl_proxy_verify and 'ssl_proxy_ca_crt' in slave_parameter %}
+{%- if ssl_proxy_verify and 'ssl_proxy_ca_crt' in slave_parameter %}
     status 501 /
-{% endif %}
+{%- endif %}
 # Rewrite part
   proxy / {{ slave_parameter.get('backend_url', '') }} {
     # As backend is trusting REMOTE_USER header unset it always
@@ -22,30 +22,31 @@
 
     transparent
     timeout 600s
-{% if ssl_proxy_verify %}
-{%   if 'ssl_proxy_ca_crt' in slave_parameter %}
-{%   endif %}
-{% else %}
+{%- if ssl_proxy_verify %}
+{%-   if 'ssl_proxy_ca_crt' in slave_parameter %}
+{%-   endif %}
+{%- else %}
     insecure_skip_verify
-{% endif %}
+{%- endif %}
   }
 }
 
+# SSL-enabled backends
 {{ https_backend_host_list|join(', ') }} {
   bind {{ local_ipv4 }}
-{% if ssl_proxy_verify and 'ssl_proxy_ca_crt' in slave_parameter %}
+{%- if ssl_proxy_verify and 'ssl_proxy_ca_crt' in slave_parameter %}
     status 501 /
-{% endif %}
+{%- endif %}
   proxy / {{ slave_parameter.get('https_backend_url', '') }} {
     # As backend is trusting REMOTE_USER header unset it always
     header_upstream -REMOTE_USER
     transparent
     timeout 600s
-{% if ssl_proxy_verify %}
-{%   if 'ssl_proxy_ca_crt' in slave_parameter %}
-{%   endif %}
-{% else %}
+{%- if ssl_proxy_verify %}
+{%-   if 'ssl_proxy_ca_crt' in slave_parameter %}
+{%-   endif %}
+{%- else %}
     insecure_skip_verify
-{% endif %}
+{%- endif %}
   }
 }

software/caddy-frontend/templates/default-virtualhost.conf.in

-{% set TRUE_VALUES = ['y', 'yes', '1', 'true'] %}
-{% set disable_no_cache_header = ('' ~ slave_parameter.get('disable-no-cache-request', '')).lower() in TRUE_VALUES %}
-{% set disable_via_header = ('' ~ slave_parameter.get('disable-via-header', '')).lower() in TRUE_VALUES %}
-{% set prefer_gzip = ('' ~ slave_parameter.get('prefer-gzip-encoding-to-backend', '')).lower() in TRUE_VALUES %}
-{% set server_alias_list =  slave_parameter.get('server-alias', '').split() %}
-{% set enable_h2 = ('' ~ slave_parameter.get('enable-http2', slave_parameter['enable_http2_by_default'])).lower() in TRUE_VALUES %}
-{% set ssl_proxy_verify = ('' ~ slave_parameter.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES %}
-{% set disabled_cookie_list =  slave_parameter.get('disabled-cookie-list', '').split() %}
-{% set https_only = ('' ~ slave_parameter.get('https-only', '')).lower() in TRUE_VALUES %}
-{% set slave_type = slave_parameter.get('type', '') %}
-{% set host_list = [slave_parameter.get('custom_domain')] + server_alias_list %}
-{% set backend_url = slave_parameter.get('https-url', slave_parameter.get('url', '')) %}
-{% set http_host_list = [] %}
-{% set https_host_list = [] %}
-{% for host in host_list %}
-{%   do http_host_list.append('http://%s:%s' % (host, http_port)) %}
-{%   do https_host_list.append('https://%s:%s' % (host, https_port)) %}
-{% endfor %}
+{%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] %}
+{%- set disable_no_cache_header = ('' ~ slave_parameter.get('disable-no-cache-request', '')).lower() in TRUE_VALUES %}
+{%- set disable_via_header = ('' ~ slave_parameter.get('disable-via-header', '')).lower() in TRUE_VALUES %}
+{%- set prefer_gzip = ('' ~ slave_parameter.get('prefer-gzip-encoding-to-backend', '')).lower() in TRUE_VALUES %}
+{%- set server_alias_list =  slave_parameter.get('server-alias', '').split() %}
+{%- set enable_h2 = ('' ~ slave_parameter.get('enable-http2', slave_parameter['enable_http2_by_default'])).lower() in TRUE_VALUES %}
+{%- set ssl_proxy_verify = ('' ~ slave_parameter.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES %}
+{%- set disabled_cookie_list =  slave_parameter.get('disabled-cookie-list', '').split() %}
+{%- set https_only = ('' ~ slave_parameter.get('https-only', '')).lower() in TRUE_VALUES %}
+{%- set slave_type = slave_parameter.get('type', '') %}
+{%- set host_list = [slave_parameter.get('custom_domain')] + server_alias_list %}
+{%- set backend_url = slave_parameter.get('https-url', slave_parameter.get('url', '')) %}
+{%- set http_host_list = [] %}
+{%- set https_host_list = [] %}
+{%- for host in host_list %}
+{%-   do http_host_list.append('http://%s:%s' % (host, http_port)) %}
+{%-   do https_host_list.append('https://%s:%s' % (host, https_port)) %}
+{%- endfor %}
+
+# SSL enabled hosts
 {{ https_host_list|join(', ') }} {
   bind {{ local_ipv4 }}
-{% if ssl_proxy_verify and 'ssl_proxy_ca_crt' in slave_parameter %}
+{%- if ssl_proxy_verify and 'ssl_proxy_ca_crt' in slave_parameter %}
     status 501 /
-{% endif %}
+{%- endif %}
   tls {{ slave_parameter.get('path_to_ssl_crt', slave_parameter.get('login_certificate')) }} {{ slave_parameter.get('path_to_ssl_key', slave_parameter.get('login_key')) }} {
-{% if slave_parameter.get('path_to_ssl_ca_crt') %}
+{%- if slave_parameter.get('path_to_ssl_ca_crt') %}
+    # Configuration of accepted clients
     clients {{ slave_parameter.get('path_to_ssl_ca_crt') }}
-{% endif %}
-{% if enable_h2 %}
+{%- endif %}
+{%- if enable_h2 %}
+    # Allow HTTP2
     alpn h2 http/1.1
-{% else %}
+{%- else %}
+    # Disallow HTTP2
     alpn http/1.1
-{% endif %}
+{%- endif %}
   }
   log / {{ slave_parameter.get('access_log') }} "{remote} {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
   errors {{ slave_parameter.get('error_log') }}
 
-{% for disabled_cookie in disabled_cookie_list %}
-{% endfor %}
+{%- for disabled_cookie in disabled_cookie_list %}
+{%- endfor %}
 
-{% if prefer_gzip %}
-{% endif %}
+{%- if prefer_gzip %}
+{%- endif %}
 
-{% if slave_type ==  'zope' and backend_url %}
+{%- if slave_type ==  'zope' and backend_url %}
+  # Zope configuration
   proxy / {{ backend_url }} {
     # As backend is trusting REMOTE_USER header unset it always
     header_upstream -REMOTE_USER
 
-{% if disable_via_header %}
+{%-   if disable_via_header %}
     header_downstream -Via
-{% endif %}
+{%-   endif %}
 
-{% if disable_no_cache_header %}
+{%-   if disable_no_cache_header %}
     header_upstream -Cache-Control
     header_upstream -Pragma
-{% endif %}
+{%-   endif %}
     transparent
     timeout 600s
-{% if ssl_proxy_verify %}
-{%   if 'ssl_proxy_ca_crt' in slave_parameter %}
-{%   endif %}
-{% else %}
+{%-   if ssl_proxy_verify %}
+{%-     if 'ssl_proxy_ca_crt' in slave_parameter %}
+{%-     endif %}
+{%-   else %}
     insecure_skip_verify
-{% endif %}
+{%-   endif %}
   }
-  {% if 'default-path' in slave_parameter %}
+  {%- if 'default-path' in slave_parameter %}
   redir 301 {
     if {path} is /
     / {scheme}://{host}/{{ slave_parameter.get('default-path') }}
   }
-  {% endif %}
+  {%- endif %}
   rewrite {
     regexp (.*)
     to /VirtualHostBase/{scheme}%2F%2F{hostonly}:{{ slave_parameter.get('virtualhostroot-https-port', '443') }}%2F{{ slave_parameter.get('path', '') }}%2FVirtualHostRoot/{1}
   }
-{% elif slave_type ==  'redirect' and backend_url %}
+{%- elif slave_type ==  'redirect' and backend_url %}
+  # Redirect configuration
   redir 302 {
     /  {{ backend_url }}{uri}
   }
-{% else %}
-  {% if 'default-path' in slave_parameter %}
+{%- else %}
+  # Default configuration
+{%-   if 'default-path' in slave_parameter %}
   redir 301 {
     if {path} is /
     / {scheme}://{host}/{{ slave_parameter.get('default-path') }}
   }
-  {% endif %}
-  {% if backend_url %}
+{%-   endif %}
+{%-   if backend_url %}
 
   proxy / {{ backend_url }} {
     # As backend is trusting REMOTE_USER header unset it always
     header_upstream -REMOTE_USER
 
-{% if disable_via_header %}
+{%-     if disable_via_header %}
     header_downstream -Via
-{% endif %}
+{%-     endif %}
 
-{% if disable_no_cache_header %}
+{%-     if disable_no_cache_header %}
     header_upstream -Cache-Control
     header_upstream -Pragma
-{% endif %}
+{%-     endif %}
     transparent
     timeout 600s
-{% if ssl_proxy_verify %}
-{%   if 'ssl_proxy_ca_crt' in slave_parameter %}
-{%   endif %}
-{% else %}
+{%-     if ssl_proxy_verify %}
+{%-       if 'ssl_proxy_ca_crt' in slave_parameter %}
+{%-       endif %}
+{%-     else %}
     insecure_skip_verify
-{% endif %}
+{%-     endif %}
   }
-  {%   endif %}
-{% endif %}
+{%-   endif %}
+{%- endif %}
 }
 
+# SSL-disabled hosts
 {{ http_host_list|join(', ') }} {
   bind {{ local_ipv4 }}
-{% if ssl_proxy_verify and 'ssl_proxy_ca_crt' in slave_parameter %}
+{%- if ssl_proxy_verify and 'ssl_proxy_ca_crt' in slave_parameter %}
     status 501 /
-{% endif %}
+{%- endif %}
 
   log / {{ slave_parameter.get('access_log') }} "{remote} {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
   errors {{ slave_parameter.get('error_log') }}
 
-{% for disabled_cookie in disabled_cookie_list %}
-{% endfor %}
+{%- for disabled_cookie in disabled_cookie_list %}
+{%- endfor %}
 
-{% if prefer_gzip %}
-{% endif %}
+{%- if prefer_gzip %}
+{%- endif %}
 
-{% if https_only %}
+{%- if https_only %}
+  # Enforced redirection to SSL-enabled host
   redir / https://{host}{uri}
-{% elif slave_type ==  'redirect' and slave_parameter.get('url', '') %}
+{%- elif slave_type ==  'redirect' and slave_parameter.get('url', '') %}
+  # Redirect configuration
   redir 302 {
     /  {{ slave_parameter.get('url', '') }}{uri}
   }
-{% elif slave_type ==  'zope' and backend_url %}
+{%- elif slave_type ==  'zope' and backend_url %}
+  # Zope configuration
   proxy / {{ backend_url }} {
     # As backend is trusting REMOTE_USER header unset it always
     header_upstream -REMOTE_USER
 
-{% if disable_via_header %}
+{%-   if disable_via_header %}
     header_downstream -Via
-{% endif %}
+{%-   endif %}
 
-{% if disable_no_cache_header %}
+{%-   if disable_no_cache_header %}
     header_upstream -Cache-Control
     header_upstream -Pragma
-{% endif %}
+{%-   endif %}
     transparent
     timeout 600s
-{% if ssl_proxy_verify %}
-{%   if 'ssl_proxy_ca_crt' in slave_parameter %}
-{%   endif %}
-{% else %}
+{%-   if ssl_proxy_verify %}
+{%-     if 'ssl_proxy_ca_crt' in slave_parameter %}
+{%-     endif %}
+{%-   else %}
     insecure_skip_verify
-{% endif %}
+{%-   endif %}
   }
-  {% if 'default-path' in slave_parameter %}
+{%-   if 'default-path' in slave_parameter %}
   redir 301 {
     if {path} is /
     / {scheme}://{host}/{{ slave_parameter.get('default-path') }}
   }
-  {% endif %}
+{%-   endif %}
   rewrite {
     regexp (.*)
     to /VirtualHostBase/{scheme}%2F%2F{hostonly}:{{ slave_parameter.get('virtualhostroot-http-port', '80') }}%2F{{ slave_parameter.get('path', '') }}%2FVirtualHostRoot/{1}
   }
-{% else %}
-  {% if 'default-path' in slave_parameter %}
+{%- else %}
+  # Default configuration
+{%-   if 'default-path' in slave_parameter %}
   redir 301 {
     if {path} is /
     / {scheme}://{host}/{{ slave_parameter.get('default-path') }}
   }
-  {% endif %}
-  {% if slave_parameter.get('url', '') %}
+{%-   endif %}
+{%-   if slave_parameter.get('url', '') %}
   proxy / {{ slave_parameter.get('url', '') }} {
     # As backend is trusting REMOTE_USER header unset it always
     header_upstream -REMOTE_USER
 
-{% if disable_via_header %}
+{%-     if disable_via_header %}
     header_downstream -Via
-{% endif %}
+{%-     endif %}
 
-{% if disable_no_cache_header %}
+{%-     if disable_no_cache_header %}
     header_upstream -Cache-Control
     header_upstream -Pragma
-{% endif %}
+{%-     endif %}
     transparent
     timeout 600s
-{% if ssl_proxy_verify %}
-{%   if 'ssl_proxy_ca_crt' in slave_parameter %}
-{%   endif %}
-{% else %}
+{%-     if ssl_proxy_verify %}
+{%-       if 'ssl_proxy_ca_crt' in slave_parameter %}
+{%-       endif %}
+{%-     else %}
     insecure_skip_verify
-{% endif %}
+{%-     endif %}
   }
-{% endif %}
-{% endif %}
-  # If nothing exist : put a nice error
-#  ErrorDocument 404 /notfound.html
-# Dadiboom
+{%-   endif %}
+{%- endif %}
 }

software/caddy-frontend/templates/nginx-notebook-slave.conf.in

-{% set url = slave_parameter.get('url') %}
-{% set https_url = slave_parameter.get('https-url', url) %}
-{% if url.startswith("http://") or url.startswith("https://") %}
-{%   set upstream = url.split("/")[2] %}
-{%   set https_upstream = https_url.split("/")[2] %}
+{%- set url = slave_parameter.get('url') %}
+{%- set https_url = slave_parameter.get('https-url', url) %}
+{%- if url.startswith("http://") or url.startswith("https://") %}
+{%-   set upstream = url.split("/")[2] %}
+{%-   set https_upstream = https_url.split("/")[2] %}
 
+# SSL-enabled
 https://{{ slave_parameter.get('custom_domain') }}:{{ nginx_https_port }} {
   bind {{ local_ipv4 }}
   log / {{ slave_parameter.get('access_log') }} "{remote} {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
   errors {{ slave_parameter.get('error_log') }}
 
   tls {{ slave_parameter.get('path_to_ssl_crt', slave_parameter.get('login_certificate')) }} {{ slave_parameter.get('path_to_ssl_key', slave_parameter.get('login_key')) }} {
-{% if slave_parameter.get('path_to_ssl_ca_crt') %}
+{%-   if slave_parameter.get('path_to_ssl_ca_crt') %}
     clients {{ slave_parameter.get('path_to_ssl_ca_crt') }}
-{% endif %}
+{%-   endif %}
     alpn http/1.1
   }
 
@@ -33,6 +34,7 @@ https://{{ slave_parameter.get('custom_domain') }}:{{ nginx_https_port }} {
   }
 }
 
+# SSL-disabled
 http://{{ slave_parameter.get('custom_domain') }}:{{ nginx_http_port }} {
   bind {{ local_ipv4 }}
   log / {{ slave_parameter.get('access_log') }} "{remote} {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
@@ -54,4 +56,4 @@ http://{{ slave_parameter.get('custom_domain') }}:{{ nginx_http_port }} {
     insecure_skip_verify
   }
 }
-{% endif %}
+{%- endif %}

software/caddy-frontend/templates/nginx.cfg.in

@@ -57,6 +57,7 @@
 
 import {{ slave_configuration_directory }}/*.conf
 
+# Catch-all and 404 for not configured instances
 :{{ port }} {
   tls {{ ssl_certificate }} {{ ssl_key }}
   bind {{ local_ip }}