caddy-frontend: Fix random 502 EOFs by adding try_duration

try_duration and try_interval are Caddy proxy's switches which allow to deal
with non working backend (https://caddyserver.com/docs/proxy)

The non working backend is the one, to which connection is lost or was not
possible to make, without sending any data.

The default try_duration=5s and try_interval=250ms are chosen, so that in
normal network conditions (with all possible problems in the network, like
lost packets) the browser will have to wait up to 5 seconds to be informed
that backend is inaccessible or for the request to start being processed,
but only a bit more than 250ms if Caddy would have to reestablish connection
to faulty backend.

In order to check it out it is advisable to setup a system, with real backend,
like apache one, and configure iptables to randomly reject packets to it:

  iptables -A INPUT -m statistic --mode random -p tcp --dport <backend_port> \
  --probability 0.05 -j REJECT --reject-with tcp-reset

Using ab or any other tool will results with lot of 502 EOF in the Caddy error
log and also reported by ab. With this configuration there are no more
errors visible to the client, which come from the problems on the network
between Caddy and the backend.

software/caddy-frontend/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg.in
-md5sum = de7e30546a952e306e2a74f8492ab419
+md5sum = 2747f9125c8dffa0c27b79a6902a55cb
 
 [template-common]
 filename = instance-common.cfg.in
@@ -22,7 +22,7 @@ md5sum = c801b7f9f11f0965677c22e6bbe9281b
 
 [template-apache-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = 2c8e1dd8df4a225b4a8d8f70688a85ca
+md5sum = da7b5804d50c727412885e8b818ca433
 
 [template-apache-replicate]
 filename = instance-apache-replicate.cfg.in
@@ -30,7 +30,7 @@ md5sum = 6a86edb96b171fbd0a59d0adc9cc906b
 
 [template-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = 4b06918875e889f850c5aca0d8c57796
+md5sum = d476a5d21d448bcad0ab48494b229e28
 
 [template-slave-configuration]
 filename = templates/custom-virtualhost.conf.in
@@ -58,11 +58,11 @@ md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
 
 [template-default-slave-virtualhost]
 filename = templates/default-virtualhost.conf.in
-md5sum = a0b5a3dbf7b1d6622a52173ef5a90e72
+md5sum = 26e634a37dd6ab17c75207d4fbbaa65a
 
 [template-cached-slave-virtualhost]
 filename = templates/cached-virtualhost.conf.in
-md5sum = c64f8ac7ec439460877ce5a5c5ccf1f7
+md5sum = 907372828d1ceb05c41240078196f439
 
 [template-log-access]
 filename = templates/template-log-access.conf.in
@@ -94,7 +94,7 @@ md5sum = 176cbca2070734a185a7ae5a4d1181c5
 
 [template-nginx-notebook-slave-virtualhost]
 filename = templates/nginx-notebook-slave.conf.in
-md5sum = 2b765db72191197122554df17ad471d1
+md5sum = ee3b5c23f1c81aa43ce7cd8f8e327f70
 
 [template-apache-lazy-script-call]
 filename = templates/apache-lazy-script-call.sh.in

software/caddy-frontend/instance-apache-frontend.cfg.in

@@ -202,6 +202,8 @@ extra-context =
     key login_ca_crt ca-custom-frontend:rendered
     key enable_http2_by_default configuration:enable-http2-by-default
     key global_disable_http2 configuration:global-disable-http2
+    key proxy_try_duration configuration:proxy-try-duration
+    key proxy_try_interval configuration:proxy-try-interval
     key access_log caddy-configuration:access-log
     key error_log caddy-configuration:error-log
     key sixtunnel_executable :sixtunnel_executable

software/caddy-frontend/instance-caddy-input-schema.json

@@ -86,6 +86,18 @@
       ],
       "title": "Enable QUIC",
       "type": "string"
+    },
+    "proxy-try-duration": {
+      "default": 5,
+      "description": "A time during which Caddy will try to establish connection with a backend. Setting it to 0 will result with immediate return of 502 EOF error to the browser, even if it would be possible to (re)connect to the backend during few moments. More info in https://caddyserver.com/docs/proxy try_durtion.",
+      "title": "Duration in seconds of trying a backend",
+      "type": "integer"
+    },
+    "proxy-try-interval": {
+      "default": 250,
+      "description": "How often Caddy will try to establish connection with a backend during proxy-try-duration. More info in https://caddyserver.com/docs/proxy try_interval",
+      "title": "Interval in milliseconds of tries during proxy-try-duration",
+      "type": "integer"
     }
   },
   "title": "Input Parameters",

software/caddy-frontend/instance.cfg.in

@@ -102,3 +102,5 @@ configuration.enable-quic = false
 configuration.mpm-graceful-shutdown-timeout = 5
 configuration.monitor-httpd-port = 8072
 configuration.frontend-name =
+configuration.proxy-try-duration = 5
+configuration.proxy-try-interval = 250

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -66,6 +66,10 @@ crl = {{ custom_ssl_directory }}/crl/
 {%   do slave_instance.__setitem__('enable_http2_by_default', enable_http2_by_default) %}
 {%   do slave_instance.__setitem__('global_disable_http2', global_disable_http2) %}
 
+{#   Pass proxy_try_duration and proxy_try_interval #}
+{%   do slave_instance.__setitem__('proxy_try_duration', proxy_try_duration) %}
+{%   do slave_instance.__setitem__('proxy_try_interval', proxy_try_interval) %}
+
 {#   Set Up log files #}
 {%   do slave_parameter_dict.__setitem__('access_log', '/'.join([caddy_log_directory, '%s_access_log' % slave_reference])) %}
 {%   do slave_parameter_dict.__setitem__('error_log', '/'.join([caddy_log_directory, '%s_error_log' % slave_reference])) %}

software/caddy-frontend/templates/cached-virtualhost.conf.in

@@ -21,6 +21,8 @@
   gzip
 # Rewrite part
   proxy / {{ slave_parameter.get('backend_url', '') }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
     # As backend is trusting REMOTE_USER header unset it always
     header_upstream -REMOTE_USER
 
@@ -42,6 +44,8 @@
   # Compress the output
   gzip
   proxy / {{ slave_parameter.get('https_backend_url', '') }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
     # As backend is trusting REMOTE_USER header unset it always
     header_upstream -REMOTE_USER
     transparent

software/caddy-frontend/templates/default-virtualhost.conf.in

@@ -67,6 +67,8 @@
 {%-   for (proxy_name, proxy_comment) in proxy_append_list %}
   # {{ proxy_comment }}
   proxy /{{ proxy_name }} {{ backend_url }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
 {%-     if proxy_name == 'prefer-gzip' %}
     without /prefer-gzip
     header_upstream Accept-Encoding gzip
@@ -138,6 +140,8 @@
 {%-   for (proxy_name, proxy_comment) in proxy_append_list %}
   # {{ proxy_comment }}
   proxy /{{ proxy_name }} {{ backend_url }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
 {%-     if proxy_name == 'prefer-gzip' %}
     without /prefer-gzip
     header_upstream Accept-Encoding gzip
@@ -216,6 +220,8 @@
 {%-   for (proxy_name, proxy_comment) in proxy_append_list %}
   # {{ proxy_comment }}
   proxy /{{ proxy_name }} {{ backend_url }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
 {%-     if proxy_name == 'prefer-gzip' %}
     without /prefer-gzip
     header_upstream Accept-Encoding gzip
@@ -281,6 +287,8 @@
 {%-   for (proxy_name, proxy_comment) in proxy_append_list %}
   # {{ proxy_comment }}
   proxy /{{ proxy_name }} {{ slave_parameter.get('url', '') }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
 {%-     if proxy_name == 'prefer-gzip' %}
     without /prefer-gzip
     header_upstream Accept-Encoding gzip

software/caddy-frontend/templates/nginx-notebook-slave.conf.in

@@ -17,6 +17,8 @@ https://{{ slave_parameter.get('custom_domain') }}:{{ slave_parameter['nginx_htt
   }
 
   proxy / {{ https_upstream }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
     transparent
     insecure_skip_verify
   }
@@ -25,6 +27,8 @@ https://{{ slave_parameter.get('custom_domain') }}:{{ slave_parameter['nginx_htt
     to /proxy/{1}
   }
   proxy /proxy/ {{ https_upstream }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
     header_upstream X-Real-IP {remote}
     header_upstream Host {host}
     websocket
@@ -42,6 +46,8 @@ http://{{ slave_parameter.get('custom_domain') }}:{{ slave_parameter['nginx_http
   errors {{ slave_parameter.get('error_log') }}
 
   proxy / {{ upstream }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
     transparent
     insecure_skip_verify
   }
@@ -50,6 +56,8 @@ http://{{ slave_parameter.get('custom_domain') }}:{{ slave_parameter['nginx_http
     to /proxy/{1}
   }
   proxy /proxy/ {{ upstream }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
     header_upstream X-Real-IP {remote}
     header_upstream Host {host}
     websocket