caddy-frontend: Cover failover URL with cache

Configuring failover URL and being able to rely on cached content is not so
simple, thus document the requirement and prove in test, that such
configuration works correctly.

software/caddy-frontend/README.caddy_frontend.rst

@@ -233,6 +233,8 @@ Please be aware that the `health-check-timeout` is really short by default, so i
 
 Thanks to using health-check it's possible to configure failover system. By providing `health-check-failover-url` or `health-check-failover-https-url` some special backend can be used to reply in case if original backend replies with error (codes like `5xx`). As a note one can setup this failover URL like `https://failover.example.com/?p=` so that the path from the incoming request will be passed as parameter. Additionally authentication to failover URL is supported with `health-check-authenticate-to-failover-backend` and SSL Proxy verification with `health-check-failover-ssl-proxy-verify` and `health-check-failover-ssl-proxy-ca-crt`.
 
+**Note**: It's important to correctly configure failover URL response, especially in case if it's expected to use `stale-if-error` simulation available while `enable_cache` is used. In order to serve pages from cache the failover URL have to return error HTTP code (like 503 SERVICE_UNAVAILABLE), so that in such case cached page will have precedence over the reply from failover URL.
+
 Examples
 ========
 

software/caddy-frontend/test/test_data/test.TestSlaveHealthCheck.test00cluster_request_instance_parameter_dict.txt

@@ -61,6 +61,7 @@
         "url": "http://@@_ipv4_address@@:@@_server_http_port@@/"
       },
       {
+        "enable_cache": true,
         "health-check": true,
         "health-check-failover-https-url": "http://@@_ipv4_address@@:@@_server_http_port@@/failover-https-url?a=b&c=",
         "health-check-failover-url": "http://@@_ipv4_address@@:@@_server_http_port@@/failover-url?a=b&c=",
@@ -183,6 +184,7 @@
           "url": "http://@@_ipv4_address@@:@@_server_http_port@@/"
         },
         {
+          "enable_cache": true,
           "health-check": true,
           "health-check-failover-https-url": "http://@@_ipv4_address@@:@@_server_http_port@@/failover-https-url?a=b&c=",
           "health-check-failover-url": "http://@@_ipv4_address@@:@@_server_http_port@@/failover-url?a=b&c=",
@@ -279,7 +281,7 @@
       "backend-client-caucase-url": "http://[@@_ipv6_address@@]:8990",
       "cluster-identification": "testing partition 0",
       "domain": "example.com",
-      "extra_slave_instance_list": "[{\"health-check\": true, \"health-check-http-method\": \"CONNECT\", \"slave_reference\": \"_health-check-connect\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"health-check\": true, \"health-check-fall\": \"7\", \"health-check-http-method\": \"POST\", \"health-check-http-path\": \"/POST-path to be encoded\", \"health-check-http-version\": \"HTTP/1.0\", \"health-check-interval\": \"15\", \"health-check-rise\": \"3\", \"health-check-timeout\": \"7\", \"slave_reference\": \"_health-check-custom\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"health-check\": true, \"slave_reference\": \"_health-check-default\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"slave_reference\": \"_health-check-disabled\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"health-check\": true, \"health-check-failover-https-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/failover-https-url?a=b&c=\", \"health-check-failover-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/failover-url?a=b&c=\", \"health-check-http-path\": \"/health-check-failover-url\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"https-only\": false, \"https-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/https-url\", \"slave_reference\": \"_health-check-failover-url\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/url\"}, {\"health-check\": true, \"health-check-authenticate-to-failover-backend\": true, \"health-check-failover-https-url\": \"https://@@_ipv4_address@@:@@_server_https_auth_port@@/failover-https-url?a=b&c=\", \"health-check-failover-url\": \"https://@@_ipv4_address@@:@@_server_https_auth_port@@/failover-url?a=b&c=\", \"health-check-http-path\": \"/health-check-failover-url-auth-to-backend\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"https-only\": false, \"https-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/https-url\", \"slave_reference\": \"_health-check-failover-url-auth-to-backend\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/url\"}, {\"health-check\": true, \"health-check-failover-https-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/failover-https-url?a=b&c=\", \"health-check-failover-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/failover-url?a=b&c=\", \"health-check-failover-url-netloc-list\": \"@@_ipv4_address@@:@@_server_netloc_a_http_port@@ @@_ipv4_address@@:@@_server_netloc_b_http_port@@\", \"health-check-http-path\": \"/health-check-failover-url\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"https-only\": false, \"https-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/https-url\", \"slave_reference\": \"_health-check-failover-url-netloc-list\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/url\"}, {\"health-check\": true, \"health-check-failover-ssl-proxy-ca-crt\": \"@@test_server_ca.certificate_pem_double@@\", \"health-check-failover-ssl-proxy-verify\": true, \"health-check-failover-url\": \"https://@@_ipv4_address@@:@@_server_https_port@@/\", \"health-check-http-path\": \"/health-check-failover-url-ssl-proxy-verified\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"slave_reference\": \"_health-check-failover-url-ssl-proxy-verified\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"health-check\": true, \"health-check-failover-ssl-proxy-verify\": true, \"health-check-failover-url\": \"https://@@_ipv4_address@@:@@_server_https_port@@/\", \"health-check-http-path\": \"/health-check-failover-url-ssl-proxy-verify-missing\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"slave_reference\": \"_health-check-failover-url-ssl-proxy-verify-missing\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"health-check\": true, \"health-check-failover-ssl-proxy-ca-crt\": \"@@another_server_ca.certificate_pem_double@@\", \"health-check-failover-ssl-proxy-verify\": true, \"health-check-failover-url\": \"https://@@_ipv4_address@@:@@_server_https_port@@/\", \"health-check-http-path\": \"/health-check-failover-url-ssl-proxy-verify-unverified\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"slave_reference\": \"_health-check-failover-url-ssl-proxy-verify-unverified\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}]",
+      "extra_slave_instance_list": "[{\"health-check\": true, \"health-check-http-method\": \"CONNECT\", \"slave_reference\": \"_health-check-connect\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"health-check\": true, \"health-check-fall\": \"7\", \"health-check-http-method\": \"POST\", \"health-check-http-path\": \"/POST-path to be encoded\", \"health-check-http-version\": \"HTTP/1.0\", \"health-check-interval\": \"15\", \"health-check-rise\": \"3\", \"health-check-timeout\": \"7\", \"slave_reference\": \"_health-check-custom\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"health-check\": true, \"slave_reference\": \"_health-check-default\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"slave_reference\": \"_health-check-disabled\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"enable_cache\": true, \"health-check\": true, \"health-check-failover-https-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/failover-https-url?a=b&c=\", \"health-check-failover-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/failover-url?a=b&c=\", \"health-check-http-path\": \"/health-check-failover-url\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"https-only\": false, \"https-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/https-url\", \"slave_reference\": \"_health-check-failover-url\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/url\"}, {\"health-check\": true, \"health-check-authenticate-to-failover-backend\": true, \"health-check-failover-https-url\": \"https://@@_ipv4_address@@:@@_server_https_auth_port@@/failover-https-url?a=b&c=\", \"health-check-failover-url\": \"https://@@_ipv4_address@@:@@_server_https_auth_port@@/failover-url?a=b&c=\", \"health-check-http-path\": \"/health-check-failover-url-auth-to-backend\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"https-only\": false, \"https-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/https-url\", \"slave_reference\": \"_health-check-failover-url-auth-to-backend\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/url\"}, {\"health-check\": true, \"health-check-failover-https-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/failover-https-url?a=b&c=\", \"health-check-failover-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/failover-url?a=b&c=\", \"health-check-failover-url-netloc-list\": \"@@_ipv4_address@@:@@_server_netloc_a_http_port@@ @@_ipv4_address@@:@@_server_netloc_b_http_port@@\", \"health-check-http-path\": \"/health-check-failover-url\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"https-only\": false, \"https-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/https-url\", \"slave_reference\": \"_health-check-failover-url-netloc-list\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/url\"}, {\"health-check\": true, \"health-check-failover-ssl-proxy-ca-crt\": \"@@test_server_ca.certificate_pem_double@@\", \"health-check-failover-ssl-proxy-verify\": true, \"health-check-failover-url\": \"https://@@_ipv4_address@@:@@_server_https_port@@/\", \"health-check-http-path\": \"/health-check-failover-url-ssl-proxy-verified\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"slave_reference\": \"_health-check-failover-url-ssl-proxy-verified\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"health-check\": true, \"health-check-failover-ssl-proxy-verify\": true, \"health-check-failover-url\": \"https://@@_ipv4_address@@:@@_server_https_port@@/\", \"health-check-http-path\": \"/health-check-failover-url-ssl-proxy-verify-missing\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"slave_reference\": \"_health-check-failover-url-ssl-proxy-verify-missing\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"health-check\": true, \"health-check-failover-ssl-proxy-ca-crt\": \"@@another_server_ca.certificate_pem_double@@\", \"health-check-failover-ssl-proxy-verify\": true, \"health-check-failover-url\": \"https://@@_ipv4_address@@:@@_server_https_port@@/\", \"health-check-http-path\": \"/health-check-failover-url-ssl-proxy-verify-unverified\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"slave_reference\": \"_health-check-failover-url-ssl-proxy-verify-unverified\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}]",
       "frontend-name": "caddy-frontend-1",
       "kedifa-caucase-url": "http://[@@_ipv6_address@@]:15090",
       "master-key-download-url": "https://[@@_ipv6_address@@]:15080/@@master-key-download-url_endpoint@@",