caddy-frontend: Log direct access to backend for cached slaves

software/caddy-frontend/buildout.hash.cfg

@@ -22,7 +22,7 @@ md5sum = c801b7f9f11f0965677c22e6bbe9281b
 
 [template-apache-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = d2dda14597c061960f0b9d38ec0d8313
+md5sum = 72e8ff0773fd0325dcbe994786156570
 
 [template-apache-replicate]
 filename = instance-apache-replicate.cfg.in
@@ -30,7 +30,7 @@ md5sum = ef06c04a5aa33b103dc1d25d0dfe8217
 
 [template-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = 0dc409d7d3226a6727ec340ebf79f8da
+md5sum = 28e4da4b27a42bf8b8d261d45b3dfa58
 
 [template-slave-configuration]
 filename = templates/custom-virtualhost.conf.in
@@ -58,7 +58,7 @@ md5sum = 0c5ef7f26a142c3ab53e835d2caa698d
 
 [template-cached-slave-virtualhost]
 filename = templates/cached-virtualhost.conf.in
-md5sum = 907372828d1ceb05c41240078196f439
+md5sum = 6ca9a3251830d602cf25e0a0389fc74b
 
 [template-log-access]
 filename = templates/template-log-access.conf.in

software/caddy-frontend/instance-apache-frontend.cfg.in

@@ -244,6 +244,7 @@ extra-context =
     key autocert caddy-directory:autocert
     key master_certificate caddy-configuration:master-certificate
     key caddy_log_directory caddy-directory:slave-log
+    key caddy_log_cache_direct_directory caddy-directory:slave-log-cache-direct
     key local_ipv4 :local_ipv4
     key local_ipv6 :local_ipv6
     key global_ipv6 slap-network-information:global-ipv6
@@ -361,6 +362,7 @@ slave-with-cache-configuration = ${directory:etc}/caddy-slave-with-cache-conf.d/
 cache = ${directory:var}/cache
 mod-ssl = ${:cache}/httpd_mod_ssl
 slave-log = ${directory:log}/httpd
+slave-log-cache-direct = ${directory:log}/httpd-cache-direct
 autocert = ${directory:srv}/autocert
 master-autocert-dir = ${:autocert}/master-autocert
 custom-ssl-directory = ${:slave-configuration}/ssl

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -50,8 +50,13 @@ content =
   from slapos.promise.plugin.${:module} import RunPromise
 output = {{ plugin_directory }}/${:name}
 
+# empty section if no cached slaves are available
+[slave-log-cache-direct-directory-dict]
+
 {# Loop thought slave list to set up slaves #}
 {% for slave_instance in slave_instance_list %}
+{%   set slave_type = slave_instance.get('type', '') %}
+{%   set enable_cache = (('' ~ slave_instance.get('enable_cache', '')).lower() in TRUE_VALUES and slave_type != 'redirect') %}
 {%   set slave_reference = slave_instance.get('slave_reference') %}
 {%   set slave_kedifa = slave_kedifa_information.get(slave_reference) %}
 {%   if slave_kedifa %}
@@ -59,7 +64,6 @@ output = {{ plugin_directory }}/${:name}
 {%   else %}
 {%     set key_download_url = 'notreadyyet' %}
 {%   endif %}
-{%   set slave_type = slave_instance.get('type', '') %}
 {%   set slave_section_title = 'dynamic-template-slave-instance-%s' % slave_reference %}
 {%   set slave_parameter_dict = generic_instance_parameter_dict.copy() %}
 {%   set slave_publish_dict = {} %}
@@ -73,6 +77,9 @@ output = {{ plugin_directory }}/${:name}
 {%   do part_list.extend([slave_logrotate_section, slave_section_title]) %}
 
 {%   set slave_log_folder = '${logrotate-directory:logrotate-backup}/' + slave_reference + "-logs" %}
+{%   if enable_cache %}
+{%     set slave_log_cache_direct_folder = '${logrotate-directory:logrotate-backup}/' + slave_reference + "-cache-direct-logs" %}
+{%   endif %}
 
 {#   Pass HTTP2 switch #}
 {%   do slave_instance.__setitem__('enable_http2_by_default', enable_http2_by_default) %}
@@ -87,6 +94,12 @@ output = {{ plugin_directory }}/${:name}
 {%   do slave_parameter_dict.__setitem__('error_log', '/'.join([caddy_log_directory, '%s_error_log' % slave_reference])) %}
 {%   do slave_instance.__setitem__('access_log', slave_parameter_dict.get('access_log')) %}
 {%   do slave_instance.__setitem__('error_log', slave_parameter_dict.get('error_log')) %}
+{%   if enable_cache %}
+{%     do slave_parameter_dict.__setitem__('access_log_cache_direct', '/'.join([caddy_log_cache_direct_directory, '%s_access_log' % slave_reference])) %}
+{%     do slave_parameter_dict.__setitem__('error_log_cache_direct', '/'.join([caddy_log_cache_direct_directory, '%s_error_log' % slave_reference])) %}
+{%     do slave_instance.__setitem__('access_log_cache_direct', slave_parameter_dict.get('access_log_cache_direct')) %}
+{%     do slave_instance.__setitem__('error_log_cache_direct', slave_parameter_dict.get('error_log_cache_direct')) %}
+{%   endif %}
 
 {#   Add slave log directory to the slave log access dict #}
 {%   do slave_log_dict.__setitem__(slave_reference, slave_log_folder) %}
@@ -102,7 +115,6 @@ output = {{ plugin_directory }}/${:name}
 {%     do slave_instance.__setitem__('custom_domain', "%s.%s" % (domain_prefix, slapparameter_dict.get('domain'))) %}
 {%   endif %}
 
-{%   set enable_cache = (('' ~ slave_instance.get('enable_cache', '')).lower() in TRUE_VALUES and slave_type != 'redirect') %}
 {%   if enable_cache and 'url' in slave_instance %}
 {%     if 'domain' in slave_instance %}
 {%       do slave_instance.__setitem__('custom_domain', slave_instance.get('domain')) %}
@@ -125,6 +137,10 @@ output = {{ plugin_directory }}/${:name}
 [slave-log-directory-dict]
 {{slave_reference}} = {{ slave_log_folder }}
 
+{%   if enable_cache %}
+[slave-log-cache-direct-directory-dict]
+{{slave_reference}}_cache_direct = {{ slave_log_cache_direct_folder }}
+{%   endif %}
 
 [slave-password]
 {{ slave_reference }} = {{ '${' + slave_password_section + ':passwd}' }}
@@ -135,6 +151,13 @@ output = {{ plugin_directory }}/${:name}
 name = ${:_buildout_section_name_}
 log = {{slave_parameter_dict.get('access_log')}} {{slave_parameter_dict.get('error_log')}}
 backup = {{ slave_log_folder }} 
+{%    if enable_cache %}
+[{{slave_logrotate_section}}_cache_direct]
+<= logrotate-entry-base
+name = ${:_buildout_section_name_}
+log = {{slave_parameter_dict.get('access_log_cache_direct')}} {{slave_parameter_dict.get('error_log_cache_direct')}}
+backup = {{ slave_log_cache_direct_folder }}
+{%    endif %}
 
 {# integrate current logs inside #}
 [{{slave_ln_section}}]
@@ -340,6 +363,12 @@ recipe = slapos.cookbook:mkdirectory
 
 {% do part_list.append('slave-log-directories') %}
 
+[slave-log-cache-direct-directories]
+<= slave-log-cache-direct-directory-dict
+recipe = slapos.cookbook:mkdirectory
+
+{% do part_list.append('slave-log-cache-direct-directories') %}
+
 [caddy-log-access]
 < = jinja2-template-base
 template = {{frontend_configuration.get('template-log-access')}}

software/caddy-frontend/templates/cached-virtualhost.conf.in

@@ -36,6 +36,13 @@
     insecure_skip_verify
 {%- endif %}
   }
+  log / {{ slave_parameter.get('access_log_cache_direct') }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
+    rotate_size 0
+  }
+
+  errors {{ slave_parameter.get('error_log_cache_direct') }} {
+    rotate_size 0
+  }
 }
 
 # SSL-enabled backends
@@ -58,4 +65,11 @@
     insecure_skip_verify
 {%- endif %}
   }
+  log / {{ slave_parameter.get('access_log_cache_direct') }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
+    rotate_size 0
+  }
+
+  errors {{ slave_parameter.get('error_log_cache_direct') }} {
+    rotate_size 0
+  }
 }

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_file_list_etc_cron_d-CADDY.txt

@@ -5,9 +5,4 @@ T-0/etc/cron.d/monitor_collect
 T-1/etc/cron.d/logrotate
 T-1/etc/cron.d/monitor-configurator
 T-1/etc/cron.d/monitor-globalstate
-T-1/etc/cron.d/monitor_collect
-T-2/etc/cron.d/logrotate
-T-2/etc/cron.d/monitor-configurator
-T-2/etc/cron.d/monitor-globalstate
-T-2/etc/cron.d/monitor_collect
-T-2/etc/cron.d/trafficserver-logrotate
\ No newline at end of file
+T-1/etc/cron.d/monitor_collect
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_file_list_log-CADDY.txt

 T-0/var/log/monitor-httpd-error.log
 T-0/var/log/slapgrid-T-0-error.log
-T-1/var/log/expose-csr_id.log
-T-2/var/log/frontend-access.log
-T-2/var/log/frontend-error.log
-T-2/var/log/httpd-csr_id/expose-csr_id.log
-T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out
\ No newline at end of file
+T-1/var/log/expose-csr_id.log
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_file_list_run-CADDY.txt

 T-0/var/run/monitor-httpd.pid
 T-0/var/run/monitor/monitor-bootstrap.pid
 T-1/var/run/kedifa.pid
-T-1/var/run/monitor/monitor-bootstrap.pid
-T-2/var/run/caddy_configuration_last_state
-T-2/var/run/graceful_configuration_state_signature
-T-2/var/run/httpd.pid
-T-2/var/run/monitor/monitor-bootstrap.pid
-T-2/var/run/validate_configuration_state_signature
\ No newline at end of file
+T-1/var/run/monitor/monitor-bootstrap.pid
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_promise_run_plugin-CADDY.txt

@@ -7,18 +7,4 @@ T-1/etc/plugin/check-free-disk-space.py: OK
 T-1/etc/plugin/expose-csr_id-ip-port-listening.py: OK
 T-1/etc/plugin/kedifa-ip-port-listening.py: OK
 T-1/etc/plugin/monitor-bootstrap-status.py: OK
-T-1/etc/plugin/monitor-httpd-listening-on-tcp.py: OK
-T-2/etc/plugin/buildout-T-2-status.py: OK
-T-2/etc/plugin/caddy_cached.py: ERROR
-T-2/etc/plugin/caddy_frontend_ipv4_http.py: OK
-T-2/etc/plugin/caddy_frontend_ipv4_https.py: OK
-T-2/etc/plugin/caddy_frontend_ipv6_http.py: OK
-T-2/etc/plugin/caddy_frontend_ipv6_https.py: OK
-T-2/etc/plugin/caddy_ssl_cached.py: ERROR
-T-2/etc/plugin/check-free-disk-space.py: OK
-T-2/etc/plugin/expose-csr_id-ip-port-listening.py: OK
-T-2/etc/plugin/frontend-caddy-configuration-promise.py: OK
-T-2/etc/plugin/monitor-bootstrap-status.py: OK
-T-2/etc/plugin/monitor-httpd-listening-on-tcp.py: OK
-T-2/etc/plugin/re6st-connectivity.py: OK
-T-2/etc/plugin/trafficserver-port-listening.py: OK
\ No newline at end of file
+T-1/etc/plugin/monitor-httpd-listening-on-tcp.py: OK
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_supervisor_state-CADDY.txt

@@ -13,20 +13,4 @@ T-1:kedifa-{hash-generic}-on-watch RUNNING
 T-1:kedifa-reloader EXITED
 T-1:monitor-httpd-{hash-generic}-on-watch EXITED
 T-1:monitor-httpd-graceful EXITED
-T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
-T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
-T-2:6tunnel-26011-{hash-generic}-on-watch RUNNING
-T-2:6tunnel-26012-{hash-generic}-on-watch RUNNING
-T-2:bootstrap-monitor EXITED
-T-2:certificate_authority-{hash-generic}-on-watch RUNNING
-T-2:crond-{hash-generic}-on-watch RUNNING
-T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
-T-2:frontend-caddy-safe-graceful EXITED
-T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
-T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
-T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
-T-2:monitor-httpd-{hash-generic}-on-watch EXITED
-T-2:monitor-httpd-graceful EXITED
-T-2:trafficserver-{hash-generic}-on-watch RUNNING
-T-2:trafficserver-reload EXITED
 watchdog:watchdog RUNNING
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_file_list_etc_cron_d-CADDY.txt

@@ -5,9 +5,4 @@ T-0/etc/cron.d/monitor_collect
 T-1/etc/cron.d/logrotate
 T-1/etc/cron.d/monitor-configurator
 T-1/etc/cron.d/monitor-globalstate
-T-1/etc/cron.d/monitor_collect
-T-2/etc/cron.d/logrotate
-T-2/etc/cron.d/monitor-configurator
-T-2/etc/cron.d/monitor-globalstate
-T-2/etc/cron.d/monitor_collect
-T-2/etc/cron.d/trafficserver-logrotate
\ No newline at end of file
+T-1/etc/cron.d/monitor_collect
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_file_list_log-CADDY.txt

 T-0/var/log/monitor-httpd-error.log
 T-0/var/log/slapgrid-T-0-error.log
-T-1/var/log/expose-csr_id.log
-T-2/var/log/frontend-access.log
-T-2/var/log/frontend-error.log
-T-2/var/log/httpd-csr_id/expose-csr_id.log
-T-2/var/log/trafficserver/manager.log
-T-2/var/log/trafficserver/traffic.out
\ No newline at end of file
+T-1/var/log/expose-csr_id.log
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_file_list_run-CADDY.txt

 T-0/var/run/monitor-httpd.pid
 T-0/var/run/monitor/monitor-bootstrap.pid
 T-1/var/run/kedifa.pid
-T-1/var/run/monitor/monitor-bootstrap.pid
-T-2/var/run/caddy_configuration_last_state
-T-2/var/run/graceful_configuration_state_signature
-T-2/var/run/httpd.pid
-T-2/var/run/monitor/monitor-bootstrap.pid
-T-2/var/run/validate_configuration_state_signature
\ No newline at end of file
+T-1/var/run/monitor/monitor-bootstrap.pid
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_promise_run_plugin-CADDY.txt

@@ -7,18 +7,4 @@ T-1/etc/plugin/check-free-disk-space.py: OK
 T-1/etc/plugin/expose-csr_id-ip-port-listening.py: OK
 T-1/etc/plugin/kedifa-ip-port-listening.py: OK
 T-1/etc/plugin/monitor-bootstrap-status.py: OK
-T-1/etc/plugin/monitor-httpd-listening-on-tcp.py: OK
-T-2/etc/plugin/buildout-T-2-status.py: OK
-T-2/etc/plugin/caddy_cached.py: ERROR
-T-2/etc/plugin/caddy_frontend_ipv4_http.py: OK
-T-2/etc/plugin/caddy_frontend_ipv4_https.py: OK
-T-2/etc/plugin/caddy_frontend_ipv6_http.py: OK
-T-2/etc/plugin/caddy_frontend_ipv6_https.py: OK
-T-2/etc/plugin/caddy_ssl_cached.py: ERROR
-T-2/etc/plugin/check-free-disk-space.py: OK
-T-2/etc/plugin/expose-csr_id-ip-port-listening.py: OK
-T-2/etc/plugin/frontend-caddy-configuration-promise.py: OK
-T-2/etc/plugin/monitor-bootstrap-status.py: OK
-T-2/etc/plugin/monitor-httpd-listening-on-tcp.py: OK
-T-2/etc/plugin/re6st-connectivity.py: OK
-T-2/etc/plugin/trafficserver-port-listening.py: OK
\ No newline at end of file
+T-1/etc/plugin/monitor-httpd-listening-on-tcp.py: OK
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_supervisor_state-CADDY.txt

@@ -13,20 +13,4 @@ T-1:kedifa-{hash-generic}-on-watch RUNNING
 T-1:kedifa-reloader EXITED
 T-1:monitor-httpd-{hash-generic}-on-watch EXITED
 T-1:monitor-httpd-graceful EXITED
-T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
-T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
-T-2:6tunnel-26011-{hash-generic}-on-watch RUNNING
-T-2:6tunnel-26012-{hash-generic}-on-watch RUNNING
-T-2:bootstrap-monitor EXITED
-T-2:certificate_authority-{hash-generic}-on-watch RUNNING
-T-2:crond-{hash-generic}-on-watch RUNNING
-T-2:expose-csr_id-{hash-generic}-on-watch RUNNING
-T-2:frontend-caddy-safe-graceful EXITED
-T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
-T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
-T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
-T-2:monitor-httpd-{hash-generic}-on-watch EXITED
-T-2:monitor-httpd-graceful EXITED
-T-2:trafficserver-{hash-generic}-on-watch RUNNING
-T-2:trafficserver-reload EXITED
 watchdog:watchdog RUNNING
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-CADDY.txt

@@ -3,6 +3,18 @@ T-0/var/log/slapgrid-T-0-error.log
 T-1/var/log/expose-csr_id.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log
+T-2/var/log/httpd-cache-direct/_enable_cache-disable-no-cache-request_access_log
+T-2/var/log/httpd-cache-direct/_enable_cache-disable-no-cache-request_error_log
+T-2/var/log/httpd-cache-direct/_enable_cache-disable-via-header_access_log
+T-2/var/log/httpd-cache-direct/_enable_cache-disable-via-header_error_log
+T-2/var/log/httpd-cache-direct/_enable_cache-ssl-proxy-verify-unverified_access_log
+T-2/var/log/httpd-cache-direct/_enable_cache-ssl-proxy-verify-unverified_error_log
+T-2/var/log/httpd-cache-direct/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_access_log
+T-2/var/log/httpd-cache-direct/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_error_log
+T-2/var/log/httpd-cache-direct/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt_access_log
+T-2/var/log/httpd-cache-direct/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt_error_log
+T-2/var/log/httpd-cache-direct/_enable_cache_access_log
+T-2/var/log/httpd-cache-direct/_enable_cache_error_log
 T-2/var/log/httpd-csr_id/expose-csr_id.log
 T-2/var/log/httpd/_Url_access_log
 T-2/var/log/httpd/_Url_error_log

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -3,6 +3,18 @@ T-0/var/log/slapgrid-T-0-error.log
 T-1/var/log/expose-csr_id.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log
+T-2/var/log/httpd-cache-direct/_enable_cache-disable-no-cache-request_access_log
+T-2/var/log/httpd-cache-direct/_enable_cache-disable-no-cache-request_error_log
+T-2/var/log/httpd-cache-direct/_enable_cache-disable-via-header_access_log
+T-2/var/log/httpd-cache-direct/_enable_cache-disable-via-header_error_log
+T-2/var/log/httpd-cache-direct/_enable_cache-ssl-proxy-verify-unverified_access_log
+T-2/var/log/httpd-cache-direct/_enable_cache-ssl-proxy-verify-unverified_error_log
+T-2/var/log/httpd-cache-direct/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_access_log
+T-2/var/log/httpd-cache-direct/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt-unverified_error_log
+T-2/var/log/httpd-cache-direct/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt_access_log
+T-2/var/log/httpd-cache-direct/_enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt_error_log
+T-2/var/log/httpd-cache-direct/_enable_cache_access_log
+T-2/var/log/httpd-cache-direct/_enable_cache_error_log
 T-2/var/log/httpd-csr_id/expose-csr_id.log
 T-2/var/log/httpd/_Url_access_log
 T-2/var/log/httpd/_Url_error_log