Fix Caddy Frontend Scalability 202010

See merge request nexedi/slapos!844

software/caddy-frontend/buildout.hash.cfg

@@ -26,11 +26,11 @@ md5sum = e7d7e1448b6420657e953026573311ca
 
 [profile-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = 59f3a67999f5fb3e595486e2b801af08
+md5sum = b70f9ce80dd927ead51b4526997b75ed
 
 [profile-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = 64d57678c12f539247fe2532c5b8d6b8
+md5sum = ab143bfa2e20725aa35940c9033fa0ee
 
 [profile-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
@@ -40,10 +40,6 @@ md5sum = de268251dafa5ad83ebf5b20636365d9
 _update_hash_filename_ = templates/Caddyfile.in
 md5sum = 2503056e35463e045db3329bb8b6fae8
 
-[caddy-backend-url-validator]
-filename = templates/caddy-backend-url-validator.in
-md5sum = 0979a03476e86bf038516c9565dadc17
-
 [template-not-found-html]
 _update_hash_filename_ = templates/notfound.html
 md5sum = 88af61e7abbf30dc99a1a2526161128d

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -164,7 +164,7 @@ context =
 {%   for url_key in ['url', 'https-url'] %}
 {%     if url_key in slave %}
 {%       set url = (slave[url_key] or '').strip() %}
-{%       if subprocess_module.call([software_parameter_dict['caddy_backend_url_validator'], url]) == 1 or not validators.url(url) %}
+{%       if not validators.url(url) %}
 {%         do slave_error_list.append('slave %s %r invalid' % (url_key, url)) %}
 {%       elif url != slave[url_key] %}
 {%         do slave_warning_list.append('slave %s %r has been converted to %r' % (url_key, slave[url_key], url)) %}
@@ -769,8 +769,9 @@ recipe = plone.recipe.command
 {#- Can be stopped on error, as does not rely on self provided service #}
 stop-on-error = True
 file = ${directory:var}/nginx-rejected.htpasswd
-command = {{ software_parameter_dict['htpasswd'] }} -cb ${:file} ${rejected-slave-password:user} ${rejected-slave-password:passwd}
-update-command = ${:command}
+{#- update-command is not needed, as if the ${:password} would change, the whole part will be recalculated #}
+password = ${rejected-slave-password:passwd}
+command = {{ software_parameter_dict['htpasswd'] }} -cb ${:file} ${rejected-slave-password:user} ${:password}
 
 [rejected-slave-template]
 recipe = slapos.recipe.template:jinja2

software/caddy-frontend/software.cfg

@@ -115,7 +115,6 @@ template_trafficserver_records_config = ${template-trafficserver-records-config:
 template_trafficserver_storage_config = ${template-trafficserver-storage-config:target}
 template_validate_script = ${template-validate-script:target}
 template_wrapper = ${template-wrapper:output}
-caddy_backend_url_validator = ${caddy-backend-url-validator:output}
 
 # directories
 bin_directory = ${buildout:bin-directory}
@@ -154,12 +153,6 @@ recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-apache-frontend.cfg.in
 mode = 0644
 
-[caddy-backend-url-validator]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/${:filename}
-output = ${buildout:directory}/caddy-backend-url-validator
-mode = 0750
-
 [profile-caddy-replicate]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -95,12 +95,13 @@ context =
 {%-   set slave_publish_dict = {} %}
 {%-   set slave_configuration_section_name = 'slave-instance-%s-configuration' % slave_reference %}
 {%-   set slave_logrotate_section = slave_reference + "-logs" %}
+{%-   set slave_log_directory_section = slave_reference + "-log-directory" %}
 {%-   set slave_password_section = slave_reference + "-password" %}
 {%-   set slave_htpasswd_section = slave_reference + "-htpasswd" %}
 {%-   set slave_ln_section = slave_reference + "-ln" %}
 {#-   extend parts #}
 {%-   do part_list.extend([slave_ln_section]) %}
-{%-   do part_list.extend([slave_logrotate_section, slave_section_title]) %}
+{%-   do part_list.extend([slave_section_title]) %}
 {%-   set slave_log_folder = '${logrotate-directory:logrotate-backup}/' + slave_reference + "-logs" %}
 {#-   Pass HTTP2 switch #}
 {%-   do slave_instance.__setitem__('enable_http2_by_default', configuration['enable-http2-by-default']) %}
@@ -151,11 +152,15 @@ context =
 {{ slave_reference }} = {{ '${' + slave_htpasswd_section + ':file}' }}
 
 {#-   Set slave logrotate entry #}
+[{{slave_log_directory_section}}]
+recipe = slapos.cookbook:mkdirectory
+log-directory = {{ '${slave-log-directory-dict:' + slave_reference.lower() + '}' }}
+
 [{{slave_logrotate_section}}]
 <= logrotate-entry-base
 name = ${:_buildout_section_name_}
 log = {{slave_parameter_dict.get('access_log')}} {{slave_parameter_dict.get('error_log')}} {{slave_parameter_dict.get('backend_log')}}
-backup = {{ slave_log_folder }}
+backup = {{ '${' + slave_log_directory_section + ':log-directory}' }}
 rotate-num = {{ dumps('' ~ configuration['rotate-num']) }}
 # disable delayed compression, as log filenames shall be stable
 delaycompress =
@@ -165,8 +170,8 @@ delaycompress =
 [{{slave_ln_section}}]
 recipe = plone.recipe.command
 stop-on-error = false
-update-command = ${:command}
-command = ln -sf {{slave_parameter_dict.get('error_log')}} {{ slave_log_folder }}/error.log && ln -sf {{slave_parameter_dict.get('access_log')}} {{ slave_log_folder }}/access.log && ln -sf {{slave_parameter_dict.get('backend_log')}} {{ slave_log_folder }}/backend.log
+log-directory = {{ '${' + slave_logrotate_section + ':backup}' }}
+command = ln -sf {{slave_parameter_dict.get('error_log')}} ${:log-directory}/error.log && ln -sf {{slave_parameter_dict.get('access_log')}} ${:log-directory}/access.log && ln -sf {{slave_parameter_dict.get('backend_log')}} ${:log-directory}/backend.log
 
 {#-   Set password for slave #}
 
@@ -180,8 +185,9 @@ recipe = plone.recipe.command
 {#- Can be stopped on error, as does not rely on self provided service #}
 stop-on-error = True
 file = {{ caddy_configuration_directory }}/.{{ slave_reference }}.htpasswd
-command = {{ software_parameter_dict['htpasswd'] }} -cb ${:file} {{ slave_reference.lower() }} {{ '${' + slave_password_section + ':passwd}' }}
-update-command = ${:command}
+{#- update-command is not needed, as if the ${:password} would change, the whole part will be recalculated #}
+password = {{ '${' + slave_password_section + ':passwd}' }}
+command = {{ software_parameter_dict['htpasswd'] }} -cb ${:file} {{ slave_reference.lower() }} ${:password}
 
 {#-   ################################################## #}
 {#-   Set Slave Certificates if needed                   #}
@@ -301,12 +307,6 @@ recipe = slapos.cookbook:publish
 {%-  endif %}
 {%- endfor %} {# Slave iteration ends for slave_instance in slave_instance_list #}
 
-[slave-log-directories]
-<= slave-log-directory-dict
-recipe = slapos.cookbook:mkdirectory
-
-{%- do part_list.append('slave-log-directories') %}
-
 {%- do part_list.append('caddy-log-access') %}
 {%- do part_list.append('slave-introspection') %}
 {#- ############################################## #}

software/caddy-frontend/templates/caddy-backend-url-validator.in

-#!${dash:location}/bin/dash
-config="https://example.com {\n  proxy / $1 {\n  }\n}"
-
-echo -e $config | ${caddy:output} -conf stdin -validate > /dev/null 2>&1