Commit 032443a4 authored by Łukasz Nowak's avatar Łukasz Nowak

XXX

parent 1d65ee00
...@@ -22,19 +22,19 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68 ...@@ -22,19 +22,19 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
[profile-frontend] [profile-frontend]
filename = instance-frontend.cfg.in filename = instance-frontend.cfg.in
md5sum = 11c152cb4a23814051eed6d6bb836c00 md5sum = c6c24579eca6ff23bcea039ce47ee78c
[profile-master] [profile-master]
filename = instance-master.cfg.in filename = instance-master.cfg.in
md5sum = e161e544c64ace05eb4d2306d604153f md5sum = 7cd3e058355018eb18b85d6cdcc0d9d5
[profile-slave-list] [profile-slave-list]
filename = instance-slave-list.cfg.in filename = instance-slave-list.cfg.in
md5sum = 5ce5113f9fa2e6e7d393b4ae8fb10dcd md5sum = d88c16fb9e5adb0f40294573921eed72
[profile-master-publish-slave-information] [profile-master-publish-slave-information]
filename = instance-master-publish-slave-information.cfg.in filename = instance-master-publish-slave-information.cfg.in
md5sum = cba4d995962f7fbeae3f61c9372c4181 md5sum = 3923ad1a985520ff38cf133259078d89
[template-frontend-haproxy-configuration] [template-frontend-haproxy-configuration]
_update_hash_filename_ = templates/frontend-haproxy.cfg.in _update_hash_filename_ = templates/frontend-haproxy.cfg.in
...@@ -102,7 +102,7 @@ md5sum = e82ccdb0b26552a1c88ff523d8fae24a ...@@ -102,7 +102,7 @@ md5sum = e82ccdb0b26552a1c88ff523d8fae24a
[profile-kedifa] [profile-kedifa]
filename = instance-kedifa.cfg.in filename = instance-kedifa.cfg.in
md5sum = d6e28c1bdc971134b8de1b5b3e50c3b6 md5sum = 5533c2d153cce0f4ef2624df14fb60ac
[template-frontend-haproxy-rsyslogd-conf] [template-frontend-haproxy-rsyslogd-conf]
_update_hash_filename_ = templates/frontend-haproxy-rsyslogd.conf.in _update_hash_filename_ = templates/frontend-haproxy-rsyslogd.conf.in
......
...@@ -190,7 +190,7 @@ single-custom-personal = dynamic-custom-personal-profile-slave-list:output ...@@ -190,7 +190,7 @@ single-custom-personal = dynamic-custom-personal-profile-slave-list:output
[frontend-configuration] [frontend-configuration]
ip-access-certificate = ${self-signed-ip-access:certificate} ip-access-certificate = ${self-signed-ip-access:certificate}
slave-introspection-configuration = ${directory:etc}/slave-introspection-httpd-nginx.conf slave-introspection-configuration = ${directory:etc}/slave-introspection-httpd-nginx.conf
slave-introspection-https-port = ${configuration:slave-introspection-https-port} frontend-introspection-https-port = ${configuration:frontend-introspection-https-port}
slave-introspection-secure_access = ${slave-introspection-frontend:connection-secure_access} slave-introspection-secure_access = ${slave-introspection-frontend:connection-secure_access}
[self-signed-ip-access] [self-signed-ip-access]
...@@ -312,7 +312,7 @@ csr = ${caucase-updater-csr:csr} ...@@ -312,7 +312,7 @@ csr = ${caucase-updater-csr:csr}
crl = ${kedifa-login-config:crl} crl = ${kedifa-login-config:crl}
kedifa-updater-mapping-file = ${directory:etc}/kedifa_updater_mapping.txt kedifa-updater-mapping-file = ${directory:etc}/kedifa_updater_mapping.txt
kedifa-updater-state-file = ${directory:srv}/kedifa_updater_state.json kedifa-updater-state-file = ${directory:srv}/kedifa_updater_state.json
slave_kedifa_information = {{ dumps(instance_parameter_dict['configuration']['slave-kedifa-information']) }} slave_kedifa_information = {{ dumps(instance_parameter_dict['configuration']['frontend-kedifa-information']) }}
[backend-client-login-config] [backend-client-login-config]
d = ${directory:backend-client-dir} d = ${directory:backend-client-dir}
...@@ -359,8 +359,8 @@ stop-on-error = True ...@@ -359,8 +359,8 @@ stop-on-error = True
[dynamic-custom-personal-profile-slave-list] [dynamic-custom-personal-profile-slave-list]
< = jinja2-template-base < = jinja2-template-base
depends = ${software-py:recipe} depends = ${software-py:recipe}
url = {{ software_parameter_dict['profile_slave_list'] }} url = {{ software_parameter_dict['profile_frontend_list'] }}
filename = instance-slave-list.cfg filename = instance-frontend-list.cfg
master_key_download_url = {{ dumps(instance_parameter_dict['configuration']['master-key-download-url']) }} master_key_download_url = {{ dumps(instance_parameter_dict['configuration']['master-key-download-url']) }}
software_type = single-custom-personal software_type = single-custom-personal
organization = {{ instance_parameter_dict['configuration']['cluster-identification'] }} organization = {{ instance_parameter_dict['configuration']['cluster-identification'] }}
...@@ -1001,7 +1001,7 @@ recipe = slapos.cookbook:requestoptional ...@@ -1001,7 +1001,7 @@ recipe = slapos.cookbook:requestoptional
name = Slave Introspection Frontend {{ instance_parameter_dict['configuration']['frontend-name'] }} name = Slave Introspection Frontend {{ instance_parameter_dict['configuration']['frontend-name'] }}
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
shared = true shared = true
config-url = https://[${slap-configuration:ipv6-random}]:{{ instance_parameter_dict['configuration']['slave-introspection-https-port'] }}/ config-url = https://[${slap-configuration:ipv6-random}]:{{ instance_parameter_dict['configuration']['frontend-introspection-https-port'] }}/
config-https-only = true config-https-only = true
return = secure_access return = secure_access
...@@ -1088,7 +1088,7 @@ context = ...@@ -1088,7 +1088,7 @@ context =
promise = check_socket_listening promise = check_socket_listening
name = slave_introspection_https.py name = slave_introspection_https.py
config-host = {{ instance_parameter_dict['ipv6-random'] }} config-host = {{ instance_parameter_dict['ipv6-random'] }}
config-port = ${frontend-configuration:slave-introspection-https-port} config-port = ${frontend-configuration:frontend-introspection-https-port}
[logrotate-entry-slave-introspection] [logrotate-entry-slave-introspection]
<= logrotate-entry-base <= logrotate-entry-base
......
...@@ -13,7 +13,7 @@ parts = ...@@ -13,7 +13,7 @@ parts =
kedifa kedifa
logrotate-entry-kedifa logrotate-entry-kedifa
promise-kedifa-http-reply promise-kedifa-http-reply
slave-kedifa-information frontend-kedifa-information
caucased caucased
caucased-promise caucased-promise
caucase-updater caucase-updater
...@@ -248,16 +248,16 @@ command-line = {{ software_parameter_dict['kedifa'] }} ...@@ -248,16 +248,16 @@ command-line = {{ software_parameter_dict['kedifa'] }}
wrapper-path = ${directory:service}/kedifa wrapper-path = ${directory:service}/kedifa
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
# Publish KeDiFa configuration for upload and download for each slave # Publish KeDiFa configuration for upload and download for each frontend
{%- set slave_kedifa_information = {} -%} {%- set frontend_kedifa_information = {} -%}
{%- for slave in instance_parameter_dict['configuration']['slave-list'] -%} {%- for frontend in instance_parameter_dict['configuration']['frontend-list'] -%}
{%- set slave_reference = slave['slave_reference'] -%} {%- set slave_reference = frontend['slave_reference'] -%}
{%- set slave_dict = {} -%} {%- set frontend_dict = {} -%}
{%- do slave_dict.__setitem__('key-generate-auth-url', 'https://[${kedifa-config:ip}]:${kedifa-config:port}/${%s-auth-random:passwd}/generateauth' % (slave_reference,)) -%} {%- do frontend_dict.__setitem__('key-generate-auth-url', 'https://[${kedifa-config:ip}]:${kedifa-config:port}/${%s-auth-random:passwd}/generateauth' % (slave_reference,)) -%}
{%- do slave_dict.__setitem__('key-upload-url', 'https://[${kedifa-config:ip}]:${kedifa-config:port}/${%s-auth-random:passwd}?auth=' % (slave_reference,)) -%} {%- do frontend_dict.__setitem__('key-upload-url', 'https://[${kedifa-config:ip}]:${kedifa-config:port}/${%s-auth-random:passwd}?auth=' % (slave_reference,)) -%}
{%- do slave_dict.__setitem__('key-download-url', 'https://[${kedifa-config:ip}]:${kedifa-config:port}/${%s-auth-random:passwd}' % (slave_reference,)) -%} {%- do frontend_dict.__setitem__('key-download-url', 'https://[${kedifa-config:ip}]:${kedifa-config:port}/${%s-auth-random:passwd}' % (slave_reference,)) -%}
{%- do slave_dict.__setitem__('kedifa-caucase-url', caucase_url ) -%} {%- do frontend_dict.__setitem__('kedifa-caucase-url', caucase_url ) -%}
{%- do slave_kedifa_information.__setitem__(slave_reference, slave_dict) %} {%- do frontend_kedifa_information.__setitem__(slave_reference, frontend_dict) %}
[{{ slave_reference }}-auth-random-generate] [{{ slave_reference }}-auth-random-generate]
recipe = plone.recipe.command recipe = plone.recipe.command
file = ${directory:reservation}/${:_buildout_section_name_} file = ${directory:reservation}/${:_buildout_section_name_}
...@@ -279,7 +279,7 @@ command = ...@@ -279,7 +279,7 @@ command =
update-command = ${:command} update-command = ${:command}
[cleanup-auth-ready] [cleanup-auth-ready]
# Cleans auth-ready as some slaves can appear and be configured, but then # Cleans auth-ready as some frontends can appear and be configured, but then
# become removed before auth-random kicks in # become removed before auth-random kicks in
recipe = slapos.recipe.build recipe = slapos.recipe.build
directory = ${directory:auth-ready} directory = ${directory:auth-ready}
...@@ -310,10 +310,10 @@ init = ...@@ -310,10 +310,10 @@ init =
<= auth-random <= auth-random
file = ${master-auth-random-generate:file} file = ${master-auth-random-generate:file}
[slave-kedifa-information] [frontend-kedifa-information]
recipe = slapos.cookbook:publish.serialised recipe = slapos.cookbook:publish.serialised
{# sort_keys are important in order to avoid shuffling parameters on each run #} {# sort_keys are important in order to avoid shuffling parameters on each run #}
slave-kedifa-information = {{ json_module.dumps(slave_kedifa_information, sort_keys=True) }} frontend-kedifa-information = {{ json_module.dumps(frontend_kedifa_information, sort_keys=True) }}
caucase-url = {{ caucase_url }} caucase-url = {{ caucase_url }}
master-key-generate-auth-url = https://[${kedifa-config:ip}]:${kedifa-config:port}/${master-auth-random:passwd}/generateauth master-key-generate-auth-url = https://[${kedifa-config:ip}]:${kedifa-config:port}/${master-auth-random:passwd}/generateauth
master-key-upload-url = https://[${kedifa-config:ip}]:${kedifa-config:port}/${master-auth-random:passwd}?auth= master-key-upload-url = https://[${kedifa-config:ip}]:${kedifa-config:port}/${master-auth-random:passwd}?auth=
......
{% set part_list = [] %} {% set part_list = [] %}
{% set slave_information_dict = {} %} {% set frontend_information_dict = {} %}
# regroup slave information from all frontends # regroup frontend information from all frontend nodes
{% for frontend, slave_list_raw in slave_information.items() %} {% for frontend_node, frontend_list_raw in frontend_information.items() %}
{% if slave_list_raw %} {% if frontend_list_raw %}
{% set slave_list = json_module.loads(slave_list_raw) %} {% set frontend_list = json_module.loads(frontend_list_raw) %}
{% else %} {% else %}
{% set slave_list = [] %} {% set frontend_list = [] %}
{% endif %} {% endif %}
{% for slave_dict in slave_list %} {% for frontend_dict in frontend_list %}
{% set slave_reference = slave_dict.pop('slave-reference') %} {% set slave_reference = frontend_dict.pop('slave-reference') %}
{% set log_access_url = slave_dict.pop('log-access', '') %} {% set log_access_url = frontend_dict.pop('log-access', '') %}
{% set current_slave_dict = slave_information_dict.get(slave_reference, {}) %} {% set current_frontend_dict = frontend_information_dict.get(slave_reference, {}) %}
{% do current_slave_dict.update(slave_dict) %} {% do current_frontend_dict.update(frontend_dict) %}
{% set log_access_list = current_slave_dict.get('log-access-urls', []) %} {% set log_access_list = current_frontend_dict.get('log-access-urls', []) %}
{% do log_access_list.append( frontend + ': ' + log_access_url) %} {% do log_access_list.append( frontend_node + ': ' + log_access_url) %}
{% do current_slave_dict.__setitem__( {% do current_frontend_dict.__setitem__(
'log-access-urls', 'log-access-urls',
log_access_list log_access_list
) %} ) %}
{% do current_slave_dict.__setitem__( {% do current_frontend_dict.__setitem__(
'replication_number', 'replication_number',
current_slave_dict.get('replication_number', 0) + 1 current_frontend_dict.get('replication_number', 0) + 1
) %} ) %}
{% do slave_information_dict.__setitem__(slave_reference, current_slave_dict) %} {% do frontend_information_dict.__setitem__(slave_reference, current_frontend_dict) %}
{% endfor %} {% endfor %}
{% endfor %} {% endfor %}
{% for slave_reference, rejected_info_list in rejected_slave_information['rejected-slave-dict'].items() %} {% for slave_reference, rejected_info_list in rejected_frontend_information['rejected-frontend-dict'].items() %}
{% if slave_reference not in slave_information_dict %} {% if slave_reference not in frontend_information_dict %}
{% do slave_information_dict.__setitem__(slave_reference, {}) %} {% do frontend_information_dict.__setitem__(slave_reference, {}) %}
{% endif %} {% endif %}
{% do slave_information_dict[slave_reference].__setitem__('request-error-list', json_module.dumps(rejected_info_list)) %} {% do frontend_information_dict[slave_reference].__setitem__('request-error-list', json_module.dumps(rejected_info_list)) %}
{% endfor %} {% endfor %}
{% for slave_reference, warning_info_list in warning_slave_information['warning-slave-dict'].items() %} {% for slave_reference, warning_info_list in warning_frontend_information['warning-frontend-dict'].items() %}
{% if slave_reference not in slave_information_dict %} {% if slave_reference not in frontend_information_dict %}
{% do slave_information_dict.__setitem__(slave_reference, {}) %} {% do frontend_information_dict.__setitem__(slave_reference, {}) %}
{% endif %} {% endif %}
{% do slave_information_dict[slave_reference].__setitem__('warning-list', json_module.dumps(warning_info_list)) %} {% do frontend_information_dict[slave_reference].__setitem__('warning-list', json_module.dumps(warning_info_list)) %}
{% endfor %} {% endfor %}
{% for slave_reference, kedifa_dict in json_module.loads(slave_kedifa_information).items() %} {% for slave_reference, kedifa_dict in json_module.loads(frontend_kedifa_information).items() %}
{% if slave_reference not in rejected_slave_information['rejected-slave-dict'] %} {% if slave_reference not in rejected_frontend_information['rejected-frontend-dict'] %}
{% if slave_reference not in slave_information_dict %} {% if slave_reference not in frontend_information_dict %}
{% do slave_information_dict.__setitem__(slave_reference, {}) %} {% do frontend_information_dict.__setitem__(slave_reference, {}) %}
{% endif %} {% endif %}
{% do slave_information_dict[slave_reference].__setitem__('key-generate-auth-url', kedifa_dict['key-generate-auth-url']) %} {% do frontend_information_dict[slave_reference].__setitem__('key-generate-auth-url', kedifa_dict['key-generate-auth-url']) %}
{% do slave_information_dict[slave_reference].__setitem__('key-upload-url', kedifa_dict['key-upload-url']) %} {% do frontend_information_dict[slave_reference].__setitem__('key-upload-url', kedifa_dict['key-upload-url']) %}
{% do slave_information_dict[slave_reference].__setitem__('kedifa-caucase-url', kedifa_dict['kedifa-caucase-url']) %} {% do frontend_information_dict[slave_reference].__setitem__('kedifa-caucase-url', kedifa_dict['kedifa-caucase-url']) %}
{% endif %} {% endif %}
{% endfor %} {% endfor %}
[common-frontend-information] [common-frontend-information]
{% for frontend_key, frontend_value in frontend_information.items() %} {% for frontend_node_key, frontend_node_value in frontend_node_information.items() %}
{{ frontend_key }} = {{ frontend_value }} {{ frontend_node_key }} = {{ frontend_node_value }}
{% endfor %} {% endfor %}
# Publish information for each slave # Publish information for each frontend
{% set active_slave_instance_list = json_module.loads(active_slave_instance_dict['active-slave-instance-list']) %} {% set active_frontend_list = json_module.loads(active_frontend_instance_dict['active-frontend-list']) %}
{% for slave_reference, slave_information in slave_information_dict.items() %} {% for slave_reference, frontend_information in frontend_information_dict.items() %}
{# Filter out destroyed, so not existing anymore, slaves #} {# Filter out destroyed, so not existing anymore, frontends #}
{# Note: This functionality is not yet covered by tests, please modify with care #} {# Note: This functionality is not yet covered by tests, please modify with care #}
{% if slave_reference in active_slave_instance_list %} {% if slave_reference in active_frontend_list %}
{% set publish_section_title = 'publish-%s' % slave_reference %} {% set publish_section_title = 'publish-%s' % slave_reference %}
{% do part_list.append(publish_section_title) %} {% do part_list.append(publish_section_title) %}
[{{ publish_section_title }}] [{{ publish_section_title }}]
...@@ -70,12 +70,12 @@ ...@@ -70,12 +70,12 @@
recipe = slapos.cookbook:publish_failsafe recipe = slapos.cookbook:publish_failsafe
-error-status-file = {{ publish_failsafe_error }}/{{ slave_reference }}-error.status -error-status-file = {{ publish_failsafe_error }}/{{ slave_reference }}-error.status
-slave-reference = {{ slave_reference }} -slave-reference = {{ slave_reference }}
{% set log_access_url = slave_information.pop('log-access-urls', None) %} {% set log_access_url = frontend_information.pop('log-access-urls', None) %}
{% if log_access_url %} {% if log_access_url %}
{# sort_keys are important in order to avoid shuffling parameters on each run #} {# sort_keys are important in order to avoid shuffling parameters on each run #}
log-access-url = {{ dumps(json_module.dumps(log_access_url, sort_keys=True)) }} log-access-url = {{ dumps(json_module.dumps(log_access_url, sort_keys=True)) }}
{% endif %} {% endif %}
{% for key, value in slave_information.items() %} {% for key, value in frontend_information.items() %}
{{ key }} = {{ dumps(value) }} {{ key }} = {{ dumps(value) }}
{% endfor %} {% endfor %}
{% endif %} {% endif %}
......
This diff is collapsed.
...@@ -12,7 +12,6 @@ ...@@ -12,7 +12,6 @@
{%- set generic_instance_parameter_dict = { 'cache_access': cache_access, 'local_ipv4': instance_parameter_dict['ipv4-random'], 'http_port': configuration['plain_http_port'], 'https_port': configuration['port']} %} {%- set generic_instance_parameter_dict = { 'cache_access': cache_access, 'local_ipv4': instance_parameter_dict['ipv4-random'], 'http_port': configuration['plain_http_port'], 'https_port': configuration['port']} %}
{%- set slave_log_dict = {} %} {%- set slave_log_dict = {} %}
{%- set slave_instance_information_list = [] %} {%- set slave_instance_information_list = [] %}
{%- set slave_instance_list = instance_parameter_dict['slave-instance-list'] %}
{%- set frontend_list = json_module.loads(configuration['frontend-list']) %} {#- XXX Upward compatibility #} {%- set frontend_list = json_module.loads(configuration['frontend-list']) %} {#- XXX Upward compatibility #}
{%- do frontend_list.extend(json_module.loads(configuration.get('extra_slave_instance_list', []))) %} {#- XXX Backward compatibility #} {%- do frontend_list.extend(json_module.loads(configuration.get('extra_slave_instance_list', []))) %} {#- XXX Backward compatibility #}
{%- if master_key_download_url %} {%- if master_key_download_url %}
...@@ -56,7 +55,7 @@ context = ...@@ -56,7 +55,7 @@ context =
{%- do configuration.__setitem__(key, ('' ~ configuration[key]).lower() in TRUE_VALUES) %} {%- do configuration.__setitem__(key, ('' ~ configuration[key]).lower() in TRUE_VALUES) %}
{%- endfor %} {%- endfor %}
{#- Loop thought slave list to set up slaves #} {#- Loop thought slave list to set up slaves #}
{%- for slave_instance in slave_instance_list %} {%- for slave_instance in frontend_list %}
{#- Prepare slave parameters: #} {#- Prepare slave parameters: #}
{#- * convert strings to booleans (as slapproxy and SlapOS Master differ a bit) #} {#- * convert strings to booleans (as slapproxy and SlapOS Master differ a bit) #}
{#- * create real lists from string lists #} {#- * create real lists from string lists #}
...@@ -388,7 +387,7 @@ local_ipv4 = {{ dumps('' ~ instance_parameter_dict['ipv4-random']) }} ...@@ -388,7 +387,7 @@ local_ipv4 = {{ dumps('' ~ instance_parameter_dict['ipv4-random']) }}
{%- if slave_type != 'redirect' %} {%- if slave_type != 'redirect' %}
{%- do backend_slave_list.append(slave_instance) %} {%- do backend_slave_list.append(slave_instance) %}
{%- endif %} {%- endif %}
{%- endfor %} {# Slave iteration ends for slave_instance in slave_instance_list #} {%- endfor %} {# Slave iteration ends for slave_instance in frontend_list #}
{%- do part_list.append('slave-introspection') %} {%- do part_list.append('slave-introspection') %}
{#- ############################################## #} {#- ############################################## #}
......
...@@ -75,8 +75,8 @@ profile_kedifa = ${profile-kedifa:target} ...@@ -75,8 +75,8 @@ profile_kedifa = ${profile-kedifa:target}
profile_logrotate_base = ${template-logrotate-base:output} profile_logrotate_base = ${template-logrotate-base:output}
profile_monitor = ${monitor-template:output} profile_monitor = ${monitor-template:output}
profile_monitor2 = ${monitor2-template:output} profile_monitor2 = ${monitor2-template:output}
profile_master_publish_slave_information = ${profile-master-publish-slave-information:target} profile_master_publish_frontend_information = ${profile-master-publish-frontend-information:target}
profile_slave_list = ${profile-slave-list:target} profile_frontend_list = ${profile-frontend-list:target}
# templates # templates
template_backend_haproxy_configuration = ${template-backend-haproxy-configuration:target} template_backend_haproxy_configuration = ${template-backend-haproxy-configuration:target}
...@@ -147,7 +147,7 @@ url = ${:_profile_base_location_}/${:filename} ...@@ -147,7 +147,7 @@ url = ${:_profile_base_location_}/${:filename}
[profile-slave-list] [profile-slave-list]
<= download-profile <= download-profile
[profile-master-publish-slave-information] [profile-master-publish-frontend-information]
<= download-profile <= download-profile
[download-template] [download-template]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment