Skip to content

slapos
slapos
Commit 032443a4 authored by Łukasz Nowak's avatar Łukasz Nowak
Browse files
Options

XXX

parent 1d65ee00
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 223 additions and 225 deletions
software/rapid-cdn/buildout.hash.cfg
View file @ 032443a4
......@@ -22,19 +22,19 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
[profile-frontend]
filename = instance-frontend.cfg.in
md5sum = 11c152cb4a23814051eed6d6bb836c00
md5sum = c6c24579eca6ff23bcea039ce47ee78c
[profile-master]
filename = instance-master.cfg.in
md5sum = e161e544c64ace05eb4d2306d604153f
md5sum = 7cd3e058355018eb18b85d6cdcc0d9d5
[profile-slave-list]
filename = instance-slave-list.cfg.in
md5sum = 5ce5113f9fa2e6e7d393b4ae8fb10dcd
md5sum = d88c16fb9e5adb0f40294573921eed72
[profile-master-publish-slave-information]
filename = instance-master-publish-slave-information.cfg.in
md5sum = cba4d995962f7fbeae3f61c9372c4181
md5sum = 3923ad1a985520ff38cf133259078d89
[template-frontend-haproxy-configuration]
_update_hash_filename_ = templates/frontend-haproxy.cfg.in
......@@ -102,7 +102,7 @@ md5sum = e82ccdb0b26552a1c88ff523d8fae24a
[profile-kedifa]
filename = instance-kedifa.cfg.in
md5sum = d6e28c1bdc971134b8de1b5b3e50c3b6
md5sum = 5533c2d153cce0f4ef2624df14fb60ac
[template-frontend-haproxy-rsyslogd-conf]
_update_hash_filename_ = templates/frontend-haproxy-rsyslogd.conf.in
......
software/rapid-cdn/instance-frontend.cfg.in
View file @ 032443a4
......@@ -190,7 +190,7 @@ single-custom-personal = dynamic-custom-personal-profile-slave-list:output
[frontend-configuration]
ip-access-certificate = ${self-signed-ip-access:certificate}
slave-introspection-configuration = ${directory:etc}/slave-introspection-httpd-nginx.conf
slave-introspection-https-port = ${configuration:slave-introspection-https-port}
frontend-introspection-https-port = ${configuration:frontend-introspection-https-port}
slave-introspection-secure_access = ${slave-introspection-frontend:connection-secure_access}
[self-signed-ip-access]
......@@ -312,7 +312,7 @@ csr = ${caucase-updater-csr:csr}
crl = ${kedifa-login-config:crl}
kedifa-updater-mapping-file = ${directory:etc}/kedifa_updater_mapping.txt
kedifa-updater-state-file = ${directory:srv}/kedifa_updater_state.json
slave_kedifa_information = {{ dumps(instance_parameter_dict['configuration']['slave-kedifa-information']) }}
slave_kedifa_information = {{ dumps(instance_parameter_dict['configuration']['frontend-kedifa-information']) }}
[backend-client-login-config]
d = ${directory:backend-client-dir}
......@@ -359,8 +359,8 @@ stop-on-error = True
[dynamic-custom-personal-profile-slave-list]
< = jinja2-template-base
depends = ${software-py:recipe}
url = {{ software_parameter_dict['profile_slave_list'] }}
filename = instance-slave-list.cfg
url = {{ software_parameter_dict['profile_frontend_list'] }}
filename = instance-frontend-list.cfg
master_key_download_url = {{ dumps(instance_parameter_dict['configuration']['master-key-download-url']) }}
software_type = single-custom-personal
organization = {{ instance_parameter_dict['configuration']['cluster-identification'] }}
......@@ -1001,7 +1001,7 @@ recipe = slapos.cookbook:requestoptional
name = Slave Introspection Frontend {{ instance_parameter_dict['configuration']['frontend-name'] }}
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
shared = true
config-url = https://[${slap-configuration:ipv6-random}]:{{ instance_parameter_dict['configuration']['slave-introspection-https-port'] }}/
config-url = https://[${slap-configuration:ipv6-random}]:{{ instance_parameter_dict['configuration']['frontend-introspection-https-port'] }}/
config-https-only = true
return = secure_access
......@@ -1088,7 +1088,7 @@ context =
promise = check_socket_listening
name = slave_introspection_https.py
config-host = {{ instance_parameter_dict['ipv6-random'] }}
config-port = ${frontend-configuration:slave-introspection-https-port}
config-port = ${frontend-configuration:frontend-introspection-https-port}
[logrotate-entry-slave-introspection]
<= logrotate-entry-base
......
software/rapid-cdn/instance-kedifa.cfg.in
View file @ 032443a4
......@@ -13,7 +13,7 @@ parts =
kedifa
logrotate-entry-kedifa
promise-kedifa-http-reply
slave-kedifa-information
frontend-kedifa-information
caucased
caucased-promise
caucase-updater
......@@ -248,16 +248,16 @@ command-line = {{ software_parameter_dict['kedifa'] }}
wrapper-path = ${directory:service}/kedifa
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
# Publish KeDiFa configuration for upload and download for each slave
{%- set slave_kedifa_information = {} -%}
{%- for slave in instance_parameter_dict['configuration']['slave-list'] -%}
{%- set slave_reference = slave['slave_reference'] -%}
{%- set slave_dict = {} -%}
{%- do slave_dict.__setitem__('key-generate-auth-url', 'https://[${kedifa-config:ip}]:${kedifa-config:port}/${%s-auth-random:passwd}/generateauth' % (slave_reference,)) -%}
{%- do slave_dict.__setitem__('key-upload-url', 'https://[${kedifa-config:ip}]:${kedifa-config:port}/${%s-auth-random:passwd}?auth=' % (slave_reference,)) -%}
{%- do slave_dict.__setitem__('key-download-url', 'https://[${kedifa-config:ip}]:${kedifa-config:port}/${%s-auth-random:passwd}' % (slave_reference,)) -%}
{%- do slave_dict.__setitem__('kedifa-caucase-url', caucase_url ) -%}
{%- do slave_kedifa_information.__setitem__(slave_reference, slave_dict) %}
# Publish KeDiFa configuration for upload and download for each frontend
{%- set frontend_kedifa_information = {} -%}
{%- for frontend in instance_parameter_dict['configuration']['frontend-list'] -%}
{%- set slave_reference = frontend['slave_reference'] -%}
{%- set frontend_dict = {} -%}
{%- do frontend_dict.__setitem__('key-generate-auth-url', 'https://[${kedifa-config:ip}]:${kedifa-config:port}/${%s-auth-random:passwd}/generateauth' % (slave_reference,)) -%}
{%- do frontend_dict.__setitem__('key-upload-url', 'https://[${kedifa-config:ip}]:${kedifa-config:port}/${%s-auth-random:passwd}?auth=' % (slave_reference,)) -%}
{%- do frontend_dict.__setitem__('key-download-url', 'https://[${kedifa-config:ip}]:${kedifa-config:port}/${%s-auth-random:passwd}' % (slave_reference,)) -%}
{%- do frontend_dict.__setitem__('kedifa-caucase-url', caucase_url ) -%}
{%- do frontend_kedifa_information.__setitem__(slave_reference, frontend_dict) %}
[{{ slave_reference }}-auth-random-generate]
recipe = plone.recipe.command
file = ${directory:reservation}/${:_buildout_section_name_}
......@@ -279,7 +279,7 @@ command =
update-command = ${:command}
[cleanup-auth-ready]
# Cleans auth-ready as some slaves can appear and be configured, but then
# Cleans auth-ready as some frontends can appear and be configured, but then
# become removed before auth-random kicks in
recipe = slapos.recipe.build
directory = ${directory:auth-ready}
......@@ -310,10 +310,10 @@ init =
<= auth-random
file = ${master-auth-random-generate:file}
[slave-kedifa-information]
[frontend-kedifa-information]
recipe = slapos.cookbook:publish.serialised
{# sort_keys are important in order to avoid shuffling parameters on each run #}
slave-kedifa-information = {{ json_module.dumps(slave_kedifa_information, sort_keys=True) }}
frontend-kedifa-information = {{ json_module.dumps(frontend_kedifa_information, sort_keys=True) }}
caucase-url = {{ caucase_url }}
master-key-generate-auth-url = https://[${kedifa-config:ip}]:${kedifa-config:port}/${master-auth-random:passwd}/generateauth
master-key-upload-url = https://[${kedifa-config:ip}]:${kedifa-config:port}/${master-auth-random:passwd}?auth=
......
software/rapid-cdn/instance-master-publish-slave-information.cfg.in
View file @ 032443a4
{% set part_list = [] %}
{% set slave_information_dict = {} %}
{% set frontend_information_dict = {} %}
# regroup slave information from all frontends
{% for frontend, slave_list_raw in slave_information.items() %}
{% if slave_list_raw %}
{% set slave_list = json_module.loads(slave_list_raw) %}
# regroup frontend information from all frontend nodes
{% for frontend_node, frontend_list_raw in frontend_information.items() %}
{% if frontend_list_raw %}
{% set frontend_list = json_module.loads(frontend_list_raw) %}
{% else %}
{% set slave_list = [] %}
{% set frontend_list = [] %}
{% endif %}
{% for slave_dict in slave_list %}
{% set slave_reference = slave_dict.pop('slave-reference') %}
{% set log_access_url = slave_dict.pop('log-access', '') %}
{% set current_slave_dict = slave_information_dict.get(slave_reference, {}) %}
{% do current_slave_dict.update(slave_dict) %}
{% set log_access_list = current_slave_dict.get('log-access-urls', []) %}
{% do log_access_list.append( frontend + ': ' + log_access_url) %}
{% do current_slave_dict.__setitem__(
{% for frontend_dict in frontend_list %}
{% set slave_reference = frontend_dict.pop('slave-reference') %}
{% set log_access_url = frontend_dict.pop('log-access', '') %}
{% set current_frontend_dict = frontend_information_dict.get(slave_reference, {}) %}
{% do current_frontend_dict.update(frontend_dict) %}
{% set log_access_list = current_frontend_dict.get('log-access-urls', []) %}
{% do log_access_list.append( frontend_node + ': ' + log_access_url) %}
{% do current_frontend_dict.__setitem__(
'log-access-urls',
log_access_list
) %}
{% do current_slave_dict.__setitem__(
{% do current_frontend_dict.__setitem__(
'replication_number',
current_slave_dict.get('replication_number', 0) + 1
current_frontend_dict.get('replication_number', 0) + 1
) %}
{% do slave_information_dict.__setitem__(slave_reference, current_slave_dict) %}
{% do frontend_information_dict.__setitem__(slave_reference, current_frontend_dict) %}
{% endfor %}
{% endfor %}
{% for slave_reference, rejected_info_list in rejected_slave_information['rejected-slave-dict'].items() %}
{% if slave_reference not in slave_information_dict %}
{% do slave_information_dict.__setitem__(slave_reference, {}) %}
{% for slave_reference, rejected_info_list in rejected_frontend_information['rejected-frontend-dict'].items() %}
{% if slave_reference not in frontend_information_dict %}
{% do frontend_information_dict.__setitem__(slave_reference, {}) %}
{% endif %}
{% do slave_information_dict[slave_reference].__setitem__('request-error-list', json_module.dumps(rejected_info_list)) %}
{% do frontend_information_dict[slave_reference].__setitem__('request-error-list', json_module.dumps(rejected_info_list)) %}
{% endfor %}
{% for slave_reference, warning_info_list in warning_slave_information['warning-slave-dict'].items() %}
{% if slave_reference not in slave_information_dict %}
{% do slave_information_dict.__setitem__(slave_reference, {}) %}
{% for slave_reference, warning_info_list in warning_frontend_information['warning-frontend-dict'].items() %}
{% if slave_reference not in frontend_information_dict %}
{% do frontend_information_dict.__setitem__(slave_reference, {}) %}
{% endif %}
{% do slave_information_dict[slave_reference].__setitem__('warning-list', json_module.dumps(warning_info_list)) %}
{% do frontend_information_dict[slave_reference].__setitem__('warning-list', json_module.dumps(warning_info_list)) %}
{% endfor %}
{% for slave_reference, kedifa_dict in json_module.loads(slave_kedifa_information).items() %}
{% if slave_reference not in rejected_slave_information['rejected-slave-dict'] %}
{% if slave_reference not in slave_information_dict %}
{% do slave_information_dict.__setitem__(slave_reference, {}) %}
{% for slave_reference, kedifa_dict in json_module.loads(frontend_kedifa_information).items() %}
{% if slave_reference not in rejected_frontend_information['rejected-frontend-dict'] %}
{% if slave_reference not in frontend_information_dict %}
{% do frontend_information_dict.__setitem__(slave_reference, {}) %}
{% endif %}
{% do slave_information_dict[slave_reference].__setitem__('key-generate-auth-url', kedifa_dict['key-generate-auth-url']) %}
{% do slave_information_dict[slave_reference].__setitem__('key-upload-url', kedifa_dict['key-upload-url']) %}
{% do slave_information_dict[slave_reference].__setitem__('kedifa-caucase-url', kedifa_dict['kedifa-caucase-url']) %}
{% do frontend_information_dict[slave_reference].__setitem__('key-generate-auth-url', kedifa_dict['key-generate-auth-url']) %}
{% do frontend_information_dict[slave_reference].__setitem__('key-upload-url', kedifa_dict['key-upload-url']) %}
{% do frontend_information_dict[slave_reference].__setitem__('kedifa-caucase-url', kedifa_dict['kedifa-caucase-url']) %}
{% endif %}
{% endfor %}
[common-frontend-information]
{% for frontend_key, frontend_value in frontend_information.items() %}
{{ frontend_key }} = {{ frontend_value }}
{% for frontend_node_key, frontend_node_value in frontend_node_information.items() %}
{{ frontend_node_key }} = {{ frontend_node_value }}
{% endfor %}
# Publish information for each slave
{% set active_slave_instance_list = json_module.loads(active_slave_instance_dict['active-slave-instance-list']) %}
{% for slave_reference, slave_information in slave_information_dict.items() %}
{# Filter out destroyed, so not existing anymore, slaves #}
# Publish information for each frontend
{% set active_frontend_list = json_module.loads(active_frontend_instance_dict['active-frontend-list']) %}
{% for slave_reference, frontend_information in frontend_information_dict.items() %}
{# Filter out destroyed, so not existing anymore, frontends #}
{# Note: This functionality is not yet covered by tests, please modify with care #}
{% if slave_reference in active_slave_instance_list %}
{% if slave_reference in active_frontend_list %}
{% set publish_section_title = 'publish-%s' % slave_reference %}
{% do part_list.append(publish_section_title) %}
[{{ publish_section_title }}]
......@@ -70,12 +70,12 @@
recipe = slapos.cookbook:publish_failsafe
-error-status-file = {{ publish_failsafe_error }}/{{ slave_reference }}-error.status
-slave-reference = {{ slave_reference }}
{% set log_access_url = slave_information.pop('log-access-urls', None) %}
{% set log_access_url = frontend_information.pop('log-access-urls', None) %}
{% if log_access_url %}
{# sort_keys are important in order to avoid shuffling parameters on each run #}
log-access-url = {{ dumps(json_module.dumps(log_access_url, sort_keys=True)) }}
{% endif %}
{% for key, value in slave_information.items() %}
{% for key, value in frontend_information.items() %}
{{ key }} = {{ dumps(value) }}
{% endfor %}
{% endif %}
......
software/rapid-cdn/instance-master.cfg.in
View file @ 032443a4
This diff is collapsed.
software/rapid-cdn/instance-slave-list.cfg.in
View file @ 032443a4
......@@ -12,7 +12,6 @@
{%- set generic_instance_parameter_dict = { 'cache_access': cache_access, 'local_ipv4': instance_parameter_dict['ipv4-random'], 'http_port': configuration['plain_http_port'], 'https_port': configuration['port']} %}
{%- set slave_log_dict = {} %}
{%- set slave_instance_information_list = [] %}
{%- set slave_instance_list = instance_parameter_dict['slave-instance-list'] %}
{%- set frontend_list = json_module.loads(configuration['frontend-list']) %} {#- XXX Upward compatibility #}
{%- do frontend_list.extend(json_module.loads(configuration.get('extra_slave_instance_list', []))) %} {#- XXX Backward compatibility #}
{%- if master_key_download_url %}
......@@ -56,7 +55,7 @@ context =
{%- do configuration.__setitem__(key, ('' ~ configuration[key]).lower() in TRUE_VALUES) %}
{%- endfor %}
{#- Loop thought slave list to set up slaves #}
{%- for slave_instance in slave_instance_list %}
{%- for slave_instance in frontend_list %}
{#- Prepare slave parameters: #}
{#- * convert strings to booleans (as slapproxy and SlapOS Master differ a bit) #}
{#- * create real lists from string lists #}
......@@ -388,7 +387,7 @@ local_ipv4 = {{ dumps('' ~ instance_parameter_dict['ipv4-random']) }}
{%- if slave_type != 'redirect' %}
{%- do backend_slave_list.append(slave_instance) %}
{%- endif %}
{%- endfor %} {# Slave iteration ends for slave_instance in slave_instance_list #}
{%- endfor %} {# Slave iteration ends for slave_instance in frontend_list #}
{%- do part_list.append('slave-introspection') %}
{#- ############################################## #}
......
software/rapid-cdn/software.cfg
View file @ 032443a4
......@@ -75,8 +75,8 @@ profile_kedifa = ${profile-kedifa:target}
profile_logrotate_base = ${template-logrotate-base:output}
profile_monitor = ${monitor-template:output}
profile_monitor2 = ${monitor2-template:output}
profile_master_publish_slave_information = ${profile-master-publish-slave-information:target}
profile_slave_list = ${profile-slave-list:target}
profile_master_publish_frontend_information = ${profile-master-publish-frontend-information:target}
profile_frontend_list = ${profile-frontend-list:target}
# templates
template_backend_haproxy_configuration = ${template-backend-haproxy-configuration:target}
......@@ -147,7 +147,7 @@ url = ${:_profile_base_location_}/${:filename}
[profile-slave-list]
<= download-profile
[profile-master-publish-slave-information]
[profile-master-publish-frontend-information]
<= download-profile
[download-template]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7