Skip to content

slapos
slapos
Commit 9bf0ae40 authored by Łukasz Nowak's avatar Łukasz Nowak
Browse files
Options

software/rapid-cdn: Drop slapparameter_dict 

This is required simplification, as the cluster needs full control over
defaults on each level.
parent 32357948
Hide whitespace changes
Inline Side-by-side
Showing with 50 additions and 56 deletions
software/rapid-cdn/README.rst
View file @ 9bf0ae40
......@@ -394,8 +394,7 @@ Profile development guidelines
Keep the naming in instance profiles:
* ``software_parameter_dict`` for values coming from software
* ``instance_parameter_dict`` for **local** values generated by the instance, except ``configuration``
* ``slapparameter_dict`` for values coming from SlapOS Master
* ``instance_parameter_dict`` for **local** values generated by the instance, except ``configuration``, coming from the SlapOS Master
Instantiated cluster structure
------------------------------
......
software/rapid-cdn/buildout.hash.cfg
View file @ 9bf0ae40
......@@ -14,7 +14,7 @@
# not need these here).
[template]
filename = instance.cfg.in
md5sum = bbeb179b698975c1a88ea7604df6740e
md5sum = 0fa7eddb6d4574e1299b967765691e15
[profile-common]
filename = instance-common.cfg.in
......@@ -22,15 +22,15 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
[profile-frontend]
filename = instance-frontend.cfg.in
md5sum = bb8129cdd89632c3c3ce86556496cb0f
md5sum = e293575e125a1219ab640a9479dfd85f
[profile-master]
filename = instance-master.cfg.in
md5sum = 3006197ddce87bd92866b76b5ce8ce08
md5sum = b225c7776cb4345f2b348b92d062bc1b
[profile-slave-list]
filename = instance-slave-list.cfg.in
md5sum = b75e42233c1b7bdd5f21971ed8907efc
md5sum = 546584898970e603ac20d576e22dec88
[profile-master-publish-slave-information]
filename = instance-master-publish-slave-information.cfg.in
......@@ -102,7 +102,7 @@ md5sum = e82ccdb0b26552a1c88ff523d8fae24a
[profile-kedifa]
filename = instance-kedifa.cfg.in
md5sum = 669da915003122e48646dc75fec239a5
md5sum = b9f1210005bcfff01235255f43d0b088
[template-frontend-haproxy-rsyslogd-conf]
_update_hash_filename_ = templates/frontend-haproxy-rsyslogd.conf.in
......
software/rapid-cdn/instance-frontend.cfg.in
View file @ 9bf0ae40
......@@ -244,7 +244,6 @@ recipe = slapos.recipe.template:jinja2
output = ${buildout:directory}/${:filename}
extensions = jinja2.ext.do
extra-context =
slapparameter_dict = {{ dumps(slapparameter_dict) }}
slap_software_type = {{ dumps(instance_parameter_dict['slap-software-type']) }}
context =
import json_module json
......@@ -252,7 +251,6 @@ context =
raw profile_logrotate_base {{ software_parameter_dict['profile_logrotate_base'] }}
raw profile_monitor {{ software_parameter_dict['profile_monitor'] }}
key slap_software_type :slap_software_type
key slapparameter_dict :slapparameter_dict
section directory directory
${:extra-context}
......@@ -276,10 +274,10 @@ crl = ${:d}/kedifa-login-crl.pem
[kedifa-login-csr]
recipe = plone.recipe.command
organization = {{ slapparameter_dict['cluster-identification'] }}
organizational_unit = {{ instance_parameter_dict['configuration.frontend-name'] }}
organization = {{ instance_parameter_dict['configuration']['cluster-identification'] }}
organizational_unit = {{ instance_parameter_dict['configuration']['frontend-name'] }}
command =
{% if slapparameter_dict['kedifa-caucase-url'] %}
{% if instance_parameter_dict['configuration']['kedifa-caucase-url'] %}
if [ ! -f ${:template-csr} ] && [ ! -f ${:key} ] ; then
{{ software_parameter_dict['openssl'] }} req -new -sha256 \
-newkey rsa:2048 -nodes -keyout ${:key} \
......@@ -298,7 +296,7 @@ stop-on-error = True
prefix='caucase-updater',
buildout_bin_directory=software_parameter_dict['bin_directory'],
updater_path='${directory:service}/kedifa-login-certificate-caucase-updater',
url=slapparameter_dict['kedifa-caucase-url'],
url=instance_parameter_dict['configuration']['kedifa-caucase-url'],
data_dir='${directory:srv}/caucase-updater',
crt_path='${kedifa-login-config:certificate}',
ca_path='${kedifa-login-config:ca-certificate}',
......@@ -308,7 +306,7 @@ stop-on-error = True
)}}
[kedifa-configuration]
caucase-url = {{ slapparameter_dict['kedifa-caucase-url'] }}
caucase-url = {{ instance_parameter_dict['configuration']['kedifa-caucase-url'] }}
ca-certificate = ${kedifa-login-config:ca-certificate}
certificate = ${kedifa-login-config:certificate}
cas-ca-certificate = ${kedifa-login-config:cas-ca-certificate}
......@@ -316,7 +314,7 @@ csr = ${caucase-updater-csr:csr}
crl = ${kedifa-login-config:crl}
kedifa-updater-mapping-file = ${directory:etc}/kedifa_updater_mapping.txt
kedifa-updater-state-file = ${directory:srv}/kedifa_updater_state.json
slave_kedifa_information = {{ dumps(slapparameter_dict['slave-kedifa-information']) }}
slave_kedifa_information = {{ dumps(instance_parameter_dict['configuration']['slave-kedifa-information']) }}
[backend-client-login-config]
d = ${directory:backend-client-dir}
......@@ -329,10 +327,10 @@ crl = ${:d}/crl.pem
[backend-client-login-csr]
recipe = plone.recipe.command
organization = {{ slapparameter_dict['cluster-identification'] }}
organization = {{ instance_parameter_dict['configuration']['cluster-identification'] }}
organizational_unit = {{ instance_parameter_dict['configuration.frontend-name'] }}
command =
{% if slapparameter_dict['backend-client-caucase-url'] %}
{% if instance_parameter_dict['configuration']['backend-client-caucase-url'] %}
if [ ! -f ${:template-csr} ] && [ ! -f ${:key} ] ; then
{{ software_parameter_dict['openssl'] }} req -new -sha256 \
-newkey rsa:2048 -nodes -keyout ${:key} \
......@@ -351,7 +349,7 @@ stop-on-error = True
prefix='backend-client-caucase-updater',
buildout_bin_directory=software_parameter_dict['bin_directory'],
updater_path='${directory:service}/backend-client-login-certificate-caucase-updater',
url=slapparameter_dict['backend-client-caucase-url'],
url=instance_parameter_dict['configuration']['backend-client-caucase-url'],
data_dir='${directory:srv}/backend-client-caucase-updater',
crt_path='${backend-client-login-config:certificate}',
ca_path='${backend-client-login-config:ca-certificate}',
......@@ -365,11 +363,11 @@ stop-on-error = True
depends = ${software-py:recipe}
url = {{ software_parameter_dict['profile_slave_list'] }}
filename = instance-slave-list.cfg
master_key_download_url = {{ dumps(slapparameter_dict['master-key-download-url']) }}
master_key_download_url = {{ dumps(instance_parameter_dict['configuration']['master-key-download-url']) }}
software_type = single-custom-personal
organization = {{ slapparameter_dict['cluster-identification'] }}
organization = {{ instance_parameter_dict['configuration']['cluster-identification'] }}
organizational-unit = {{ instance_parameter_dict['configuration.frontend-name'] }}
backend-client-caucase-url = {{ slapparameter_dict['backend-client-caucase-url'] }}
backend-client-caucase-url = {{ instance_parameter_dict['configuration']['backend-client-caucase-url'] }}
url-ready-file = ${directory:var}/url-ready.txt
http3-enable = {{ FRONTEND_HTTP3 }}
extra-context =
......@@ -808,7 +806,7 @@ graceful-command = ${backend-haproxy-validate:output} && kill -USR2 $(cat ${:pid
http-port = ${configuration:backend-haproxy-http-port}
https-port = ${configuration:backend-haproxy-https-port}
# Caucase related configuration
caucase-url = {{ slapparameter_dict['backend-client-caucase-url'] }}
caucase-url = {{ instance_parameter_dict['configuration']['backend-client-caucase-url'] }}
ca-certificate = ${backend-client-login-config:ca-certificate}
certificate = ${backend-client-login-config:certificate}
cas-ca-certificate = ${backend-client-login-config:cas-ca-certificate}
......@@ -819,7 +817,7 @@ statistic-certificate = ${self-signed-ip-access:certificate}
statistic-port = ${configuration:backend-haproxy-statistic-port}
statistic-username = ${monitor-instance-parameter:username}
statistic-password = ${monitor-htpasswd:passwd}
statistic-identification = {{ instance_parameter_dict['configuration.frontend-name'] + ' @ ' + slapparameter_dict['cluster-identification'] }}
statistic-identification = {{ instance_parameter_dict['configuration.frontend-name'] + ' @ ' + instance_parameter_dict['configuration']['cluster-identification'] }}
statistic-frontend-secure_access = ${backend-haproxy-statistic-frontend:connection-secure_access}
version-hash = ${version-hash:value}
node-id = ${frontend-node-id:value}
......
software/rapid-cdn/instance-kedifa.cfg.in
View file @ 9bf0ae40
......@@ -84,7 +84,7 @@ expose-csr-var = ${:var}/expose-csr
[kedifa-csr]
recipe = plone.recipe.command
organization = {{ slapparameter_dict['cluster-identification'] }}
organization = {{ instance_parameter_dict['configuration']['cluster-identification'] }}
organizational_unit = Kedifa Partition
command =
if [ ! -f ${:template-csr} ] && [ ! -f ${:key} ] ; then
......@@ -188,13 +188,11 @@ init =
recipe = slapos.recipe.template:jinja2
output = ${buildout:directory}/${:filename}
extra-context =
slapparameter_dict = {{ dumps(slapparameter_dict) }}
slap_software_type = {{ dumps(instance_parameter_dict['slap-software-type']) }}
context =
import json_module json
raw profile_common {{ software_parameter_dict['profile_common'] }}
key slap_software_type :slap_software_type
key slapparameter_dict :slapparameter_dict
section directory directory
${:extra-context}
......@@ -252,7 +250,7 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
# Publish KeDiFa configuration for upload and download for each slave
{%- set slave_kedifa_information = {} -%}
{%- for slave in slapparameter_dict['slave-list'] -%}
{%- for slave in instance_parameter_dict['configuration']['slave-list'] -%}
{%- set slave_reference = slave['slave_reference'] -%}
{%- set slave_dict = {} -%}
{%- do slave_dict.__setitem__('key-generate-auth-url', 'https://[${kedifa-config:ip}]:${kedifa-config:port}/${%s-auth-random:passwd}/generateauth' % (slave_reference,)) -%}
......
software/rapid-cdn/instance-master.cfg.in
View file @ 9bf0ae40
......@@ -101,8 +101,8 @@
]
%}
{%- set FRONTEND_NODE_SLAVE_PASSED_KEY_LIST = FRONTEND_NODE_SLAVE_PASSED_KEY_LIST_SCHEMA + FRONTEND_NODE_SLAVE_PASSED_KEY_LIST_INTERNAL %}
{% set aikc_enabled = slapparameter_dict.get('automatic-internal-kedifa-caucase-csr', 'true').lower() in TRUE_VALUES %}
{% set aibcc_enabled = slapparameter_dict.get('automatic-internal-backend-client-caucase-csr', 'true').lower() in TRUE_VALUES %}
{% set aikc_enabled = instance_parameter_dict['configuration'].get('automatic-internal-kedifa-caucase-csr', 'true').lower() in TRUE_VALUES %}
{% set aibcc_enabled = instance_parameter_dict['configuration'].get('automatic-internal-backend-client-caucase-csr', 'true').lower() in TRUE_VALUES %}
{# Ports 8401, 8402 and 8410+1..N are reserved for monitor ports on various partitions #}
{% set master_partition_monitor_monitor_httpd_port = 8401 %}
{% set kedifa_partition_monitor_httpd_port = 8402 %}
......@@ -123,19 +123,19 @@ context =
{% set part_list = [] %}
{% set single_type_key = 'single-' %}
{% set frontend_type = "%s%s" % (single_type_key, 'custom-personal') %}
{% set frontend_quantity = slapparameter_dict.pop('-frontend-quantity', '1') | int %}
{% set frontend_quantity = instance_parameter_dict['configuration'].pop('-frontend-quantity', '1') | int %}
{% set slave_list_name = 'extra_slave_instance_list' %}
{% set frontend_list = [] %}
{% set frontend_section_list = [] %}
{% set request_dict = {} %}
# XXX Dirty hack, not possible to define default value before
{% set sla_computer_1_key = '-sla-1-computer_guid' %}
{% if not sla_computer_1_key in slapparameter_dict %}
{% do slapparameter_dict.__setitem__(sla_computer_1_key, '${slap-connection:computer-id}') %}
{% if not sla_computer_1_key in instance_parameter_dict['configuration'] %}
{% do instance_parameter_dict['configuration'].__setitem__(sla_computer_1_key, '${slap-connection:computer-id}') %}
{% endif %}
{% set sla_computer_kedifa_key = '-sla-kedifa-computer_guid' %}
{% if not sla_computer_kedifa_key in slapparameter_dict %}
{% do slapparameter_dict.__setitem__(sla_computer_kedifa_key, '${slap-connection:computer-id}') %}
{% if not sla_computer_kedifa_key in instance_parameter_dict['configuration'] %}
{% do instance_parameter_dict['configuration'].__setitem__(sla_computer_kedifa_key, '${slap-connection:computer-id}') %}
{% endif %}
# Here we request individually each frontend.
......@@ -153,29 +153,29 @@ context =
{% set config_key = "-frontend-config-%s-" % i %}
{% set config_key_length = config_key | length %}
{% set config_dict = {} %}
{% for key in list(slapparameter_dict.keys()) %}
{% for key in list(instance_parameter_dict['configuration'].keys()) %}
{% if key.startswith(sla_key) %}
{% do sla_dict.__setitem__(key[sla_key_length:], slapparameter_dict.pop(key)) %}
{% do sla_dict.__setitem__(key[sla_key_length:], instance_parameter_dict['configuration'].pop(key)) %}
# We check for specific configuration regarding the frontend
{% elif key.startswith(config_key) %}
{% set node_config_key = key[config_key_length:] %}
{% if node_config_key in ('enable-http3', 'http3-port') %}
{% do config_dict.__setitem__(node_config_key, slapparameter_dict.pop(key) or slapparameter_dict.get(node_config_key) or NODE_DEFAULT_KEY_VALUE[node_config_key]) %}
{% do config_dict.__setitem__(node_config_key, instance_parameter_dict['configuration'].pop(key) or instance_parameter_dict['configuration'].get(node_config_key) or NODE_DEFAULT_KEY_VALUE[node_config_key]) %}
{% else %}
{% do config_dict.__setitem__(node_config_key, slapparameter_dict.pop(key)) %}
{% do config_dict.__setitem__(node_config_key, instance_parameter_dict['configuration'].pop(key)) %}
{% endif %}
{% endif %}
{% endfor %}
{% if 'http3-port' not in config_dict %}
{% do config_dict.__setitem__('http3-port', slapparameter_dict.get('http3-port') or NODE_DEFAULT_KEY_VALUE['http3-port']) %}
{% do config_dict.__setitem__('http3-port', instance_parameter_dict['configuration'].get('http3-port') or NODE_DEFAULT_KEY_VALUE['http3-port']) %}
{% endif %}
{% if 'enable-http3' not in config_dict %}
{% do config_dict.__setitem__('enable-http3', slapparameter_dict.get('enable-http3') or NODE_DEFAULT_KEY_VALUE['enable-http3']) %}
{% do config_dict.__setitem__('enable-http3', instance_parameter_dict['configuration'].get('enable-http3') or NODE_DEFAULT_KEY_VALUE['enable-http3']) %}
{% endif %}
{% do config_dict.__setitem__('monitor-httpd-port', frontend_monitor_httpd_base_port + i) %}
{% do config_dict.__setitem__('backend-client-caucase-url', caucase_url) %}
{% set state_key = "-frontend-%s-state" % i %}
{% set frontend_state = slapparameter_dict.pop(state_key, None) %}
{% set frontend_state = instance_parameter_dict['configuration'].pop(state_key, None) %}
{% if frontend_state != 'destroyed' %}
{% do frontend_list.append(frontend_name) %}
{% do frontend_section_list.append(request_section_title) %}
......@@ -189,7 +189,7 @@ context =
'state': frontend_state
} %}
{% set frontend_software_url_key = "-frontend-%s-software-release-url" % i %}
{% do request_content_dict.__setitem__('software-url', slapparameter_dict.get(frontend_software_url_key) or '${slap-connection:software-release-url}') %}
{% do request_content_dict.__setitem__('software-url', instance_parameter_dict['configuration'].get(frontend_software_url_key) or '${slap-connection:software-release-url}') %}
{% do request_dict.__setitem__(request_section_title, request_content_dict) %}
{% endfor %}
......@@ -382,7 +382,7 @@ monitor-httpd-port = {{ master_partition_monitor_monitor_httpd_port }}
[replicate]
<= slap-connection
recipe = slapos.cookbook:requestoptional.serialised
config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
config-monitor-cors-domains = {{ instance_parameter_dict['configuration'].get('monitor-cors-domains', 'monitor.app.officejs.com') }}
config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-password = ${monitor-htpasswd:passwd}
......@@ -392,8 +392,8 @@ return = slave-instance-information-list monitor-base-url backend-client-csr-url
{#- Send only needed parameters to frontend nodes #}
{%- set base_node_configuration_dict = {} %}
{%- for key in FRONTEND_NODE_PASSED_KEY_LIST %}
{%- if key in slapparameter_dict %}
{%- do base_node_configuration_dict.__setitem__(key, slapparameter_dict[key]) %}
{%- if key in instance_parameter_dict['configuration'] %}
{%- do base_node_configuration_dict.__setitem__(key, instance_parameter_dict['configuration'][key]) %}
{%- endif %}
{%- endfor %}
{% for section, frontend_request in request_dict.items() %}
......@@ -436,7 +436,7 @@ return =
{% set warning_list = [] %}
{% for key in ['apache-certificate', 'apache-key'] %}
{% if key in slapparameter_dict %}
{% if key in instance_parameter_dict['configuration'] %}
{% do warning_list.append('%s is obsolete, please use master-key-upload-url' % (key, )) %}
{% endif %}
{% endfor %}
......@@ -444,7 +444,7 @@ return =
[publish-information]
<= monitor-publish
recipe = slapos.cookbook:publish
domain = {{ slapparameter_dict.get('domain') }}
domain = {{ instance_parameter_dict['configuration'].get('domain') }}
slave-amount = {{ instance_parameter_dict['slave-instance-list'] | length }}
accepted-slave-amount = {{ authorized_slave_list | length }}
rejected-slave-amount = {{ rejected_slave_dict | length }}
......@@ -517,21 +517,21 @@ custom-group = instance-publish-slave-information:output
[request-kedifa]
<= slap-connection
recipe = slapos.cookbook:requestoptional.serialised
config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
config-monitor-cors-domains = {{ instance_parameter_dict['configuration'].get('monitor-cors-domains', 'monitor.app.officejs.com') }}
config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-password = ${monitor-htpasswd:passwd}
config-monitor-httpd-port = {{ kedifa_partition_monitor_httpd_port }}
{% for key in ['kedifa_port', 'caucase_port'] -%}
{%- if key in slapparameter_dict %}
config-{{ key }} = {{ dumps(slapparameter_dict[key]) }}
{%- if key in instance_parameter_dict['configuration'] %}
config-{{ key }} = {{ dumps(instance_parameter_dict['configuration'][key]) }}
{%- endif %}
{%- endfor %}
config-slave-list = {{ dumps(authorized_slave_list) }}
config-cluster-identification = {{ instance_parameter_dict['root-instance-title'] }}
{% set software_url_key = "-kedifa-software-release-url" %}
{% if software_url_key in slapparameter_dict %}
software-url = {{ slapparameter_dict.pop(software_url_key) }}
{% if software_url_key in instance_parameter_dict['configuration'] %}
software-url = {{ instance_parameter_dict['configuration'].pop(software_url_key) }}
{% else %}
software-url = ${slap-connection:software-release-url}
{% endif %}
......@@ -540,9 +540,9 @@ name = kedifa
return = slave-kedifa-information master-key-generate-auth-url master-key-upload-url master-key-download-url caucase-url kedifa-csr-url csr-certificate monitor-base-url
{% set sla_kedifa_key = "-sla-kedifa-" %}
{% set sla_kedifa_key_length = sla_kedifa_key | length %}
{% for key in list(slapparameter_dict.keys()) %}
{% for key in list(instance_parameter_dict['configuration'].keys()) %}
{% if key.startswith(sla_kedifa_key) %}
sla-{{ key[sla_kedifa_key_length:] }} = {{ slapparameter_dict.pop(key) }}
sla-{{ key[sla_kedifa_key_length:] }} = {{ instance_parameter_dict['configuration'].pop(key) }}
{% endif %}
{% endfor %}
......
software/rapid-cdn/instance-slave-list.cfg.in
View file @ 9bf0ae40
......@@ -222,7 +222,7 @@ context =
{#- Set slave domain if none was defined #}
{%- if slave_instance.get('custom_domain', None) == None %}
{%- set domain_prefix = slave_instance.get('slave_reference').replace("-", "").replace("_", "").lower() %}
{%- do slave_instance.__setitem__('custom_domain', "%s.%s" % (domain_prefix, slapparameter_dict.get('domain'))) %}
{%- do slave_instance.__setitem__('custom_domain', "%s.%s" % (domain_prefix, instance_parameter_dict['configuration'].get('domain'))) %}
{%- endif %}
{%- do slave_publish_dict.__setitem__('domain', slave_instance.get('custom_domain')) %}
{%- do slave_publish_dict.__setitem__('url', "http://%s" % slave_instance.get('custom_domain')) %}
......
software/rapid-cdn/instance.cfg.in
View file @ 9bf0ae40
......@@ -15,7 +15,6 @@ extensions = jinja2.ext.do
extra-context =
context =
import json_module json
key slapparameter_dict slap-configuration:configuration
section instance_parameter_dict slap-configuration
section software_parameter_dict software-parameter-section
${:extra-context}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7