Commit 39aaccab authored by Łukasz Nowak's avatar Łukasz Nowak

XXX rapid-cdn: Clean up instance-master.cfg.in

Cleanups:

 * dropped needless definition of aibcc_enabled
 * renamed master_partition_monitor_monitor_httpd_port to
   MASTER_PARTITION_MONITOR_HTTPD_PORT
 * convert buildout comments to Jinja2 comments
 * strip whitespaces, so that generated file is readable
 * correctly indent on Jinja2 not on buildout
 * drop obvious comments
 * minimize amount of variables, by dropping used only once
 * pass all modules with _module suffix
 * avoid passing needless software_type
 * group definition of all globals into one place
 * name all globals with UPPERCASE

XXX: Clean up until it will be possible to modify it a lot
parent c233dda5
Pipeline #35603 failed with stage
in 0 seconds
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[template] [template]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = a7d4304064abb7bbac413aa4f90bd6b9 md5sum = 75513fafea441a7fced667bf9fcbdd61
[profile-common] [profile-common]
filename = instance-common.cfg.in filename = instance-common.cfg.in
...@@ -26,7 +26,7 @@ md5sum = b590577a715632f1befdf1a503780a35 ...@@ -26,7 +26,7 @@ md5sum = b590577a715632f1befdf1a503780a35
[profile-master] [profile-master]
filename = instance-master.cfg.in filename = instance-master.cfg.in
md5sum = b7e016a3fc7734a18b4a7b4a48952f4b md5sum = 38aac4e4c0681a065c1fffe4768d3d7b
[profile-slave-list] [profile-slave-list]
filename = instance-slave-list.cfg.in filename = instance-slave-list.cfg.in
......
{% set aibcc_enabled = True %} {%- import "caucase" as CAUCASE with context %}
{% import "caucase" as caucase with context %} {#- BEGIN: Definition of global variables of the profile #}
{#- DANGER! DANGER! #} {%- set NAME_BASE = 'caddy-frontend' %} {#- Hardcoded name of the requested frontend node. Attention: Changing it will result with requesting new nodes. #}
{#- Avoid touching the NAME_BASE, as it will result with backward incompatible cluster setup #}
{%- set NAME_BASE = 'caddy-frontend' %}
{#- DANGER! DANGER! #}
{%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%} {%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
{%- set GOOD_CIPHER_LIST = [ {%- set GOOD_CIPHER_LIST = [
'ECDHE-ECDSA-AES256-GCM-SHA384', 'ECDHE-ECDSA-AES256-GCM-SHA384',
...@@ -26,7 +23,6 @@ ...@@ -26,7 +23,6 @@
{%- for key, value in CIPHER_TRANSLATION_DICT.items() %} {%- for key, value in CIPHER_TRANSLATION_DICT.items() %}
{%- do GOOD_CIPHER_LIST.append(value) %} {%- do GOOD_CIPHER_LIST.append(value) %}
{%- endfor %} {%- endfor %}
{#- Allow to pass only some parameters to frontend nodes #}
{%- set FRONTEND_PARAMETER_DICT = { {%- set FRONTEND_PARAMETER_DICT = {
'apache-certificate': instance_parameter_dict['configuration'].get('apache-certificate', CLUSTER_DEFAULTS['apache-certificate']), 'apache-certificate': instance_parameter_dict['configuration'].get('apache-certificate', CLUSTER_DEFAULTS['apache-certificate']),
'apache-key': instance_parameter_dict['configuration'].get('apache-key', CLUSTER_DEFAULTS['apache-key']), 'apache-key': instance_parameter_dict['configuration'].get('apache-key', CLUSTER_DEFAULTS['apache-key']),
...@@ -48,15 +44,14 @@ ...@@ -48,15 +44,14 @@
] ]
%} %}
{%- set FRONTEND_NODE_SLAVE_PASSED_KEY_LIST = FRONTEND_NODE_SLAVE_PASSED_KEY_LIST_SCHEMA + FRONTEND_NODE_SLAVE_PASSED_KEY_LIST_INTERNAL %} {%- set FRONTEND_NODE_SLAVE_PASSED_KEY_LIST = FRONTEND_NODE_SLAVE_PASSED_KEY_LIST_SCHEMA + FRONTEND_NODE_SLAVE_PASSED_KEY_LIST_INTERNAL %}
{% set aikc_enabled = instance_parameter_dict['configuration'].get('automatic-internal-kedifa-caucase-csr', CLUSTER_DEFAULTS['automatic-internal-kedifa-caucase-csr'] | string).lower() in TRUE_VALUES %} {%- set AIKC_ENABLED = instance_parameter_dict['configuration'].get('automatic-internal-kedifa-caucase-csr', CLUSTER_DEFAULTS['automatic-internal-kedifa-caucase-csr'] | string).lower() in TRUE_VALUES %}
{% set aibcc_enabled = instance_parameter_dict['configuration'].get('automatic-internal-backend-client-caucase-csr', CLUSTER_DEFAULTS['automatic-internal-backend-client-caucase-csr'] | string).lower() in TRUE_VALUES %} {%- set AIBCC_ENABLED = instance_parameter_dict['configuration'].get('automatic-internal-backend-client-caucase-csr', CLUSTER_DEFAULTS['automatic-internal-backend-client-caucase-csr'] | string).lower() in TRUE_VALUES %}
{# Ports 8401, 8402 and 8410+1..N are reserved for monitor ports on various partitions #} {#- Ports 8401, 8402 and 8410+1..N are reserved for monitor ports on various partitions #}
{% set master_partition_monitor_monitor_httpd_port = 8401 %} {%- set MASTER_PARTITION_MONITOR_HTTPD_PORT = 8401 %}
{% set kedifa_partition_monitor_httpd_port = 8402 %} {%- set KEDIFA_PARTITION_MONITOR_HTTPD_PORT = 8402 %}
{% set frontend_monitor_httpd_base_port = 8410 %} {%- set FRONTEND_MONITOR_HTTPD_PORT = 8410 %}
{% set caucase_host = '[' ~ instance_parameter_dict['ipv6-random'] ~ ']' %} {%- set CAUCASE_NETLOC = '[' ~ instance_parameter_dict['ipv6-random'] ~ ']' ~ ':' ~ instance_parameter_dict['configuration'].get('caucase_backend_client_port', '8990') %}
{% set caucase_netloc = caucase_host ~ ':' ~ instance_parameter_dict['configuration'].get('caucase_backend_client_port', '8990') %} {%- set CAUCASE_URL = 'http://' ~ CAUCASE_NETLOC %}
{% set caucase_url = 'http://' ~ caucase_netloc %}
[jinja2-template-base] [jinja2-template-base]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
output = ${buildout:directory}/${:filename} output = ${buildout:directory}/${:filename}
...@@ -66,27 +61,22 @@ context = ...@@ -66,27 +61,22 @@ context =
raw profile_common {{ software_parameter_dict['profile_common'] }} raw profile_common {{ software_parameter_dict['profile_common'] }}
${:extra-context} ${:extra-context}
{% set popen = functools_module.partial(subprocess_module.Popen, stdout=subprocess_module.PIPE, stderr=subprocess_module.STDOUT, stdin=subprocess_module.PIPE) %} {%- set POPEN_FUNCTION = functools_module.partial(subprocess_module.Popen, stdout=subprocess_module.PIPE, stderr=subprocess_module.STDOUT, stdin=subprocess_module.PIPE) %}
{% set part_list = [] %} {%- set PART_LIST = [] %}
{% set single_type_key = 'single-' %} {%- set FRONTEND_LIST = [] %}
{% set frontend_type = "%s%s" % (single_type_key, 'custom-personal') %} {%- set FRONTEND_SECTION_LIST = [] %}
{% set frontend_quantity = instance_parameter_dict['configuration'].pop('-frontend-quantity', CLUSTER_DEFAULTS['-frontend-quantity']) | int %} {%- set REQUEST_DICT = {} %}
{% set frontend_list = [] %} {#- # XXX Dirty hack, not possible to define default value before #}
{% set frontend_section_list = [] %} {%- if not '-sla-1-computer_guid' in instance_parameter_dict['configuration'] %}
{% set request_dict = {} %} {%- do instance_parameter_dict['configuration'].__setitem__('-sla-1-computer_guid', '${slap-connection:computer-id}') %}
# XXX Dirty hack, not possible to define default value before {%- endif %}
{% set sla_computer_1_key = '-sla-1-computer_guid' %} {%- if not '-sla-kedifa-computer_guid' in instance_parameter_dict['configuration'] %}
{% if not sla_computer_1_key in instance_parameter_dict['configuration'] %} {%- do instance_parameter_dict['configuration'].__setitem__('-sla-kedifa-computer_guid', '${slap-connection:computer-id}') %}
{% do instance_parameter_dict['configuration'].__setitem__(sla_computer_1_key, '${slap-connection:computer-id}') %} {%- endif %}
{% endif %}
{% set sla_computer_kedifa_key = '-sla-kedifa-computer_guid' %} {#- Here we request individually each frontend.
{% if not sla_computer_kedifa_key in instance_parameter_dict['configuration'] %} The presence of sla parameters is checked and added if found #}
{% do instance_parameter_dict['configuration'].__setitem__(sla_computer_kedifa_key, '${slap-connection:computer-id}') %} {%- set NODE_SPECIFIC_PARAMETER_DICT = {
{% endif %}
# Here we request individually each frontend.
# The presence of sla parameters is checked and added if found
{% set NODE_SPECIFIC_PARAMETER_DICT = {
'backend-haproxy-http-port': 21080, 'backend-haproxy-http-port': 21080,
'backend-haproxy-https-port': 21443, 'backend-haproxy-https-port': 21443,
'backend-haproxy-statistic-port': 21444, 'backend-haproxy-statistic-port': 21444,
...@@ -99,239 +89,243 @@ context = ...@@ -99,239 +89,243 @@ context =
'request-timeout': CLUSTER_DEFAULTS['request-timeout'], 'request-timeout': CLUSTER_DEFAULTS['request-timeout'],
'slave-introspection-https-port': 22443, 'slave-introspection-https-port': 22443,
} %} } %}
{% for i in range(1, frontend_quantity + 1) %} {%- set AUTHORIZED_SLAVE_LIST = [] %}
{% set frontend_name = "%s-%s" % (NAME_BASE, i) %} {%- set REJECTED_SLAVE_DICT = {} %}
{% set request_section_title = 'request-%s' % frontend_name %} {%- set CRITICAL_REJECTED_SLAVE_DICT = {} %}
{% set sla_key = "-sla-%s-" % i %} {%- set WARNING_SLAVE_DICT = {} %}
{% set sla_key_length = sla_key | length %} {%- set USED_HOST_LIST = [] %}
{% set sla_dict = {} %} {#- END: Definition of global variables of the profile #}
{% set config_key = "-frontend-config-%s-" % i %} {#- BEGIN: REQUEST_DICT preparation #}
{% set config_key_length = config_key | length %} {%- for i in range(1, (instance_parameter_dict['configuration'].pop('-frontend-quantity', CLUSTER_DEFAULTS['-frontend-quantity']) | int) + 1) %}
{% set config_dict = {} %} {%- set frontend_name = "%s-%s" % (NAME_BASE, i) %}
{% for key in list(instance_parameter_dict['configuration'].keys()) %} {%- set request_section_title = 'request-%s' % frontend_name %}
{% if key.startswith(sla_key) %} {%- set sla_key = "-sla-%s-" % i %}
{% do sla_dict.__setitem__(key[sla_key_length:], instance_parameter_dict['configuration'].pop(key)) %} {%- set sla_key_length = sla_key | length %}
# We check for specific configuration regarding the frontend {%- set sla_dict = {} %}
{% elif key.startswith(config_key) %} {%- set config_key = "-frontend-config-%s-" % i %}
{% set node_config_key = key[config_key_length:] %} {%- set config_key_length = config_key | length %}
{% if node_config_key in NODE_SPECIFIC_PARAMETER_DICT.keys() %} {%- set config_dict = {} %}
{% do config_dict.__setitem__(node_config_key, instance_parameter_dict['configuration'].pop(key) or instance_parameter_dict['configuration'].get(node_config_key) or NODE_SPECIFIC_PARAMETER_DICT[node_config_key]) %} {%- for key in list(instance_parameter_dict['configuration'].keys()) %}
{% else %} {%- if key.startswith(sla_key) %}
{% do config_dict.__setitem__(node_config_key, instance_parameter_dict['configuration'].pop(key)) %} {%- do sla_dict.__setitem__(key[sla_key_length:], instance_parameter_dict['configuration'].pop(key)) %}
{% endif %} {#- We check for specific configuration regarding the frontend #}
{% endif %} {%- elif key.startswith(config_key) %}
{% endfor %} {%- set node_config_key = key[config_key_length:] %}
{% for node_config_key in NODE_SPECIFIC_PARAMETER_DICT.keys() %} {%- if node_config_key in NODE_SPECIFIC_PARAMETER_DICT.keys() %}
{% if node_config_key not in config_dict %} {%- do config_dict.__setitem__(node_config_key, instance_parameter_dict['configuration'].pop(key) or instance_parameter_dict['configuration'].get(node_config_key) or NODE_SPECIFIC_PARAMETER_DICT[node_config_key]) %}
{% do config_dict.__setitem__(node_config_key, instance_parameter_dict['configuration'].get(node_config_key) or NODE_SPECIFIC_PARAMETER_DICT[node_config_key]) %} {%- else %}
{% endif %} {%- do config_dict.__setitem__(node_config_key, instance_parameter_dict['configuration'].pop(key)) %}
{% endfor %} {%- endif %}
{% do config_dict.__setitem__('monitor-httpd-port', frontend_monitor_httpd_base_port + i) %} {%- endif %}
{% do config_dict.__setitem__('backend-client-caucase-url', caucase_url) %} {%- endfor %}
{% set state_key = "-frontend-%s-state" % i %} {%- for node_config_key in NODE_SPECIFIC_PARAMETER_DICT.keys() %}
{% set frontend_state = instance_parameter_dict['configuration'].pop(state_key, None) %} {%- if node_config_key not in config_dict %}
{% if frontend_state != 'destroyed' %} {%- do config_dict.__setitem__(node_config_key, instance_parameter_dict['configuration'].get(node_config_key) or NODE_SPECIFIC_PARAMETER_DICT[node_config_key]) %}
{% do frontend_list.append(frontend_name) %} {%- endif %}
{% do frontend_section_list.append(request_section_title) %} {%- endfor %}
{% endif %} {%- do config_dict.__setitem__('monitor-httpd-port', FRONTEND_MONITOR_HTTPD_PORT + i) %}
{% do part_list.append(request_section_title) %} {%- do config_dict.__setitem__('backend-client-caucase-url', CAUCASE_URL) %}
# Filling request dict for slave {%- set state_key = "-frontend-%s-state" % i %}
{% set request_content_dict = { {%- set frontend_state = instance_parameter_dict['configuration'].pop(state_key, None) %}
{%- if frontend_state != 'destroyed' %}
{%- do FRONTEND_LIST.append(frontend_name) %}
{%- do FRONTEND_SECTION_LIST.append(request_section_title) %}
{%- endif %}
{%- do PART_LIST.append(request_section_title) %}
{#- Filling request dict for slave #}
{%- set request_content_dict = {
'config': config_dict, 'config': config_dict,
'name': frontend_name, 'name': frontend_name,
'sla': sla_dict, 'sla': sla_dict,
'state': frontend_state 'state': frontend_state
} %} } %}
{% set frontend_software_url_key = "-frontend-%s-software-release-url" % i %} {%- set frontend_software_url_key = "-frontend-%s-software-release-url" % i %}
{% do request_content_dict.__setitem__('software-url', instance_parameter_dict['configuration'].get(frontend_software_url_key) or '${slap-connection:software-release-url}') %} {%- do request_content_dict.__setitem__('software-url', instance_parameter_dict['configuration'].get(frontend_software_url_key) or '${slap-connection:software-release-url}') %}
{% do request_dict.__setitem__(request_section_title, request_content_dict) %} {%- do REQUEST_DICT.__setitem__(request_section_title, request_content_dict) %}
{% endfor %} {%- endfor %}
{#- END: REQUEST_DICT preparation#}
{% set authorized_slave_string_list = [] %}
{% set authorized_slave_list = [] %} {#- BEGIN: Slave processing #}
{% set rejected_slave_dict = {} %} {%- for slave in sorted(instance_parameter_dict['slave-instance-list'], key=operator_module.itemgetter('slave_reference')) %}
{% set critical_rejected_slave_dict = {} %} {%- set slave_error_list = [] %}
{% set warning_slave_dict = {} %} {%- set slave_critical_error_list = [] %}
{% set used_host_list = [] %} {%- set slave_warning_list = [] %}
{% for slave in sorted(instance_parameter_dict['slave-instance-list'], key=operator_module.itemgetter('slave_reference')) %} {%- set slave_server_alias_unclashed = [] %}
{% set slave_error_list = [] %} {%- set slave_type = slave.get('type') %}
{% set slave_critical_error_list = [] %} {%- if slave_type not in [None, '', 'default', 'zope', 'redirect', 'notebook', 'websocket'] %}
{% set slave_warning_list = [] %} {%- do slave_error_list.append('type:%s is not supported' % (slave_type,)) %}
{% set slave_server_alias_unclashed = [] %} {%- endif %}
{% set slave_type = slave.get('type') %} {#- Check health-check-* #}
{% if slave_type not in [None, '', 'default', 'zope', 'redirect', 'notebook', 'websocket'] %} {%- set health_check = (str(slave.get('health-check', False)) or SLAVE_DEFAULTS['health-check']).lower() %}
{% do slave_error_list.append('type:%s is not supported' % (slave_type,)) %} {%- if health_check in TRUE_VALUES %}
{% endif %} {%- set health_check_http_method = slave.get('health-check-http-method') or SLAVE_DEFAULTS['health-check-http-method'] %}
{# Check health-check-* #} {%- if health_check_http_method not in ['GET', 'OPTIONS', 'CONNECT', 'POST'] %}
{% set health_check = (str(slave.get('health-check', False)) or SLAVE_DEFAULTS['health-check']).lower() %} {%- do slave_error_list.append('Wrong health-check-http-method %s' % (health_check_http_method,)) %}
{% if health_check in TRUE_VALUES %} {%- endif %}
{% set health_check_http_method = slave.get('health-check-http-method') or SLAVE_DEFAULTS['health-check-http-method'] %} {%- set health_check_http_path = slave.get('health-check-http-path') or SLAVE_DEFAULTS['health-check-http-path'] %}
{% if health_check_http_method not in ['GET', 'OPTIONS', 'CONNECT', 'POST'] %} {%- set health_check_http_version = slave.get('health-check-http-version') or SLAVE_DEFAULTS['health-check-http-version'] %}
{% do slave_error_list.append('Wrong health-check-http-method %s' % (health_check_http_method,)) %} {%- if health_check_http_version not in ['HTTP/1.1', 'HTTP/1.0'] %}
{% endif %} {%- do slave_error_list.append('Wrong health-check-http-version %s' % (health_check_http_version,)) %}
{% set health_check_http_path = slave.get('health-check-http-path') or SLAVE_DEFAULTS['health-check-http-path'] %} {%- endif %}
{% set health_check_http_version = slave.get('health-check-http-version') or SLAVE_DEFAULTS['health-check-http-version'] %} {%- set health_check_timeout = (slave.get('health-check-timeout') or SLAVE_DEFAULTS['health-check-timeout']) | int(false) %}
{% if health_check_http_version not in ['HTTP/1.1', 'HTTP/1.0'] %} {%- if health_check_timeout is false or health_check_timeout <= 0 %}
{% do slave_error_list.append('Wrong health-check-http-version %s' % (health_check_http_version,)) %} {%- do slave_error_list.append('Wrong health-check-timeout %s' % (slave.get('health-check-timeout'),)) %}
{% endif %} {%- endif %}
{% set health_check_timeout = (slave.get('health-check-timeout') or SLAVE_DEFAULTS['health-check-timeout']) | int(false) %} {%- set health_check_interval = (slave.get('health-check-interval') or SLAVE_DEFAULTS['health-check-interval']) | int(false) %}
{% if health_check_timeout is false or health_check_timeout <= 0 %} {%- if health_check_interval is false or health_check_interval <= 0 %}
{% do slave_error_list.append('Wrong health-check-timeout %s' % (slave.get('health-check-timeout'),)) %} {%- do slave_error_list.append('Wrong health-check-interval %s' % (slave.get('health-check-interval'),)) %}
{% endif %} {%- endif %}
{% set health_check_interval = (slave.get('health-check-interval') or SLAVE_DEFAULTS['health-check-interval']) | int(false) %} {%- set health_check_rise = (slave.get('health-check-rise') or SLAVE_DEFAULTS['health-check-rise']) | int(false) %}
{% if health_check_interval is false or health_check_interval <= 0 %} {%- if health_check_rise is false or health_check_rise <= 0 %}
{% do slave_error_list.append('Wrong health-check-interval %s' % (slave.get('health-check-interval'),)) %} {%- do slave_error_list.append('Wrong health-check-rise %s' % (slave.get('health-check-rise'),)) %}
{% endif %} {%- endif %}
{% set health_check_rise = (slave.get('health-check-rise') or SLAVE_DEFAULTS['health-check-rise']) | int(false) %} {%- set health_check_fall = (slave.get('health-check-fall') or SLAVE_DEFAULTS['health-check-fall']) | int(false) %}
{% if health_check_rise is false or health_check_rise <= 0 %} {%- if health_check_fall is false or health_check_fall <= 0 %}
{% do slave_error_list.append('Wrong health-check-rise %s' % (slave.get('health-check-rise'),)) %} {%- do slave_error_list.append('Wrong health-check-fall %s' % (slave.get('health-check-fall'),)) %}
{% endif %} {%- endif %}
{% set health_check_fall = (slave.get('health-check-fall') or SLAVE_DEFAULTS['health-check-fall']) | int(false) %} {%- endif %}
{% if health_check_fall is false or health_check_fall <= 0 %} {#- Check virtualhostroot-http-port and virtualhostroot-https-port #}
{% do slave_error_list.append('Wrong health-check-fall %s' % (slave.get('health-check-fall'),)) %} {%- for key in ['virtualhostroot-http-port', 'virtualhostroot-https-port'] %}
{% endif %} {%- set value = (slave.get(key) or SLAVE_DEFAULTS[key]) | int(false) %}
{% endif %} {%- if value is false or value < 0 %}
{# Check virtualhostroot-http-port and virtualhostroot-https-port #} {%- do slave_error_list.append('Wrong %s %r' % (key, slave.get(key))) %}
{% for key in ['virtualhostroot-http-port', 'virtualhostroot-https-port'] %} {%- endif %}
{% set value = (slave.get(key) or SLAVE_DEFAULTS[key]) | int(false) %} {%- endfor %}
{% if value is false or value < 0 %} {#- Check ciphers #}
{% do slave_error_list.append('Wrong %s %r' % (key, slave.get(key))) %} {%- set slave_cipher_list = (slave.get('ciphers') or '').strip().split() %}
{% endif %} {%- if len(slave_cipher_list) > 0 %}
{% endfor %} {%- for cipher in slave_cipher_list %}
{# Check ciphers #} {%- if cipher not in GOOD_CIPHER_LIST %}
{% set slave_cipher_list = (slave.get('ciphers') or '').strip().split() %} {%- if cipher in CIPHER_TRANSLATION_DICT %}
{% if len(slave_cipher_list) > 0 %} {#- Real translation happens in instance-slave-list.cfg.in #}
{% for cipher in slave_cipher_list %} {%- do slave_warning_list.append('Cipher %r translated to %r' % (cipher, CIPHER_TRANSLATION_DICT[cipher])) %}
{% if cipher not in GOOD_CIPHER_LIST %} {%- else %}
{% if cipher in CIPHER_TRANSLATION_DICT %} {%- do slave_error_list.append('Cipher %r is not supported.' % (cipher,)) %}
{# Real translation happens in instance-slave-list.cfg.in #} {%- endif %}
{% do slave_warning_list.append('Cipher %r translated to %r' % (cipher, CIPHER_TRANSLATION_DICT[cipher])) %} {%- endif %}
{% else %} {%- endfor %}
{% do slave_error_list.append('Cipher %r is not supported.' % (cipher,)) %} {%- endif %}
{% endif %} {#- Check strict-transport-security #}
{% endif %} {%- set strict_transport_security = (slave.get('strict-transport-security') or SLAVE_DEFAULTS['strict-transport-security']) | int(false) %}
{% endfor %} {%- if strict_transport_security is false or strict_transport_security < 0 %}
{% endif %} {%- do slave_error_list.append('Wrong strict-transport-security %s' % (slave.get('strict-transport-security'),)) %}
{# Check strict-transport-security #} {%- endif %}
{% set strict_transport_security = (slave.get('strict-transport-security') or SLAVE_DEFAULTS['strict-transport-security']) | int(false) %} {%- set custom_domain = slave.get('custom_domain') %}
{% if strict_transport_security is false or strict_transport_security < 0 %} {%- if custom_domain and custom_domain in USED_HOST_LIST %}
{% do slave_error_list.append('Wrong strict-transport-security %s' % (slave.get('strict-transport-security'),)) %} {%- set message = 'custom_domain %r clashes' % (custom_domain,) %}
{% endif %} {%- do slave_error_list.append(message) %}
{% set custom_domain = slave.get('custom_domain') %} {%- do slave_critical_error_list.append(message) %}
{% if custom_domain and custom_domain in used_host_list %} {%- else %}
{% set message = 'custom_domain %r clashes' % (custom_domain,) %} {%- do USED_HOST_LIST.append(custom_domain) %}
{% do slave_error_list.append(message) %} {%- endif %}
{% do slave_critical_error_list.append(message) %} {%- if slave.get('server-alias') %}
{% else %} {%- for slave_alias in ('' ~ slave['server-alias']).split() %}
{% do used_host_list.append(custom_domain) %} {%- if slave_alias.startswith('*.') %}
{% endif %} {%- set clean_slave_alias = slave_alias[2:] %}
{% if slave.get('server-alias') %} {%- else %}
{% for slave_alias in ('' ~ slave['server-alias']).split() %} {%- set clean_slave_alias = slave_alias %}
{% if slave_alias.startswith('*.') %} {%- endif %}
{% set clean_slave_alias = slave_alias[2:] %} {%- if not validators_module.domain(clean_slave_alias) %}
{% else %} {%- do slave_error_list.append('server-alias \'%s\' not valid' % (slave_alias,)) %}
{% set clean_slave_alias = slave_alias %} {%- else %}
{% endif %} {%- if slave_alias in slave_server_alias_unclashed or slave_alias == custom_domain %}
{% if not validators.domain(clean_slave_alias) %} {#- optionally do something about reporting back that server-alias has been unclashed #}
{% do slave_error_list.append('server-alias \'%s\' not valid' % (slave_alias,)) %} {%- elif slave_alias in USED_HOST_LIST %}
{% else %} {%- set message = 'server-alias \'%s\' clashes' % (slave_alias,) %}
{% if slave_alias in slave_server_alias_unclashed or slave_alias == custom_domain %} {%- do slave_error_list.append(message) %}
{# optionally do something about reporting back that server-alias has been unclashed #} {%- do slave_critical_error_list.append(message) %}
{% elif slave_alias in used_host_list %} {%- else %}
{% set message = 'server-alias \'%s\' clashes' % (slave_alias,) %} {%- do slave_server_alias_unclashed.append(slave_alias) %}
{% do slave_error_list.append(message) %} {%- do USED_HOST_LIST.append(slave_alias) %}
{% do slave_critical_error_list.append(message) %} {%- endif %}
{% else %} {%- endif %}
{% do slave_server_alias_unclashed.append(slave_alias) %} {%- endfor %}
{% do used_host_list.append(slave_alias) %} {%- do slave.__setitem__('server-alias', ' '.join(slave_server_alias_unclashed)) %}
{% endif %} {%- endif %}
{% endif %} {%- for url_key in ['url', 'https-url', 'health-check-failover-url', 'health-check-failover-https-url'] %}
{% endfor %} {%- if url_key in slave %}
{% do slave.__setitem__('server-alias', ' '.join(slave_server_alias_unclashed)) %} {%- set url = ('' ~ slave[url_key] or '').strip() %}
{% endif %} {%- if not validators_module.url(url) %}
{% for url_key in ['url', 'https-url', 'health-check-failover-url', 'health-check-failover-https-url'] %} {%- do slave_error_list.append('slave %s %r invalid' % (url_key, url)) %}
{% if url_key in slave %} {%- elif url != slave[url_key] %}
{% set url = ('' ~ slave[url_key] or '').strip() %} {%- do slave_warning_list.append('slave %s %r has been converted to %r' % (url_key, slave[url_key], url)) %}
{% if not validators.url(url) %} {%- endif %}
{% do slave_error_list.append('slave %s %r invalid' % (url_key, url)) %} {%- endif %}
{% elif url != slave[url_key] %} {%- endfor %}
{% do slave_warning_list.append('slave %s %r has been converted to %r' % (url_key, slave[url_key], url)) %} {%- for url_key in ['url-netloc-list', 'https-url-netloc-list', 'health-check-failover-url-netloc-list'] %}
{% endif %} {%- if url_key in slave %}
{% endif %} {%- for netloc in slave[url_key].split() %}
{% endfor %} {%- if not software_module.validate_netloc(netloc) %}
{% for url_key in ['url-netloc-list', 'https-url-netloc-list', 'health-check-failover-url-netloc-list'] %} {%- do slave_error_list.append('slave %s %r invalid' % (url_key, netloc)) %}
{% if url_key in slave %} {%- endif %}
{% for netloc in slave[url_key].split() %} {%- endfor %}
{% if not software.validate_netloc(netloc) %} {%- endif %}
{% do slave_error_list.append('slave %s %r invalid' % (url_key, netloc)) %} {%- endfor %}
{% endif %} {%- for k in ['ssl_proxy_ca_crt', 'health-check-failover-ssl-proxy-ca-crt'] %}
{% endfor %} {%- if k in slave %}
{% endif %} {%- set crt = ('' ~ slave.get(k) or '') %}
{% endfor %} {%- set check_popen = POPEN_FUNCTION([software_parameter_dict['openssl'], 'x509', '-noout']) %}
{% for k in ['ssl_proxy_ca_crt', 'health-check-failover-ssl-proxy-ca-crt'] %} {%- do check_popen.communicate(crt.encode()) %}
{% if k in slave %} {%- if check_popen.returncode != 0 %}
{% set crt = ('' ~ slave.get(k) or '') %} {%- do slave_error_list.append('%s is invalid' % (k,)) %}
{% set check_popen = popen([software_parameter_dict['openssl'], 'x509', '-noout']) %} {%- endif %}
{% do check_popen.communicate(crt.encode()) %} {%- endif %}
{% if check_popen.returncode != 0 %} {%- endfor %}
{% do slave_error_list.append('%s is invalid' % (k,)) %} {#- BBB: SlapOS Master non-zero knowledge BEGIN #}
{% endif %} {%- for key in ['ssl_key', 'ssl_crt', 'ssl_ca_crt'] %}
{% endif %} {%- if key in slave %}
{% endfor %} {%- do slave_warning_list.append('%s is obsolete, please use key-upload-url' % (key,)) %}
{# BBB: SlapOS Master non-zero knowledge BEGIN #} {%- endif %}
{% for key in ['ssl_key', 'ssl_crt', 'ssl_ca_crt'] %} {%- endfor %}
{% if key in slave %} {%- if slave.get('ssl_ca_crt') and not (slave.get('ssl_crt') and slave.get('ssl_key')) %}
{% do slave_warning_list.append('%s is obsolete, please use key-upload-url' % (key,)) %} {%- do slave_error_list.append('ssl_ca_crt is present, so ssl_crt and ssl_key are required') %}
{% endif %} {%- endif %}
{% endfor %} {%- if slave.get('ssl_key') and slave.get('ssl_crt') %}
{% if slave.get('ssl_ca_crt') and not (slave.get('ssl_crt') and slave.get('ssl_key')) %} {%- set key_popen = POPEN_FUNCTION([software_parameter_dict['openssl'], 'rsa', '-noout', '-modulus']) %}
{% do slave_error_list.append('ssl_ca_crt is present, so ssl_crt and ssl_key are required') %} {%- set crt_popen = POPEN_FUNCTION([software_parameter_dict['openssl'], 'x509', '-noout', '-modulus']) %}
{% endif %} {%- set key_modulus = key_popen.communicate(('' ~ slave['ssl_key']).encode())[0] | trim %}
{% if slave.get('ssl_key') and slave.get('ssl_crt') %} {%- set crt_modulus = crt_popen.communicate(('' ~ slave['ssl_crt']).encode())[0] | trim %}
{% set key_popen = popen([software_parameter_dict['openssl'], 'rsa', '-noout', '-modulus']) %} {%- if not key_modulus or key_modulus != crt_modulus %}
{% set crt_popen = popen([software_parameter_dict['openssl'], 'x509', '-noout', '-modulus']) %} {%- do slave_error_list.append('slave ssl_key and ssl_crt does not match') %}
{% set key_modulus = key_popen.communicate(('' ~ slave['ssl_key']).encode())[0] | trim %} {%- endif %}
{% set crt_modulus = crt_popen.communicate(('' ~ slave['ssl_crt']).encode())[0] | trim %} {%- endif %}
{% if not key_modulus or key_modulus != crt_modulus %} {#- BBB: SlapOS Master non-zero knowledge END #}
{% do slave_error_list.append('slave ssl_key and ssl_crt does not match') %} {%- if slave.get('custom_domain') %}
{% endif %} {%- set slave_custom_domain = '' ~ slave['custom_domain'] %}
{% endif %} {%- if slave_custom_domain.startswith('*.') %}
{# BBB: SlapOS Master non-zero knowledge END #} {%- set clean_custom_domain = slave_custom_domain[2:] %}
{% if slave.get('custom_domain') %} {%- else %}
{% set slave_custom_domain = '' ~ slave['custom_domain'] %} {%- set clean_custom_domain = slave_custom_domain %}
{% if slave_custom_domain.startswith('*.') %} {%- endif %}
{% set clean_custom_domain = slave_custom_domain[2:] %} {%- if not validators_module.domain(clean_custom_domain) %}
{% else %} {%- do slave_error_list.append('custom_domain %r invalid' % (slave['custom_domain'],)) %}
{% set clean_custom_domain = slave_custom_domain %} {%- endif %}
{% endif %} {%- endif %}
{% if not validators.domain(clean_custom_domain) %} {%- if len(slave_error_list) == 0 %}
{% do slave_error_list.append('custom_domain %r invalid' % (slave['custom_domain'],)) %} {#- Cleanup slave from not needed keys which come from implementation of SlapOS Master #}
{% endif %} {#- Send only controlled information about the slave to node #}
{% endif %} {%- set authorized_slave = {} %}
{% if len(slave_error_list) == 0 %} {%- for key in FRONTEND_NODE_SLAVE_PASSED_KEY_LIST %}
{# Cleanup slave from not needed keys which come from implementation of SlapOS Master #} {%- if key in slave %}
{# Send only controlled information about the slave to node #} {%- do authorized_slave.__setitem__(key, slave[key]) %}
{% set authorized_slave = {} %} {%- endif %}
{% for key in FRONTEND_NODE_SLAVE_PASSED_KEY_LIST %} {%- endfor %}
{% if key in slave %} {%- do AUTHORIZED_SLAVE_LIST.append(authorized_slave) %}
{% do authorized_slave.__setitem__(key, slave[key]) %} {%- else %}
{% endif %} {%- do REJECTED_SLAVE_DICT.__setitem__(slave.get('slave_reference'), sorted(slave_error_list)) %}
{% endfor %} {%- endif %}
{% do authorized_slave_list.append(authorized_slave) %} {%- if len(slave_critical_error_list) > 0 %}
{% else %} {%- do CRITICAL_REJECTED_SLAVE_DICT.__setitem__(slave.get('slave_reference'), sorted(slave_critical_error_list)) %}
{% do rejected_slave_dict.__setitem__(slave.get('slave_reference'), sorted(slave_error_list)) %} {%- endif %}
{% endif %} {%- if len(slave_warning_list) > 0 %}
{% if len(slave_critical_error_list) > 0 %} {%- do WARNING_SLAVE_DICT.__setitem__(slave.get('slave_reference'), sorted(slave_warning_list)) %}
{% do critical_rejected_slave_dict.__setitem__(slave.get('slave_reference'), sorted(slave_critical_error_list)) %} {%- endif %}
{% endif %} {%- endfor %}
{% if len(slave_warning_list) > 0 %} {%- do AUTHORIZED_SLAVE_LIST.sort(key=operator_module.itemgetter('slave_reference')) %}
{% do warning_slave_dict.__setitem__(slave.get('slave_reference'), sorted(slave_warning_list)) %} {#- END: Slave processing #}
{% endif %}
{% endfor %}
{% do authorized_slave_list.sort(key=operator_module.itemgetter('slave_reference')) %}
[monitor-instance-parameter] [monitor-instance-parameter]
monitor-httpd-port = {{ master_partition_monitor_monitor_httpd_port }} monitor-httpd-port = {{ MASTER_PARTITION_MONITOR_HTTPD_PORT }}
[replicate] [replicate]
<= slap-connection <= slap-connection
...@@ -340,124 +334,121 @@ config-monitor-cors-domains = {{ instance_parameter_dict['configuration'].get('m ...@@ -340,124 +334,121 @@ config-monitor-cors-domains = {{ instance_parameter_dict['configuration'].get('m
config-monitor-username = ${monitor-instance-parameter:username} config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-password = ${monitor-htpasswd:passwd} config-monitor-password = ${monitor-htpasswd:passwd}
software-type = {{frontend_type}} software-type = single-custom-personal
return = slave-instance-information-list monitor-base-url backend-client-csr-url kedifa-csr-url csr-certificate backend-haproxy-statistic-url node-information-json return = slave-instance-information-list monitor-base-url backend-client-csr-url kedifa-csr-url csr-certificate backend-haproxy-statistic-url node-information-json
{#- Send only needed parameters to frontend nodes #} {#- BEGIN: Frontend node request #}
{%- set base_node_configuration_dict = {} %} {%- set base_node_configuration_dict = {} %}
{%- for key, value in FRONTEND_PARAMETER_DICT.items() %} {%- for key, value in FRONTEND_PARAMETER_DICT.items() %}
{%- do base_node_configuration_dict.__setitem__(key, value) %} {%- do base_node_configuration_dict.__setitem__(key, value) %}
{%- endfor %} {%- endfor %}
{% for section, frontend_request in request_dict.items() %} {%- for section, frontend_request in REQUEST_DICT.items() %}
{% set state = frontend_request.get('state', '') %} {%- set state = frontend_request.get('state', '') %}
[{{section}}] [{{section}}]
<= replicate <= replicate
name = {{ frontend_request.get('name') }} name = {{ frontend_request.get('name') }}
software-url = {{ frontend_request['software-url'] }} software-url = {{ frontend_request['software-url'] }}
{% if state %} {%- if state %}
state = {{ state }} state = {{ state }}
{% endif %} {%- endif %}
{# Do not send additional parameters for destroyed nodes #} {#- Do not send additional parameters for destroyed nodes #}
{% if state != 'destroyed' %} {%- if state != 'destroyed' %}
config-slave-kedifa-information = ${request-kedifa:connection-slave-kedifa-information} config-slave-kedifa-information = ${request-kedifa:connection-slave-kedifa-information}
config-kedifa-caucase-url = ${request-kedifa:connection-caucase-url} config-kedifa-caucase-url = ${request-kedifa:connection-caucase-url}
config-backend-client-caucase-url = {{ caucase_url }} config-backend-client-caucase-url = {{ CAUCASE_URL }}
config-master-key-download-url = ${request-kedifa:connection-master-key-download-url} config-master-key-download-url = ${request-kedifa:connection-master-key-download-url}
config-cluster-identification = {{ instance_parameter_dict['root-instance-title'] }} config-cluster-identification = {{ instance_parameter_dict['root-instance-title'] }}
{% set node_configuration_dict = {} %} {%- set node_configuration_dict = {} %}
{% do node_configuration_dict.update(frontend_request.get('config')) %} {%- do node_configuration_dict.update(frontend_request.get('config')) %}
{# sort_keys are important in order to avoid shuffling parameters on each run #} {#- sort_keys are important in order to avoid shuffling parameters on each run #}
{% do node_configuration_dict.__setitem__("extra_slave_instance_list", json_module.dumps(authorized_slave_list, sort_keys=True)) %} {%- do node_configuration_dict.__setitem__("extra_slave_instance_list", json_module.dumps(AUTHORIZED_SLAVE_LIST, sort_keys=True)) %}
{% do node_configuration_dict.__setitem__("frontend-name", frontend_request.get('name')) %} {%- do node_configuration_dict.__setitem__("frontend-name", frontend_request.get('name')) %}
{%- for config_key, config_value in node_configuration_dict.items() %} {%- for config_key, config_value in node_configuration_dict.items() %}
config-{{ config_key }} = {{ dumps(config_value) }} config-{{ config_key }} = {{ dumps(config_value) }}
{% endfor -%} {%- endfor %}
{%- for config_key, config_value in base_node_configuration_dict.items() %} {%- for config_key, config_value in base_node_configuration_dict.items() %}
config-{{ config_key }} = {{ dumps(config_value) }} config-{{ config_key }} = {{ dumps(config_value) }}
{% endfor -%} {%- endfor %}
{% if frontend_request.get('sla') %} {%- if frontend_request.get('sla') %}
{% for parameter, value in frontend_request.get('sla').items() %} {%- for parameter, value in frontend_request.get('sla').items() %}
sla-{{ parameter }} = {{ value }} sla-{{ parameter }} = {{ value }}
{% endfor %} {%- endfor %}
{% endif %} {%- endif %}
{% else %} {%- else %}
{# Ignore return for destroyed nodes #} {#- Ignore return for destroyed nodes #}
return = return =
{% endif %} {%- endif %}
{% endfor %} {%- endfor %}
{#- END: Frontend node request #}
{% set warning_list = [] %}
{% for key in ['apache-certificate', 'apache-key'] %}
{% if key in instance_parameter_dict['configuration'] %}
{% do warning_list.append('%s is obsolete, please use master-key-upload-url' % (key, )) %}
{% endif %}
{% endfor %}
[publish-information] [publish-information]
<= monitor-publish <= monitor-publish
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish
domain = {{ instance_parameter_dict['configuration'].get('domain') }} domain = {{ instance_parameter_dict['configuration'].get('domain') }}
slave-amount = {{ instance_parameter_dict['slave-instance-list'] | length }} slave-amount = {{ instance_parameter_dict['slave-instance-list'] | length }}
accepted-slave-amount = {{ authorized_slave_list | length }} accepted-slave-amount = {{ AUTHORIZED_SLAVE_LIST | length }}
rejected-slave-amount = {{ rejected_slave_dict | length }} rejected-slave-amount = {{ REJECTED_SLAVE_DICT | length }}
backend-client-caucase-url = {{ caucase_url }} backend-client-caucase-url = {{ CAUCASE_URL }}
{# sort_keys are important in order to avoid shuffling parameters on each run #} {#- sort_keys are important in order to avoid shuffling parameters on each run #}
rejected-slave-dict = {{ dumps(json_module.dumps(rejected_slave_dict, sort_keys=True)) }} rejected-slave-dict = {{ dumps(json_module.dumps(REJECTED_SLAVE_DICT, sort_keys=True)) }}
rejected-slave-promise-url = ${rejected-slave-promise:config-url} rejected-slave-promise-url = ${rejected-slave-promise:config-url}
publish-failsafe-error-promise-url = ${publish-failsafe-error-promise:config-url} publish-failsafe-error-promise-url = ${publish-failsafe-error-promise:config-url}
master-key-upload-url = ${request-kedifa:connection-master-key-upload-url} master-key-upload-url = ${request-kedifa:connection-master-key-upload-url}
master-key-generate-auth-url = ${request-kedifa:connection-master-key-generate-auth-url} master-key-generate-auth-url = ${request-kedifa:connection-master-key-generate-auth-url}
kedifa-caucase-url = ${request-kedifa:connection-caucase-url} kedifa-caucase-url = ${request-kedifa:connection-caucase-url}
{% if len(warning_list) > 0 %} {%- set warning_list = [] %}
{# sort_keys are important in order to avoid shuffling parameters on each run #} {%- for key in ['apache-certificate', 'apache-key'] %}
{%- if key in instance_parameter_dict['configuration'] %}
{%- do warning_list.append('%s is obsolete, please use master-key-upload-url' % (key, )) %}
{%- endif %}
{%- endfor %}
{%- if len(warning_list) > 0 %}
{#- sort_keys are important in order to avoid shuffling parameters on each run #}
warning-list = {{ dumps(json_module.dumps(warning_list, sort_keys=True)) }} warning-list = {{ dumps(json_module.dumps(warning_list, sort_keys=True)) }}
{% endif %} {%- endif %}
{% if len(warning_slave_dict) > 0 %} {%- if len(WARNING_SLAVE_DICT) > 0 %}
{# sort_keys are important in order to avoid shuffling parameters on each run #} {#- sort_keys are important in order to avoid shuffling parameters on each run #}
warning-slave-dict = {{ dumps(json_module.dumps(warning_slave_dict, sort_keys=True)) }} warning-slave-dict = {{ dumps(json_module.dumps(WARNING_SLAVE_DICT, sort_keys=True)) }}
{% endif %} {%- endif %}
{% if not aikc_enabled or not aibcc_enabled %} {%- if not AIKC_ENABLED or not AIBCC_ENABLED %}
{% for index, frontend in enumerate(frontend_list) %} {%- for index, frontend in enumerate(FRONTEND_LIST) %}
{% set section_part = '${request-' + frontend %} {%- set section_part = '${request-' + frontend %}
frontend-node-{{ index + 1 }}-csr-certificate = {{ section_part }}:connection-csr-certificate} frontend-node-{{ index + 1 }}-csr-certificate = {{ section_part }}:connection-csr-certificate}
{% endfor %} {%- endfor %}
{% endif %} {%- endif %}
{% if not aikc_enabled %} {%- if not AIKC_ENABLED %}
kedifa-csr-url = ${request-kedifa:connection-kedifa-csr-url} kedifa-csr-url = ${request-kedifa:connection-kedifa-csr-url}
kedifa-csr-certificate = ${request-kedifa:connection-csr-certificate} kedifa-csr-certificate = ${request-kedifa:connection-csr-certificate}
{% for index, frontend in enumerate(frontend_list) %} {%- for index, frontend in enumerate(FRONTEND_LIST) %}
{% set section_part = '${request-' + frontend %} {%- set section_part = '${request-' + frontend %}
frontend-node-{{ index + 1 }}-kedifa-csr-url = {{ section_part }}:connection-kedifa-csr-url} frontend-node-{{ index + 1 }}-kedifa-csr-url = {{ section_part }}:connection-kedifa-csr-url}
{% endfor %} {%- endfor %}
{% endif %} {%- endif %}
{% for index, frontend in enumerate(frontend_list) %} {%- for index, frontend in enumerate(FRONTEND_LIST) %}
{% set section_part = '${request-' + frontend %} {%- set section_part = '${request-' + frontend %}
frontend-node-{{ index + 1 }}-backend-haproxy-statistic-url = {{ section_part }}:connection-backend-haproxy-statistic-url} frontend-node-{{ index + 1 }}-backend-haproxy-statistic-url = {{ section_part }}:connection-backend-haproxy-statistic-url}
frontend-node-{{ index + 1 }}-node-information-json = ${frontend-information:frontend-node-{{ index + 1 }}-node-information-json} frontend-node-{{ index + 1 }}-node-information-json = ${frontend-information:frontend-node-{{ index + 1 }}-node-information-json}
{% endfor %} {%- endfor %}
{% if not aibcc_enabled %} {%- if not AIBCC_ENABLED %}
{% for index, frontend in enumerate(frontend_list) %} {%- for index, frontend in enumerate(FRONTEND_LIST) %}
{% set section_part = '${request-' + frontend %} {%- set section_part = '${request-' + frontend %}
frontend-node-{{ index + 1 }}-backend-client-csr-url = {{ section_part }}:connection-backend-client-csr-url} frontend-node-{{ index + 1 }}-backend-client-csr-url = {{ section_part }}:connection-backend-client-csr-url}
{% endfor %} {%- endfor %}
{% endif %} {%- endif %}
# Generate promises for requested nodes {#- Generate promises for requested frontend nodes #}
{% for index, frontend in enumerate(frontend_list) %} {%- for index, frontend in enumerate(FRONTEND_LIST) %}
{% set part_name = 'promise-backend-haproxy-statistic-url-' + frontend %} {%- set part_name = 'promise-backend-haproxy-statistic-url-' + frontend %}
{% do part_list.append(part_name) %} {%- do PART_LIST.append(part_name) %}
{% set section_part = '${request-' + frontend %} {%- set section_part = '${request-' + frontend %}
[{{ part_name }}] [{{ part_name }}]
<= monitor-promise-base <= monitor-promise-base
promise = check_url_available promise = check_url_available
name = check-backend-haproxy-statistic-url-frontend-node-{{ index + 1 }}.py name = check-backend-haproxy-statistic-url-frontend-node-{{ index + 1 }}.py
config-url = config-url =
{{ section_part }}:connection-backend-haproxy-statistic-url} {{ section_part }}:connection-backend-haproxy-statistic-url}
{% endfor %} {%- endfor %}
#----------------------------
#--
#-- Publish slave information
[publish-slave-information] [publish-slave-information]
recipe = slapos.cookbook:switch-softwaretype recipe = slapos.cookbook:switch-softwaretype
default = instance-publish-slave-information:output default = instance-publish-slave-information:output
...@@ -472,61 +463,60 @@ recipe = slapos.cookbook:requestoptional.serialised ...@@ -472,61 +463,60 @@ recipe = slapos.cookbook:requestoptional.serialised
config-monitor-cors-domains = {{ instance_parameter_dict['configuration'].get('monitor-cors-domains', 'monitor.app.officejs.com') }} config-monitor-cors-domains = {{ instance_parameter_dict['configuration'].get('monitor-cors-domains', 'monitor.app.officejs.com') }}
config-monitor-username = ${monitor-instance-parameter:username} config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-password = ${monitor-htpasswd:passwd} config-monitor-password = ${monitor-htpasswd:passwd}
config-monitor-httpd-port = {{ kedifa_partition_monitor_httpd_port }} config-monitor-httpd-port = {{ KEDIFA_PARTITION_MONITOR_HTTPD_PORT }}
config-caucase_port = {{ dumps(instance_parameter_dict['configuration'].get('caucase_port', 8890)) }} config-caucase_port = {{ dumps(instance_parameter_dict['configuration'].get('caucase_port', 8890)) }}
config-kedifa_port = {{ dumps(instance_parameter_dict['configuration'].get('kedifa_port', 7879)) }} config-kedifa_port = {{ dumps(instance_parameter_dict['configuration'].get('kedifa_port', 7879)) }}
config-rotate-num = {{ dumps(instance_parameter_dict['configuration'].get('rotate-num', 4000)) }} config-rotate-num = {{ dumps(instance_parameter_dict['configuration'].get('rotate-num', 4000)) }}
{% for key in ['kedifa_port', 'caucase_port'] -%} {%- for key in ['kedifa_port', 'caucase_port'] -%}
{%- if key in instance_parameter_dict['configuration'] %} {%- if key in instance_parameter_dict['configuration'] %}
config-{{ key }} = {{ dumps(instance_parameter_dict['configuration'][key]) }} config-{{ key }} = {{ dumps(instance_parameter_dict['configuration'][key]) }}
{%- endif %} {%- endif %}
{%- endfor %} {%- endfor %}
config-slave-list = {{ dumps(authorized_slave_list) }} config-slave-list = {{ dumps(AUTHORIZED_SLAVE_LIST) }}
config-cluster-identification = {{ instance_parameter_dict['root-instance-title'] }} config-cluster-identification = {{ instance_parameter_dict['root-instance-title'] }}
{% set software_url_key = "-kedifa-software-release-url" %} {%- if '-kedifa-software-release-url' in instance_parameter_dict['configuration'] %}
{% if software_url_key in instance_parameter_dict['configuration'] %} software-url = {{ instance_parameter_dict['configuration'].pop('-kedifa-software-release-url') }}
software-url = {{ instance_parameter_dict['configuration'].pop(software_url_key) }} {%- else %}
{% else %}
software-url = ${slap-connection:software-release-url} software-url = ${slap-connection:software-release-url}
{% endif %} {%- endif %}
software-type = kedifa software-type = kedifa
name = kedifa name = kedifa
return = slave-kedifa-information master-key-generate-auth-url master-key-upload-url master-key-download-url caucase-url kedifa-csr-url csr-certificate monitor-base-url return = slave-kedifa-information master-key-generate-auth-url master-key-upload-url master-key-download-url caucase-url kedifa-csr-url csr-certificate monitor-base-url
{% set sla_kedifa_key = "-sla-kedifa-" %} {%- set sla_kedifa_key = "-sla-kedifa-" %}
{% set sla_kedifa_key_length = sla_kedifa_key | length %} {%- set sla_kedifa_key_length = sla_kedifa_key | length %}
{% for key in list(instance_parameter_dict['configuration'].keys()) %} {%- for key in list(instance_parameter_dict['configuration'].keys()) %}
{% if key.startswith(sla_kedifa_key) %} {%- if key.startswith(sla_kedifa_key) %}
sla-{{ key[sla_kedifa_key_length:] }} = {{ instance_parameter_dict['configuration'].pop(key) }} sla-{{ key[sla_kedifa_key_length:] }} = {{ instance_parameter_dict['configuration'].pop(key) }}
{% endif %} {%- endif %}
{% endfor %} {%- endfor %}
[rejected-slave-information] [rejected-slave-information]
rejected-slave-dict = {{ dumps(rejected_slave_dict) }} rejected-slave-dict = {{ dumps(REJECTED_SLAVE_DICT) }}
[warning-slave-information] [warning-slave-information]
warning-slave-dict = {{ dumps(warning_slave_dict) }} warning-slave-dict = {{ dumps(WARNING_SLAVE_DICT) }}
[slave-information] [slave-information]
{% for frontend_section in frontend_section_list %} {%- for frontend_section in FRONTEND_SECTION_LIST %}
{{ frontend_section }} = {{ "${%s:connection-slave-instance-information-list}" % frontend_section }} {{ frontend_section }} = {{ "${%s:connection-slave-instance-information-list}" % frontend_section }}
{% endfor %} {%- endfor %}
[active-slave-instance] [active-slave-instance]
{% set active_slave_instance_list = [] %} {%- set active_slave_instance_list = [] %}
{% for slave_instance in instance_parameter_dict['slave-instance-list'] %} {%- for slave_instance in instance_parameter_dict['slave-instance-list'] %}
{# Provide a list of slave titles send by master, in order to filter out already destroyed slaves #} {#- Provide a list of slave titles send by master, in order to filter out already destroyed slaves #}
{# Note: This functionality is not yet covered by tests, please modify with care #} {#- Note: This functionality is not yet covered by tests, please modify with care #}
{% do active_slave_instance_list.append(slave_instance['slave_reference']) %} {%- do active_slave_instance_list.append(slave_instance['slave_reference']) %}
{% endfor %} {%- endfor %}
{# sort_keys are important in order to avoid shuffling parameters on each run #} {#- sort_keys are important in order to avoid shuffling parameters on each run #}
active-slave-instance-list = {{ json_module.dumps(active_slave_instance_list, sort_keys=True) }} active-slave-instance-list = {{ json_module.dumps(active_slave_instance_list, sort_keys=True) }}
[frontend-information] [frontend-information]
{% for index, frontend in enumerate(frontend_list) %} {%- for index, frontend in enumerate(FRONTEND_LIST) %}
{% set section_part = '${request-' + frontend %} {%- set section_part = '${request-' + frontend %}
frontend-node-{{ index + 1 }}-node-information-json = {{ section_part }}:connection-node-information-json} frontend-node-{{ index + 1 }}-node-information-json = {{ section_part }}:connection-node-information-json}
{% endfor %} {%- endfor %}
[instance-publish-slave-information] [instance-publish-slave-information]
< = jinja2-template-base < = jinja2-template-base
...@@ -544,9 +534,9 @@ extra-context = ...@@ -544,9 +534,9 @@ extra-context =
[monitor-base-url-dict] [monitor-base-url-dict]
kedifa = ${request-kedifa:connection-monitor-base-url} kedifa = ${request-kedifa:connection-monitor-base-url}
{% for frontend in frontend_section_list %} {%- for frontend in FRONTEND_SECTION_LIST %}
{{ frontend }} = {{ '${' + frontend + ':connection-monitor-base-url}' }} {{ frontend }} = {{ '${' + frontend + ':connection-monitor-base-url}' }}
{% endfor %} {%- endfor %}
[directory] [directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
...@@ -554,16 +544,15 @@ bin = ${buildout:directory}/bin/ ...@@ -554,16 +544,15 @@ bin = ${buildout:directory}/bin/
srv = ${buildout:directory}/srv/ srv = ${buildout:directory}/srv/
tmp = ${buildout:directory}/tmp/ tmp = ${buildout:directory}/tmp/
backup = ${:srv}/backup backup = ${:srv}/backup
# CAUCASE directories
caucased = ${:srv}/caucased caucased = ${:srv}/caucased
backup-caucased = ${:backup}/caucased backup-caucased = ${:backup}/caucased
# NGINX
master-introspection-var = ${:var}/master-introspection-nginx master-introspection-var = ${:var}/master-introspection-nginx
# slaves
publish-failsafe-error-var = ${:var}/publish-failsafe-error publish-failsafe-error-var = ${:var}/publish-failsafe-error
publish-failsafe-check-var = ${:var}/publish-failsafe-check publish-failsafe-check-var = ${:var}/publish-failsafe-check
service = ${:etc}/service
promise-output = ${:srv}/promise-output
{% if aikc_enabled %} {%- if AIKC_ENABLED %}
[directory] [directory]
aikc = ${:srv}/aikc aikc = ${:srv}/aikc
...@@ -593,12 +582,12 @@ command = ...@@ -593,12 +582,12 @@ command =
update-command = ${:command} update-command = ${:command}
csr = ${aikc-config:csr} csr = ${aikc-config:csr}
key = ${aikc-config:key} key = ${aikc-config:key}
{#- Can be stopped on error, as does not rely on self provided service #} {#- Can be stopped on error, as does not rely on self provided service #}
stop-on-error = True stop-on-error = True
[aikc-caucase-wrapper] [aikc-caucase-wrapper]
{# jinja2 instead of wrapper is used with context to remove py'u' #} {#- jinja2 instead of wrapper is used with context to remove py'u' #}
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
context = context =
key caucase_url aikc-config:caucase-url key caucase_url aikc-config:caucase-url
...@@ -614,10 +603,10 @@ inline = ...@@ -614,10 +603,10 @@ inline =
output = ${directory:bin}/aikc-caucase-wrapper output = ${directory:bin}/aikc-caucase-wrapper
{% do part_list.append('aikc-create-user') %} {%- do PART_LIST.append('aikc-create-user') %}
[aikc-create-user] [aikc-create-user]
recipe = plone.recipe.command recipe = plone.recipe.command
{#- The called command is smart enough to survive errors and retry #} {#- The called command is smart enough to survive errors and retry #}
stop-on-error = False stop-on-error = False
update-command = ${:command} update-command = ${:command}
csr_id = ${directory:aikc}/csr_id csr_id = ${directory:aikc}/csr_id
...@@ -631,9 +620,9 @@ command = ...@@ -631,9 +620,9 @@ command =
touch ${aikc-config:user-created} touch ${aikc-config:user-created}
fi fi
{% do part_list.append('aikc-user-caucase-updater') %} {%- do PART_LIST.append('aikc-user-caucase-updater') %}
{% do part_list.append('aikc-user-caucase-updater-promise') %} {%- do PART_LIST.append('aikc-user-caucase-updater-promise') %}
{{ caucase.updater( {{ CAUCASE.updater(
prefix='aikc-user-caucase-updater', prefix='aikc-user-caucase-updater',
buildout_bin_directory=software_parameter_dict['bin_directory'], buildout_bin_directory=software_parameter_dict['bin_directory'],
updater_path='${directory:service}/aikc-user-caucase-updater', updater_path='${directory:service}/aikc-user-caucase-updater',
...@@ -654,14 +643,14 @@ command-line = {{ software_parameter_dict['caucase_csr_sign_check'] }} ...@@ -654,14 +643,14 @@ command-line = {{ software_parameter_dict['caucase_csr_sign_check'] }}
${aikc-config:key} ${aikc-config:key}
wrapper-path = ${directory:bin}/aikc-caucase-csr-sign-check wrapper-path = ${directory:bin}/aikc-caucase-csr-sign-check
{% do part_list.append('aikc-sign-promise') %} {%- do PART_LIST.append('aikc-sign-promise') %}
[aikc-sign-promise] [aikc-sign-promise]
<= monitor-promise-base <= monitor-promise-base
promise = check_command_execute promise = check_command_execute
name = ${:_buildout_section_name_}.py name = ${:_buildout_section_name_}.py
config-command = ${aikc-sign-promise-wrapper:wrapper-path} config-command = ${aikc-sign-promise-wrapper:wrapper-path}
{% for csr in frontend_list + ['kedifa'] %} {%- for csr in FRONTEND_LIST + ['kedifa'] %}
[aikc-{{ csr }}-wrapper] [aikc-{{ csr }}-wrapper]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/aikc-{{ csr }}-wrapper wrapper-path = ${directory:bin}/aikc-{{ csr }}-wrapper
...@@ -673,23 +662,23 @@ command-line = {{ software_parameter_dict['smart_caucase_signer'] }} ...@@ -673,23 +662,23 @@ command-line = {{ software_parameter_dict['smart_caucase_signer'] }}
${request-{{ csr }}:connection-kedifa-csr-url} ${request-{{ csr }}:connection-kedifa-csr-url}
"${request-{{ csr }}:connection-csr-certificate}" "${request-{{ csr }}:connection-csr-certificate}"
{% do part_list.append('aikc-%s' % (csr,)) %} {%- do PART_LIST.append('aikc-%s' % (csr,)) %}
[aikc-{{ csr }}] [aikc-{{ csr }}]
recipe = plone.recipe.command recipe = plone.recipe.command
{#- The called command is smart enough to survive errors and retry #} {#- The called command is smart enough to survive errors and retry #}
stop-on-error = False stop-on-error = False
command = command =
${aikc-{{ csr }}-wrapper:wrapper-path} ${aikc-{{ csr }}-wrapper:wrapper-path}
update-command = ${:command} update-command = ${:command}
{% endfor %} {%- endfor %}
{% endif %} {# if aikc_enabled #} {%- endif %} {# if AIKC_ENABLED #}
{% if aibcc_enabled %} {%- if AIBCC_ENABLED %}
[directory] [directory]
aibcc = ${:srv}/aibcc aibcc = ${:srv}/aibcc
[aibcc-config] [aibcc-config]
caucase-url = {{ caucase_url }} caucase-url = {{ CAUCASE_URL }}
csr = ${directory:aibcc}/csr.pem csr = ${directory:aibcc}/csr.pem
key = ${directory:aibcc}/key.pem key = ${directory:aibcc}/key.pem
...@@ -714,12 +703,12 @@ command = ...@@ -714,12 +703,12 @@ command =
update-command = ${:command} update-command = ${:command}
csr = ${aibcc-config:csr} csr = ${aibcc-config:csr}
key = ${aibcc-config:key} key = ${aibcc-config:key}
{#- Can be stopped on error, as does not rely on self provided service #} {#- Can be stopped on error, as does not rely on self provided service #}
stop-on-error = True stop-on-error = True
[aibcc-caucase-wrapper] [aibcc-caucase-wrapper]
{# jinja2 instead of wrapper is used with context to remove py'u' #} {#- jinja2 instead of wrapper is used with context to remove py'u' #}
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
context = context =
key caucase_url aibcc-config:caucase-url key caucase_url aibcc-config:caucase-url
...@@ -736,12 +725,12 @@ inline = ...@@ -736,12 +725,12 @@ inline =
output = ${directory:bin}/aibcc-caucase-wrapper output = ${directory:bin}/aibcc-caucase-wrapper
{% do part_list.append('aibcc-create-user') %} {%- do PART_LIST.append('aibcc-create-user') %}
[aibcc-create-user] [aibcc-create-user]
recipe = plone.recipe.command recipe = plone.recipe.command
# the caucase for this part is provided in this profile, so we can't fail # the caucase for this part is provided in this profile, so we can't fail
# as otherwise caucase will never be started... # as otherwise caucase will never be started...
{#- XXX: Create promise #} {#- XXX: Create promise #}
stop-on-error = False stop-on-error = False
update-command = ${:command} update-command = ${:command}
csr_id = ${directory:aibcc}/csr_id csr_id = ${directory:aibcc}/csr_id
...@@ -755,9 +744,9 @@ command = ...@@ -755,9 +744,9 @@ command =
touch ${aibcc-config:user-created} touch ${aibcc-config:user-created}
fi fi
{% do part_list.append('aibcc-user-caucase-updater') %} {%- do PART_LIST.append('aibcc-user-caucase-updater') %}
{% do part_list.append('aibcc-user-caucase-updater-promise') %} {%- do PART_LIST.append('aibcc-user-caucase-updater-promise') %}
{{ caucase.updater( {{ CAUCASE.updater(
prefix='aibcc-user-caucase-updater', prefix='aibcc-user-caucase-updater',
buildout_bin_directory=software_parameter_dict['bin_directory'], buildout_bin_directory=software_parameter_dict['bin_directory'],
updater_path='${directory:service}/aibcc-user-caucase-updater', updater_path='${directory:service}/aibcc-user-caucase-updater',
...@@ -778,14 +767,14 @@ command-line = {{ software_parameter_dict['caucase_csr_sign_check'] }} ...@@ -778,14 +767,14 @@ command-line = {{ software_parameter_dict['caucase_csr_sign_check'] }}
${aibcc-config:key} ${aibcc-config:key}
wrapper-path = ${directory:bin}/aibcc-caucase-csr-sign-check wrapper-path = ${directory:bin}/aibcc-caucase-csr-sign-check
{% do part_list.append('aibcc-sign-promise') %} {%- do PART_LIST.append('aibcc-sign-promise') %}
[aibcc-sign-promise] [aibcc-sign-promise]
<= monitor-promise-base <= monitor-promise-base
promise = check_command_execute promise = check_command_execute
name = ${:_buildout_section_name_}.py name = ${:_buildout_section_name_}.py
config-command = ${aibcc-sign-promise-wrapper:wrapper-path} config-command = ${aibcc-sign-promise-wrapper:wrapper-path}
{% for csr in frontend_list %} {%- for csr in FRONTEND_LIST %}
[aibcc-{{ csr }}-wrapper] [aibcc-{{ csr }}-wrapper]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/aibcc-{{ csr }}-wrapper wrapper-path = ${directory:bin}/aibcc-{{ csr }}-wrapper
...@@ -797,16 +786,16 @@ command-line = {{ software_parameter_dict['smart_caucase_signer'] }} ...@@ -797,16 +786,16 @@ command-line = {{ software_parameter_dict['smart_caucase_signer'] }}
${request-{{ csr }}:connection-backend-client-csr-url} ${request-{{ csr }}:connection-backend-client-csr-url}
"${request-{{ csr }}:connection-csr-certificate}" "${request-{{ csr }}:connection-csr-certificate}"
{% do part_list.append('aibcc-%s' % (csr,)) %} {%- do PART_LIST.append('aibcc-%s' % (csr,)) %}
[aibcc-{{ csr }}] [aibcc-{{ csr }}]
recipe = plone.recipe.command recipe = plone.recipe.command
{#- The called command is smart enough to survive errors and retry #} {#- The called command is smart enough to survive errors and retry #}
stop-on-error = False stop-on-error = False
command = command =
${aibcc-{{ csr }}-wrapper:wrapper-path} ${aibcc-{{ csr }}-wrapper:wrapper-path}
update-command = ${:command} update-command = ${:command}
{% endfor %} {%- endfor %}
{% endif %} {# if aibcc_enabled #} {%- endif %} {# if AIBCC_ENABLED #}
[rejected-slave-json] [rejected-slave-json]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -814,19 +803,15 @@ filename = rejected-slave.json ...@@ -814,19 +803,15 @@ filename = rejected-slave.json
directory = ${directory:promise-output} directory = ${directory:promise-output}
output = ${:directory}/${:filename} output = ${:directory}/${:filename}
url = {{ software_parameter_dict['template_empty'] }} url = {{ software_parameter_dict['template_empty'] }}
{% if critical_rejected_slave_dict %} {%- if CRITICAL_REJECTED_SLAVE_DICT %}
{# sort_keys are important in order to avoid shuffling parameters on each run #} {# sort_keys are important in order to avoid shuffling parameters on each run #}
content = {{ dumps(json_module.dumps(critical_rejected_slave_dict, indent=2, sort_keys=True)) }} content = {{ dumps(json_module.dumps(CRITICAL_REJECTED_SLAVE_DICT, indent=2, sort_keys=True)) }}
{% else %} {%- else %}
content = content =
{% endif %} {%- endif %}
context = context =
key content :content key content :content
[directory]
service = ${:etc}/service
promise-output = ${:srv}/promise-output
[master-introspection-server-configuration] [master-introspection-server-configuration]
ip = {{ instance_parameter_dict['ipv6-random'] }} ip = {{ instance_parameter_dict['ipv6-random'] }}
port = 14455 port = 14455
...@@ -1004,13 +989,13 @@ config-filename = ${master-key-download-url-ready:output} ...@@ -1004,13 +989,13 @@ config-filename = ${master-key-download-url-ready:output}
[caucased-backend-client] [caucased-backend-client]
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
{{ caucase.caucased( {{ CAUCASE.caucased(
prefix='caucased-backend-client', prefix='caucased-backend-client',
buildout_bin_directory=software_parameter_dict['bin_directory'], buildout_bin_directory=software_parameter_dict['bin_directory'],
caucased_path='${directory:service}/caucased-backend-client', caucased_path='${directory:service}/caucased-backend-client',
backup_dir='${directory:backup-caucased}', backup_dir='${directory:backup-caucased}',
data_dir='${directory:caucased}', data_dir='${directory:caucased}',
netloc=caucase_netloc, netloc=CAUCASE_NETLOC,
tmp='${directory:tmp}', tmp='${directory:tmp}',
service_auto_approve_count=0, service_auto_approve_count=0,
user_auto_approve_count=1, user_auto_approve_count=1,
...@@ -1055,6 +1040,6 @@ parts = ...@@ -1055,6 +1040,6 @@ parts =
master-key-generate-auth-url-ready-promise master-key-generate-auth-url-ready-promise
master-key-download-url-ready-promise master-key-download-url-ready-promise
master-introspection-frontend master-introspection-frontend
{% for part in part_list %} {%- for part in PART_LIST %}
{{ ' %s' % part }} {{ ' %s' % part }}
{% endfor %} {%- endfor %}
...@@ -99,12 +99,10 @@ extra-context = ...@@ -99,12 +99,10 @@ extra-context =
import subprocess_module subprocess import subprocess_module subprocess
import functools_module functools import functools_module functools
import operator_module operator import operator_module operator
import validators validators import validators_module validators
import software software import software_module software
key CLUSTER_DEFAULTS instance-default:defaults key CLUSTER_DEFAULTS instance-default:defaults
key SLAVE_DEFAULTS instance-slave-default:defaults key SLAVE_DEFAULTS instance-slave-default:defaults
# Must match the key id in [switch-softwaretype] which uses this section.
raw software_type RootSoftwareInstance-default-custom-personal-replicate
[dynamic-profile-kedifa] [dynamic-profile-kedifa]
< = jinja2-template-base < = jinja2-template-base
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment