{% set aibcc_enabled = True %} {% import "caucase" as caucase with context %} {#- DANGER! DANGER! #} {#- Avoid touching the NAME_BASE, as it will result with backward incompatible cluster setup #} {%- set NAME_BASE = 'caddy-frontend' %} {#- DANGER! DANGER! #} {%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%} {%- set GOOD_CIPHER_LIST = ['ECDHE-ECDSA-AES256-GCM-SHA384', 'ECDHE-RSA-AES256-GCM-SHA384', 'ECDHE-ECDSA-AES128-GCM-SHA256', 'ECDHE-RSA-AES128-GCM-SHA256', 'ECDHE-ECDSA-WITH-CHACHA20-POLY1305', 'ECDHE-RSA-WITH-CHACHA20-POLY1305', 'ECDHE-RSA-AES256-CBC-SHA', 'ECDHE-RSA-AES128-CBC-SHA', 'ECDHE-ECDSA-AES256-CBC-SHA', 'ECDHE-ECDSA-AES128-CBC-SHA', 'RSA-AES256-CBC-SHA', 'RSA-AES128-CBC-SHA', 'ECDHE-RSA-3DES-EDE-CBC-SHA', 'RSA-3DES-EDE-CBC-SHA'] %} {#- Allow to pass only some parameters to frontend nodes #} {%- set FRONTEND_NODE_PASSED_KEY_LIST = [ 'plain_http_port', 'port', 'apache-certificate', 'apache-key', 'domain', 'enable-http2-by-default', 'mpm-graceful-shutdown-timeout', 're6st-verification-url', 'backend-connect-timeout', 'backend-connect-retries', 'ciphers', 'request-timeout', 'authenticate-to-backend', ] %} {#- SlapOS Master (but not slapproxy!) merges slave's instance and connection parameters, so the slave information passed to nodes have to be limited only to instance related keys #} {#- Note: As a result, this feature is very hard to be tested with slapproxy, as it does not pollute the slave information, this kind of whitelist is implemented #} {%- set FRONTEND_NODE_SLAVE_PASSED_KEY_LIST_SCHEMA = [ 'authenticate-to-backend', 'backend-connect-retries', 'backend-connect-timeout', 'ciphers', 'custom_domain', 'default-path', 'disable-no-cache-request', 'disable-via-header', 'disabled-cookie-list', 'enable-http2', 'enable_cache', 'health-check', 'health-check-authenticate-to-failover-backend', 'health-check-failover-https-url', 'health-check-failover-https-url-netloc-list', 'health-check-failover-ssl-proxy-ca-crt', 'health-check-failover-ssl-proxy-verify', 'health-check-failover-url', 'health-check-failover-url-netloc-list', 'health-check-fall', 'health-check-http-method', 'health-check-http-path', 'health-check-http-version', 'health-check-interval', 'health-check-rise', 'health-check-timeout', 'https-only', 'https-url', 'https-url-netloc-list', 'monitor-ipv4-test', 'monitor-ipv6-test', 'path', 'prefer-gzip-encoding-to-backend', 'request-timeout', 'server-alias', 'ssl-proxy-verify', 'ssl_ca_crt', 'ssl_crt', 'ssl_key', 'ssl_proxy_ca_crt', 'strict-transport-security', 'strict-transport-security-preload', 'strict-transport-security-sub-domains', 'type', 'url', 'url-netloc-list', 'virtualhostroot-http-port', 'virtualhostroot-https-port', 'websocket-path-list', 'websocket-transparent', ] %} {%- set FRONTEND_NODE_SLAVE_PASSED_KEY_LIST_INTERNAL = [ 'slave_reference', ] %} {%- set FRONTEND_NODE_SLAVE_PASSED_KEY_LIST = FRONTEND_NODE_SLAVE_PASSED_KEY_LIST_SCHEMA + FRONTEND_NODE_SLAVE_PASSED_KEY_LIST_INTERNAL %} {% set aikc_enabled = slapparameter_dict.get('automatic-internal-kedifa-caucase-csr', 'true').lower() in TRUE_VALUES %} {% set aibcc_enabled = slapparameter_dict.get('automatic-internal-backend-client-caucase-csr', 'true').lower() in TRUE_VALUES %} {# Ports 8401, 8402 and 8410+1..N are reserved for monitor ports on various partitions #} {% set master_partition_monitor_monitor_httpd_port = 8401 %} {% set kedifa_partition_monitor_httpd_port = 8402 %} {% set frontend_monitor_httpd_base_port = 8410 %} {% set caucase_host = '[' ~ instance_parameter_dict['ipv6-random'] ~ ']' %} {% set caucase_netloc = caucase_host ~ ':' ~ instance_parameter_dict['configuration.caucase_backend_client_port'] %} {% set caucase_url = 'http://' ~ caucase_netloc %} [jinja2-template-base] recipe = slapos.recipe.template:jinja2 output = ${buildout:directory}/${:filename} extra-context = context = import json_module json raw profile_common {{ software_parameter_dict['profile_common'] }} ${:extra-context} {% set popen = functools_module.partial(subprocess_module.Popen, stdout=subprocess_module.PIPE, stderr=subprocess_module.STDOUT, stdin=subprocess_module.PIPE) %} {% set part_list = [] %} {% set single_type_key = 'single-' %} {% set frontend_type = "%s%s" % (single_type_key, 'custom-personal') %} {% set frontend_quantity = slapparameter_dict.pop('-frontend-quantity', '1') | int %} {% set slave_list_name = 'extra_slave_instance_list' %} {% set frontend_list = [] %} {% set frontend_section_list = [] %} {% set request_dict = {} %} # XXX Dirty hack, not possible to define default value before {% set sla_computer_1_key = '-sla-1-computer_guid' %} {% if not sla_computer_1_key in slapparameter_dict %} {% do slapparameter_dict.__setitem__(sla_computer_1_key, '${slap-connection:computer-id}') %} {% endif %} {% set sla_computer_kedifa_key = '-sla-kedifa-computer_guid' %} {% if not sla_computer_kedifa_key in slapparameter_dict %} {% do slapparameter_dict.__setitem__(sla_computer_kedifa_key, '${slap-connection:computer-id}') %} {% endif %} # Here we request individually each frontend. # The presence of sla parameters is checked and added if found {% for i in range(1, frontend_quantity + 1) %} {% set frontend_name = "%s-%s" % (NAME_BASE, i) %} {% set request_section_title = 'request-%s' % frontend_name %} {% set sla_key = "-sla-%s-" % i %} {% set sla_key_length = sla_key | length %} {% set sla_dict = {} %} {% set config_key = "-frontend-config-%s-" % i %} {% set config_key_length = config_key | length %} {% set config_dict = {} %} {% for key in list(slapparameter_dict.keys()) %} {% if key.startswith(sla_key) %} {% do sla_dict.__setitem__(key[sla_key_length:], slapparameter_dict.pop(key)) %} # We check for specific configuration regarding the frontend {% elif key.startswith(config_key) %} {% do config_dict.__setitem__(key[config_key_length:], slapparameter_dict.pop(key)) %} {% endif %} {% endfor %} {% do config_dict.__setitem__('monitor-httpd-port', frontend_monitor_httpd_base_port + i) %} {% do config_dict.__setitem__('backend-client-caucase-url', caucase_url) %} {% set state_key = "-frontend-%s-state" % i %} {% set frontend_state = slapparameter_dict.pop(state_key, None) %} {% if frontend_state != 'destroyed' %} {% do frontend_list.append(frontend_name) %} {% do frontend_section_list.append(request_section_title) %} {% endif %} {% do part_list.append(request_section_title) %} # Filling request dict for slave {% set request_content_dict = { 'config': config_dict, 'name': frontend_name, 'sla': sla_dict, 'state': frontend_state } %} {% set frontend_software_url_key = "-frontend-%s-software-release-url" % i %} {% do request_content_dict.__setitem__('software-url', slapparameter_dict.get(frontend_software_url_key) or '${slap-connection:software-release-url}') %} {% do request_dict.__setitem__(request_section_title, request_content_dict) %} {% endfor %} {% set authorized_slave_string_list = [] %} {% set authorized_slave_list = [] %} {% set rejected_slave_dict = {} %} {% set critical_rejected_slave_dict = {} %} {% set warning_slave_dict = {} %} {% set used_host_list = [] %} {% for slave in sorted(instance_parameter_dict['slave-instance-list'], key=operator_module.itemgetter('slave_reference')) %} {% set slave_error_list = [] %} {% set slave_critical_error_list = [] %} {% set slave_warning_list = [] %} {% set slave_server_alias_unclashed = [] %} {% set slave_type = slave.get('type') %} {% if slave_type not in [None, '', 'default', 'zope', 'redirect', 'notebook', 'websocket'] %} {% do slave_error_list.append('type:%s is not supported' % (slave_type,)) %} {% endif %} {# Check health-check-* #} {% set health_check = (str(slave.get('health-check', False)) or 'false').lower() %} {% if health_check in TRUE_VALUES %} {% set health_check_http_method = slave.get('health-check-http-method') or 'GET' %} {% if health_check_http_method not in ['GET', 'OPTIONS', 'CONNECT', 'POST'] %} {% do slave_error_list.append('Wrong health-check-http-method %s' % (health_check_http_method,)) %} {% endif %} {% set health_check_http_path = slave.get('health-check-http-path') or '/' %} {% set health_check_http_version = slave.get('health-check-http-version') or 'HTTP/1.1' %} {% if health_check_http_version not in ['HTTP/1.1', 'HTTP/1.0'] %} {% do slave_error_list.append('Wrong health-check-http-version %s' % (health_check_http_version,)) %} {% endif %} {% set health_check_timeout = (slave.get('health-check-timeout') or '2') | int(false) %} {% if health_check_timeout is false or health_check_timeout <= 0 %} {% do slave_error_list.append('Wrong health-check-timeout %s' % (slave.get('health-check-timeout'),)) %} {% endif %} {% set health_check_interval = (slave.get('health-check-interval') or '5') | int(false) %} {% if health_check_interval is false or health_check_interval <= 0 %} {% do slave_error_list.append('Wrong health-check-interval %s' % (slave.get('health-check-interval'),)) %} {% endif %} {% set health_check_rise = (slave.get('health-check-rise') or '1') | int(false) %} {% if health_check_rise is false or health_check_rise <= 0 %} {% do slave_error_list.append('Wrong health-check-rise %s' % (slave.get('health-check-rise'),)) %} {% endif %} {% set health_check_fall = (slave.get('health-check-fall') or '1') | int(false) %} {% if health_check_fall is false or health_check_fall <= 0 %} {% do slave_error_list.append('Wrong health-check-fall %s' % (slave.get('health-check-fall'),)) %} {% endif %} {% endif %} {# Check virtualhostroot-http-port and virtualhostroot-https-port #} {% for key in ['virtualhostroot-http-port', 'virtualhostroot-https-port'] %} {% set value = (slave.get(key) or '1') | int(false) %} {% if value is false or value < 0 %} {% do slave_error_list.append('Wrong %s %r' % (key, slave.get(key))) %} {% endif %} {% endfor %} {# Check ciphers #} {% set slave_cipher_list = slave.get('ciphers', '').strip().split() %} {% if slave_cipher_list %} {% for cipher in slave_cipher_list %} {% if cipher not in GOOD_CIPHER_LIST %} {% do slave_error_list.append('Cipher %r is not supported.' % (cipher,)) %} {% endif %} {% endfor %} {% endif %} {# Check strict-transport-security #} {% set strict_transport_security = (slave.get('strict-transport-security') or '0') | int(false) %} {% if strict_transport_security is false or strict_transport_security < 0 %} {% do slave_error_list.append('Wrong strict-transport-security %s' % (slave.get('strict-transport-security'),)) %} {% endif %} {% set custom_domain = slave.get('custom_domain') %} {% if custom_domain and custom_domain in used_host_list %} {% set message = 'custom_domain %r clashes' % (custom_domain,) %} {% do slave_error_list.append(message) %} {% do slave_critical_error_list.append(message) %} {% else %} {% do used_host_list.append(custom_domain) %} {% endif %} {% if slave.get('server-alias') %} {% for slave_alias in ('' ~ slave['server-alias']).split() %} {% if slave_alias.startswith('*.') %} {% set clean_slave_alias = slave_alias[2:] %} {% else %} {% set clean_slave_alias = slave_alias %} {% endif %} {% if not validators.domain(clean_slave_alias) %} {% do slave_error_list.append('server-alias \'%s\' not valid' % (slave_alias,)) %} {% else %} {% if slave_alias in slave_server_alias_unclashed or slave_alias == custom_domain %} {# optionally do something about reporting back that server-alias has been unclashed #} {% elif slave_alias in used_host_list %} {% set message = 'server-alias \'%s\' clashes' % (slave_alias,) %} {% do slave_error_list.append(message) %} {% do slave_critical_error_list.append(message) %} {% else %} {% do slave_server_alias_unclashed.append(slave_alias) %} {% do used_host_list.append(slave_alias) %} {% endif %} {% endif %} {% endfor %} {% do slave.__setitem__('server-alias', ' '.join(slave_server_alias_unclashed)) %} {% endif %} {% for url_key in ['url', 'https-url', 'health-check-failover-url', 'health-check-failover-https-url'] %} {% if url_key in slave %} {% set url = (slave[url_key] or '').strip() %} {% if not validators.url(url) %} {% do slave_error_list.append('slave %s %r invalid' % (url_key, url)) %} {% elif url != slave[url_key] %} {% do slave_warning_list.append('slave %s %r has been converted to %r' % (url_key, slave[url_key], url)) %} {% endif %} {% endif %} {% endfor %} {% for url_key in ['url-netloc-list', 'https-url-netloc-list', 'health-check-failover-url-netloc-list'] %} {% if url_key in slave %} {% for netloc in slave[url_key].split() %} {% if not software.validate_netloc(netloc) %} {% do slave_error_list.append('slave %s %r invalid' % (url_key, netloc)) %} {% endif %} {% endfor %} {% endif %} {% endfor %} {% for k in ['ssl_proxy_ca_crt', 'health-check-failover-ssl-proxy-ca-crt'] %} {% if k in slave %} {% set crt = slave.get(k, '') %} {% set check_popen = popen([software_parameter_dict['openssl'], 'x509', '-noout']) %} {% do check_popen.communicate(crt.encode()) %} {% if check_popen.returncode != 0 %} {% do slave_error_list.append('%s is invalid' % (k,)) %} {% endif %} {% endif %} {% endfor %} {# BBB: SlapOS Master non-zero knowledge BEGIN #} {% for key in ['ssl_key', 'ssl_crt', 'ssl_ca_crt'] %} {% if key in slave %} {% do slave_warning_list.append('%s is obsolete, please use key-upload-url' % (key,)) %} {% endif %} {% endfor %} {% if slave.get('ssl_ca_crt') and not (slave.get('ssl_crt') and slave.get('ssl_key')) %} {% do slave_error_list.append('ssl_ca_crt is present, so ssl_crt and ssl_key are required') %} {% endif %} {% if slave.get('ssl_key') and slave.get('ssl_crt') %} {% set key_popen = popen([software_parameter_dict['openssl'], 'rsa', '-noout', '-modulus']) %} {% set crt_popen = popen([software_parameter_dict['openssl'], 'x509', '-noout', '-modulus']) %} {% set key_modulus = key_popen.communicate(slave['ssl_key'].encode())[0] | trim %} {% set crt_modulus = crt_popen.communicate(slave['ssl_crt'].encode())[0] | trim %} {% if not key_modulus or key_modulus != crt_modulus %} {% do slave_error_list.append('slave ssl_key and ssl_crt does not match') %} {% endif %} {% endif %} {# BBB: SlapOS Master non-zero knowledge END #} {% if slave.get('custom_domain') %} {% set slave_custom_domain = '' ~ slave['custom_domain'] %} {% if slave_custom_domain.startswith('*.') %} {% set clean_custom_domain = slave_custom_domain[2:] %} {% else %} {% set clean_custom_domain = slave_custom_domain %} {% endif %} {% if not validators.domain(clean_custom_domain) %} {% do slave_error_list.append('custom_domain %r invalid' % (slave['custom_domain'],)) %} {% endif %} {% endif %} {% if len(slave_error_list) == 0 %} {# Cleanup slave from not needed keys which come from implementation of SlapOS Master #} {# Send only controlled information about the slave to node #} {% set authorized_slave = {} %} {% for key in FRONTEND_NODE_SLAVE_PASSED_KEY_LIST + FRONTEND_NODE_SLAVE_PASSED_KEY_LIST %} {% if key in slave %} {% do authorized_slave.__setitem__(key, slave[key]) %} {% endif %} {% endfor %} {% do authorized_slave_list.append(authorized_slave) %} {% else %} {% do rejected_slave_dict.__setitem__(slave.get('slave_reference'), sorted(slave_error_list)) %} {% endif %} {% if len(slave_critical_error_list) > 0 %} {% do critical_rejected_slave_dict.__setitem__(slave.get('slave_reference'), sorted(slave_critical_error_list)) %} {% endif %} {% if len(slave_warning_list) > 0 %} {% do warning_slave_dict.__setitem__(slave.get('slave_reference'), sorted(slave_warning_list)) %} {% endif %} {% endfor %} {% do authorized_slave_list.sort(key=operator_module.itemgetter('slave_reference')) %} [monitor-instance-parameter] monitor-httpd-port = {{ master_partition_monitor_monitor_httpd_port }} [replicate] <= slap-connection recipe = slapos.cookbook:requestoptional.serialised config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }} config-monitor-username = ${monitor-instance-parameter:username} config-monitor-password = ${monitor-htpasswd:passwd} software-type = {{frontend_type}} return = slave-instance-information-list monitor-base-url backend-client-csr-url kedifa-csr-url csr-certificate backend-haproxy-statistic-url node-information-json {#- Send only needed parameters to frontend nodes #} {%- set base_node_configuration_dict = {} %} {%- for key in FRONTEND_NODE_PASSED_KEY_LIST %} {%- if key in slapparameter_dict %} {%- do base_node_configuration_dict.__setitem__(key, slapparameter_dict[key]) %} {%- endif %} {%- endfor %} {% for section, frontend_request in request_dict.items() %} {% set state = frontend_request.get('state', '') %} [{{section}}] <= replicate name = {{ frontend_request.get('name') }} software-url = {{ frontend_request['software-url'] }} {% if state %} state = {{ state }} {% endif %} {# Do not send additional parameters for destroyed nodes #} {% if state != 'destroyed' %} config-slave-kedifa-information = ${request-kedifa:connection-slave-kedifa-information} config-kedifa-caucase-url = ${request-kedifa:connection-caucase-url} config-backend-client-caucase-url = {{ caucase_url }} config-master-key-download-url = ${request-kedifa:connection-master-key-download-url} config-cluster-identification = {{ instance_parameter_dict['root-instance-title'] }} {% set node_configuration_dict = {} %} {% do node_configuration_dict.update(frontend_request.get('config')) %} {# sort_keys are important in order to avoid shuffling parameters on each run #} {% do node_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list, sort_keys=True)) %} {% do node_configuration_dict.__setitem__("frontend-name", frontend_request.get('name')) %} {%- for config_key, config_value in node_configuration_dict.items() %} config-{{ config_key }} = {{ dumps(config_value) }} {% endfor -%} {%- for config_key, config_value in base_node_configuration_dict.items() %} config-{{ config_key }} = {{ dumps(config_value) }} {% endfor -%} {% if frontend_request.get('sla') %} {% for parameter, value in frontend_request.get('sla').items() %} sla-{{ parameter }} = {{ value }} {% endfor %} {% endif %} {% else %} {# Ignore return for destroyed nodes #} return = {% endif %} {% endfor %} {% set warning_list = [] %} {% for key in ['apache-certificate', 'apache-key'] %} {% if key in slapparameter_dict %} {% do warning_list.append('%s is obsolete, please use master-key-upload-url' % (key, )) %} {% endif %} {% endfor %} [publish-information] <= monitor-publish recipe = slapos.cookbook:publish domain = {{ slapparameter_dict.get('domain') }} slave-amount = {{ instance_parameter_dict['slave-instance-list'] | length }} accepted-slave-amount = {{ authorized_slave_list | length }} rejected-slave-amount = {{ rejected_slave_dict | length }} backend-client-caucase-url = {{ caucase_url }} {# sort_keys are important in order to avoid shuffling parameters on each run #} rejected-slave-dict = {{ dumps(json_module.dumps(rejected_slave_dict, sort_keys=True)) }} rejected-slave-promise-url = ${rejected-slave-promise:config-url} master-key-upload-url = ${request-kedifa:connection-master-key-upload-url} master-key-generate-auth-url = ${request-kedifa:connection-master-key-generate-auth-url} kedifa-caucase-url = ${request-kedifa:connection-caucase-url} {% if len(warning_list) > 0 %} {# sort_keys are important in order to avoid shuffling parameters on each run #} warning-list = {{ dumps(json_module.dumps(warning_list, sort_keys=True)) }} {% endif %} {% if len(warning_slave_dict) > 0 %} {# sort_keys are important in order to avoid shuffling parameters on each run #} warning-slave-dict = {{ dumps(json_module.dumps(warning_slave_dict, sort_keys=True)) }} {% endif %} {% if not aikc_enabled or not aibcc_enabled %} {% for index, frontend in enumerate(frontend_list) %} {% set section_part = '${request-' + frontend %} frontend-node-{{ index + 1 }}-csr-certificate = {{ section_part }}:connection-csr-certificate} {% endfor %} {% endif %} {% if not aikc_enabled %} kedifa-csr-url = ${request-kedifa:connection-kedifa-csr-url} kedifa-csr-certificate = ${request-kedifa:connection-csr-certificate} {% for index, frontend in enumerate(frontend_list) %} {% set section_part = '${request-' + frontend %} frontend-node-{{ index + 1 }}-kedifa-csr-url = {{ section_part }}:connection-kedifa-csr-url} {% endfor %} {% endif %} {% for index, frontend in enumerate(frontend_list) %} {% set section_part = '${request-' + frontend %} frontend-node-{{ index + 1 }}-backend-haproxy-statistic-url = {{ section_part }}:connection-backend-haproxy-statistic-url} frontend-node-{{ index + 1 }}-node-information-json = ${frontend-information:frontend-node-{{ index + 1 }}-node-information-json} {% endfor %} {% if not aibcc_enabled %} {% for index, frontend in enumerate(frontend_list) %} {% set section_part = '${request-' + frontend %} frontend-node-{{ index + 1 }}-backend-client-csr-url = {{ section_part }}:connection-backend-client-csr-url} {% endfor %} {% endif %} # Generate promises for requested nodes {% for index, frontend in enumerate(frontend_list) %} {% set part_name = 'promise-backend-haproxy-statistic-url-' + frontend %} {% do part_list.append(part_name) %} {% set section_part = '${request-' + frontend %} [{{ part_name }}] <= monitor-promise-base promise = check_url_available name = check-backend-haproxy-statistic-url-frontend-node-{{ index + 1 }}.py config-url = {{ section_part }}:connection-backend-haproxy-statistic-url} {% endfor %} #---------------------------- #-- #-- Publish slave information [publish-slave-information] recipe = slapos.cookbook:switch-softwaretype default = instance-publish-slave-information:output RootSoftwareInstance = ${:default} replicate = instance-publish-slave-information:output custom-personal = instance-publish-slave-information:output custom-group = instance-publish-slave-information:output [request-kedifa] <= slap-connection recipe = slapos.cookbook:requestoptional.serialised config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }} config-monitor-username = ${monitor-instance-parameter:username} config-monitor-password = ${monitor-htpasswd:passwd} config-monitor-httpd-port = {{ kedifa_partition_monitor_httpd_port }} {% for key in ['kedifa_port', 'caucase_port'] -%} {%- if key in slapparameter_dict %} config-{{ key }} = {{ dumps(slapparameter_dict[key]) }} {%- endif %} {%- endfor %} config-slave-list = {{ dumps(authorized_slave_list) }} config-cluster-identification = {{ instance_parameter_dict['root-instance-title'] }} {% set software_url_key = "-kedifa-software-release-url" %} {% if software_url_key in slapparameter_dict %} software-url = {{ slapparameter_dict.pop(software_url_key) }} {% else %} software-url = ${slap-connection:software-release-url} {% endif %} software-type = kedifa name = kedifa return = slave-kedifa-information master-key-generate-auth-url master-key-upload-url master-key-download-url caucase-url kedifa-csr-url csr-certificate monitor-base-url {% set sla_kedifa_key = "-sla-kedifa-" %} {% set sla_kedifa_key_length = sla_kedifa_key | length %} {% for key in list(slapparameter_dict.keys()) %} {% if key.startswith(sla_kedifa_key) %} sla-{{ key[sla_kedifa_key_length:] }} = {{ slapparameter_dict.pop(key) }} {% endif %} {% endfor %} [rejected-slave-information] rejected-slave-dict = {{ dumps(rejected_slave_dict) }} [warning-slave-information] warning-slave-dict = {{ dumps(warning_slave_dict) }} [slave-information] {% for frontend_section in frontend_section_list %} {{ frontend_section }} = {{ "${%s:connection-slave-instance-information-list}" % frontend_section }} {% endfor %} [active-slave-instance] {% set active_slave_instance_list = [] %} {% for slave_instance in instance_parameter_dict['slave-instance-list'] %} {# Provide a list of slave titles send by master, in order to filter out already destroyed slaves #} {# Note: This functionality is not yet covered by tests, please modify with care #} {% do active_slave_instance_list.append(slave_instance['slave_reference']) %} {% endfor %} {# sort_keys are important in order to avoid shuffling parameters on each run #} active-slave-instance-list = {{ json_module.dumps(active_slave_instance_list, sort_keys=True) }} [frontend-information] {% for index, frontend in enumerate(frontend_list) %} {% set section_part = '${request-' + frontend %} frontend-node-{{ index + 1 }}-node-information-json = {{ section_part }}:connection-node-information-json} {% endfor %} [instance-publish-slave-information] < = jinja2-template-base url = {{ software_parameter_dict['profile_master_publish_slave_information'] }} filename = instance-publish-slave-information.cfg extensions = jinja2.ext.do extra-context = section slave_information slave-information section frontend_information frontend-information section rejected_slave_information rejected-slave-information section active_slave_instance_dict active-slave-instance section warning_slave_information warning-slave-information key slave_kedifa_information request-kedifa:connection-slave-kedifa-information [monitor-base-url-dict] kedifa = ${request-kedifa:connection-monitor-base-url} {% for frontend in frontend_section_list %} {{ frontend }} = {{ '${' + frontend + ':connection-monitor-base-url}' }} {% endfor %} [directory] recipe = slapos.cookbook:mkdirectory bin = ${buildout:directory}/bin/ srv = ${buildout:directory}/srv/ tmp = ${buildout:directory}/tmp/ backup = ${:srv}/backup # CAUCASE directories caucased = ${:srv}/caucased backup-caucased = ${:backup}/caucased # NGINX rejected-var = ${:var}/rejected-nginx {% if aikc_enabled %} [directory] aikc = ${:srv}/aikc [aikc-config] caucase-url = ${request-kedifa:connection-caucase-url} csr = ${directory:aikc}/csr.pem key = ${directory:aikc}/key.pem ca-certificate = ${directory:aikc}/cas-ca-certificate.pem crl = ${directory:aikc}/crl.pem user-ca-certificate = ${directory:aikc}/user-ca-certificate.pem user-crl = ${directory:aikc}/user-crl.pem user-created = ${directory:aikc}/user-created data_dir = ${directory:aikc}/caucase-updater [aikc-user-csr] recipe = plone.recipe.command organization = {{ instance_parameter_dict['root-instance-title'] }} organizational_unit = Automatic Internal Kedifa Caucase CSR command = if [ ! -f ${:csr} ] && [ ! -f ${:key} ] ; then {{ software_parameter_dict['openssl'] }} req -new -sha256 \ -newkey rsa:2048 -nodes -keyout ${:key} \ -subj "/O=${:organization}/OU=${:organizational_unit}" \ -out ${:csr} fi update-command = ${:command} csr = ${aikc-config:csr} key = ${aikc-config:key} {#- Can be stopped on error, as does not rely on self provided service #} stop-on-error = True [aikc-caucase-wrapper] {# jinja2 instead of wrapper is used with context to remove py'u' #} recipe = slapos.recipe.template:jinja2 context = key caucase_url aikc-config:caucase-url inline = #!{{ software_parameter_dict['dash'] }}/bin/dash exec {{ software_parameter_dict['bin_directory'] }}/caucase \ --ca-url {{ '{{ caucase_url }}' }} \ --ca-crt ${aikc-config:ca-certificate} \ --user-ca-crt ${aikc-config:user-ca-certificate} \ --user-crl ${aikc-config:user-crl} \ --crl ${aikc-config:crl} \ "$@" output = ${directory:bin}/aikc-caucase-wrapper {% do part_list.append('aikc-create-user') %} [aikc-create-user] recipe = plone.recipe.command {#- The called command is smart enough to survive errors and retry #} stop-on-error = False update-command = ${:command} csr_id = ${directory:aikc}/csr_id command = if ! [ -f ${aikc-config:user-created} ] ; then ${aikc-caucase-wrapper:output} --mode user --send-csr ${aikc-user-csr:csr} > ${:csr_id} || exit 1 cut -d ' ' -f 1 ${:csr_id} || exit 1 csr_id=`cut -d ' ' -f 1 ${:csr_id}` sleep 1 ${aikc-caucase-wrapper:output} --mode user --get-crt $csr_id ${aikc-config:key} || exit 1 touch ${aikc-config:user-created} fi {% do part_list.append('aikc-user-caucase-updater') %} {% do part_list.append('aikc-user-caucase-updater-promise') %} {{ caucase.updater( prefix='aikc-user-caucase-updater', buildout_bin_directory=software_parameter_dict['bin_directory'], updater_path='${directory:service}/aikc-user-caucase-updater', url='${aikc-config:caucase-url}', data_dir='${aikc-config:data_dir}', crt_path='${aikc-config:key}', ca_path='${aikc-config:user-ca-certificate}', crl_path='${aikc-config:user-crl}', key_path='${aikc-config:key}', mode='user', )}} [aikc-sign-promise-wrapper] recipe = slapos.cookbook:wrapper command-line = {{ software_parameter_dict['caucase_csr_sign_check'] }} ${aikc-config:caucase-url} ${aikc-config:ca-certificate} ${aikc-config:key} wrapper-path = ${directory:bin}/aikc-caucase-csr-sign-check {% do part_list.append('aikc-sign-promise') %} [aikc-sign-promise] <= monitor-promise-base promise = check_command_execute name = ${:_buildout_section_name_}.py config-command = ${aikc-sign-promise-wrapper:wrapper-path} {% for csr in frontend_list + ['kedifa'] %} [aikc-{{ csr }}-wrapper] recipe = slapos.cookbook:wrapper wrapper-path = ${directory:bin}/aikc-{{ csr }}-wrapper command-line = {{ software_parameter_dict['smart_caucase_signer'] }} ${aikc-config:caucase-url} ${aikc-config:ca-certificate} ${directory:aikc}/{{ csr }}-done ${aikc-config:key} ${request-{{ csr }}:connection-kedifa-csr-url} "${request-{{ csr }}:connection-csr-certificate}" {% do part_list.append('aikc-%s' % (csr,)) %} [aikc-{{ csr }}] recipe = plone.recipe.command {#- The called command is smart enough to survive errors and retry #} stop-on-error = False command = ${aikc-{{ csr }}-wrapper:wrapper-path} update-command = ${:command} {% endfor %} {% endif %} {# if aikc_enabled #} {% if aibcc_enabled %} [directory] aibcc = ${:srv}/aibcc [aibcc-config] caucase-url = {{ caucase_url }} csr = ${directory:aibcc}/csr.pem key = ${directory:aibcc}/key.pem ca-certificate = ${directory:aibcc}/cas-ca-certificate.pem crl = ${directory:aibcc}/crl.pem user-ca-certificate = ${directory:aibcc}/user-ca-certificate.pem user-crl = ${directory:aibcc}/user-crl.pem user-created = ${directory:aibcc}/user-created data_dir = ${directory:aibcc}/caucase-updater [aibcc-user-csr] recipe = plone.recipe.command organization = {{ instance_parameter_dict['root-instance-title'] }} organizational_unit = Automatic Sign Backend Client Caucase CSR command = if [ ! -f ${:csr} ] && [ ! -f ${:key} ] ; then {{ software_parameter_dict['openssl'] }} req -new -sha256 \ -newkey rsa:2048 -nodes -keyout ${:key} \ -subj "/O=${:organization}/OU=${:organizational_unit}" \ -out ${:csr} fi update-command = ${:command} csr = ${aibcc-config:csr} key = ${aibcc-config:key} {#- Can be stopped on error, as does not rely on self provided service #} stop-on-error = True [aibcc-caucase-wrapper] {# jinja2 instead of wrapper is used with context to remove py'u' #} recipe = slapos.recipe.template:jinja2 context = key caucase_url aibcc-config:caucase-url inline = #!{{ software_parameter_dict['dash'] }}/bin/dash exec {{ software_parameter_dict['bin_directory'] }}/caucase \ --ca-url {{ '{{ caucase_url }}' }} \ --ca-crt ${aibcc-config:ca-certificate} \ --user-ca-crt ${aibcc-config:user-ca-certificate} \ --user-crl ${aibcc-config:user-crl} \ --crl ${aibcc-config:crl} \ "$@" output = ${directory:bin}/aibcc-caucase-wrapper {% do part_list.append('aibcc-create-user') %} [aibcc-create-user] recipe = plone.recipe.command # the caucase for this part is provided in this profile, so we can't fail # as otherwise caucase will never be started... {#- XXX: Create promise #} stop-on-error = False update-command = ${:command} csr_id = ${directory:aibcc}/csr_id command = if ! [ -f ${aibcc-config:user-created} ] ; then ${aibcc-caucase-wrapper:output} --mode user --send-csr ${aibcc-user-csr:csr} > ${:csr_id} || exit 1 cut -d ' ' -f 1 ${:csr_id} || exit 1 csr_id=`cut -d ' ' -f 1 ${:csr_id}` sleep 1 ${aibcc-caucase-wrapper:output} --mode user --get-crt $csr_id ${aibcc-config:key} || exit 1 touch ${aibcc-config:user-created} fi {% do part_list.append('aibcc-user-caucase-updater') %} {% do part_list.append('aibcc-user-caucase-updater-promise') %} {{ caucase.updater( prefix='aibcc-user-caucase-updater', buildout_bin_directory=software_parameter_dict['bin_directory'], updater_path='${directory:service}/aibcc-user-caucase-updater', url='${aibcc-config:caucase-url}', data_dir='${aibcc-config:data_dir}', crt_path='${aibcc-config:key}', ca_path='${aibcc-config:user-ca-certificate}', crl_path='${aibcc-config:user-crl}', key_path='${aibcc-config:key}', mode='user', )}} [aibcc-sign-promise-wrapper] recipe = slapos.cookbook:wrapper command-line = {{ software_parameter_dict['caucase_csr_sign_check'] }} ${aibcc-config:caucase-url} ${aibcc-config:ca-certificate} ${aibcc-config:key} wrapper-path = ${directory:bin}/aibcc-caucase-csr-sign-check {% do part_list.append('aibcc-sign-promise') %} [aibcc-sign-promise] <= monitor-promise-base promise = check_command_execute name = ${:_buildout_section_name_}.py config-command = ${aibcc-sign-promise-wrapper:wrapper-path} {% for csr in frontend_list %} [aibcc-{{ csr }}-wrapper] recipe = slapos.cookbook:wrapper wrapper-path = ${directory:bin}/aibcc-{{ csr }}-wrapper command-line = {{ software_parameter_dict['smart_caucase_signer'] }} ${aibcc-config:caucase-url} ${aibcc-config:ca-certificate} ${directory:aibcc}/{{ csr }}-done ${aibcc-config:key} ${request-{{ csr }}:connection-backend-client-csr-url} "${request-{{ csr }}:connection-csr-certificate}" {% do part_list.append('aibcc-%s' % (csr,)) %} [aibcc-{{ csr }}] recipe = plone.recipe.command {#- The called command is smart enough to survive errors and retry #} stop-on-error = False command = ${aibcc-{{ csr }}-wrapper:wrapper-path} update-command = ${:command} {% endfor %} {% endif %} {# if aibcc_enabled #} [rejected-slave-json] recipe = slapos.recipe.template:jinja2 filename = rejected-slave.json directory = ${directory:promise-output} output = ${:directory}/${:filename} url = {{ software_parameter_dict['template_empty'] }} {% if critical_rejected_slave_dict %} {# sort_keys are important in order to avoid shuffling parameters on each run #} content = {{ dumps(json_module.dumps(critical_rejected_slave_dict, indent=2, sort_keys=True)) }} {% else %} content = {% endif %} context = key content :content [directory] service = ${:etc}/service promise-output = ${:srv}/promise-output [rejected-slave-publish-configuration] ip = {{ instance_parameter_dict['ipv6-random'] }} port = 14455 [rejected-slave-publish] directory = ${rejected-slave-json:directory} url = https://${rejected-slave-password:user}:${rejected-slave-password:passwd}@[${rejected-slave-publish-configuration:ip}]:${rejected-slave-publish-configuration:port}/${rejected-slave-json:filename} recipe = slapos.cookbook:wrapper command-line = {{ software_parameter_dict['nginx'] }} -c ${rejected-slave-template:output} wrapper-path = ${directory:service}/rejected-slave-publish hash-existing-files = ${buildout:directory}/software_release/buildout.cfg hash-files = ${rejected-slave-template:output} ${rejected-slave-certificate:certificate} [rejected-slave-certificate] recipe = plone.recipe.command certificate = ${directory:etc}/rejected-slave.pem key = ${:certificate} {#- Can be stopped on error, as does not rely on self provided service #} stop-on-error = True update-command = ${:command} command = [ -f ${:certificate} ] && {{ software_parameter_dict['findutils'] }}/bin/find ${:certificate} -type f -mtime +3 -delete if ! [ -f ${:certificate} ] ; then openssl req -new -newkey rsa:2048 -sha256 -subj \ "/CN=${rejected-slave-publish-configuration:ip}" \ -days 5 -nodes -x509 -keyout ${:certificate} -out ${:certificate} fi [rejected-slave-password] recipe = slapos.cookbook:generate.password storage-path = ${directory:etc}/.rejected-slave.passwd bytes = 8 user = admin [rejected-slave-htpasswd] recipe = plone.recipe.command {#- Can be stopped on error, as does not rely on self provided service #} stop-on-error = True file = ${directory:var}/nginx-rejected.htpasswd {#- update-command is not needed, as if the ${:password} would change, the whole part will be recalculated #} password = ${rejected-slave-password:passwd} command = {{ software_parameter_dict['htpasswd'] }} -cb ${:file} ${rejected-slave-password:user} ${:password} [rejected-slave-template] recipe = slapos.recipe.template:jinja2 var = ${directory:rejected-var} pid = ${directory:var}/nginx-rejected.pid inline = daemon off; pid ${:pid}; error_log stderr; events { } http { include {{ software_parameter_dict['nginx_mime'] }}; server { server_name_in_redirect off; port_in_redirect off; error_log stderr; access_log /dev/null; listen [${rejected-slave-publish-configuration:ip}]:${rejected-slave-publish-configuration:port} ssl; ssl_certificate ${rejected-slave-certificate:certificate}; ssl_certificate_key ${rejected-slave-certificate:certificate}; default_type application/octet-stream; client_body_temp_path ${:var} 1 2; proxy_temp_path ${:var} 1 2; fastcgi_temp_path ${:var} 1 2; uwsgi_temp_path ${:var} 1 2; scgi_temp_path ${:var} 1 2; location / { alias ${rejected-slave-json:directory}/; autoindex off; sendfile on; sendfile_max_chunk 1m; auth_basic "Rejected slave template"; auth_basic_user_file ${rejected-slave-htpasswd:file}; } } } output = ${directory:etc}/nginx-rejected-slave.conf [promise-rejected-slave-publish-ip-port] <= monitor-promise-base promise = check_socket_listening name = rejected-slave-publish-ip-port-listening.py config-host = ${rejected-slave-publish-configuration:ip} config-port = ${rejected-slave-publish-configuration:port} [rejected-slave-promise] <= monitor-promise-base promise = check_file_state name = rejected-slave.py config-filename = ${rejected-slave-json:output} config-state = empty config-url = ${rejected-slave-publish:url} [master-key-upload-url-ready] recipe = slapos.recipe.build directory = ${directory:var} output = ${directory:var}/${:_buildout_section_name_}.txt key = ${request-kedifa:connection-master-key-upload-url} init = import os # protect against early init if os.path.isdir(options['directory']): with open(options['output'], 'w') as fh: if 'NotReadyYet' in options['key']: fh.write('NotReadyYet') else: fh.write('') [master-key-upload-url-ready-promise] <= monitor-promise-base promise = check_file_state name = ${:_buildout_section_name_}.py config-filename = ${master-key-upload-url-ready:output} config-state = empty [master-key-generate-auth-url-ready] <= master-key-upload-url-ready key = ${request-kedifa:connection-master-key-generate-auth-url} [master-key-generate-auth-url-ready-promise] <= master-key-upload-url-ready-promise config-filename = ${master-key-generate-auth-url-ready:output} [master-key-download-url-ready] <= master-key-upload-url-ready key = ${request-kedifa:connection-master-key-download-url} [master-key-download-url-ready-promise] <= master-key-upload-url-ready-promise config-filename = ${master-key-download-url-ready:output} [caucased-backend-client] hash-existing-files = ${buildout:directory}/software_release/buildout.cfg {{ caucase.caucased( prefix='caucased-backend-client', buildout_bin_directory=software_parameter_dict['bin_directory'], caucased_path='${directory:service}/caucased-backend-client', backup_dir='${directory:backup-caucased}', data_dir='${directory:caucased}', netloc=caucase_netloc, tmp='${directory:tmp}', service_auto_approve_count=0, user_auto_approve_count=1, key_len=2048 )}} [buildout] extends = {{ software_parameter_dict['profile_common'] }} {{ software_parameter_dict['profile_monitor2'] }} parts = monitor-base publish-slave-information publish-information request-kedifa rejected-slave-promise promise-rejected-slave-publish-ip-port caucased-backend-client caucased-backend-client-promise master-key-upload-url-ready-promise master-key-generate-auth-url-ready-promise master-key-download-url-ready-promise {% for part in part_list %} {{ ' %s' % part }} {% endfor %}