XXX simplify!!

software/rapid-cdn/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg.in
-md5sum = ea2edd5496ccc65a729a00a8f792ab28
+md5sum = 7ebf557dd088d26b21c95c6749c94642
 
 [profile-common]
 filename = instance-common.cfg.in
@@ -22,11 +22,11 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
 
 [profile-frontend]
 filename = instance-frontend.cfg.in
-md5sum = 2dc6f1e1ad7ab7be4c8cd262e095ece5
+md5sum = a0b28604769b1fe3fd745ef9683984e7
 
 [profile-master]
 filename = instance-master.cfg.in
-md5sum = 2532366077a0b084234aeb6303e9a307
+md5sum = c8103be1781b3289e52c863921e9436a
 
 [profile-slave-list]
 filename = instance-slave-list.cfg.in
@@ -98,7 +98,7 @@ md5sum = 04bc664aa0159acaafec49a6bc36e84b
 
 [software-py]
 filename = software.py
-md5sum = e82ccdb0b26552a1c88ff523d8fae24a
+md5sum = 487233af12c5b41d636a44a21a9daa64
 
 [profile-kedifa]
 filename = instance-kedifa.cfg.in

software/rapid-cdn/instance-master.cfg.in

 {%- import "caucase" as CAUCASE with context %}
 {#- BEGIN: Definition of global variables of the profile #}
-{%- do instance_parameter_dict['configuration'].setdefault('expert', {}) %}
-{%- do instance_parameter_dict['configuration'].setdefault('kedifa-node', {}) %}
-{%- do instance_parameter_dict['configuration']['kedifa-node'].setdefault('expert', {}) %}
+{%- set configuration = instance_parameter_dict['configuration'].copy() %}
+{%- do software_module.merge_dict(configuration, CLUSTER_DEFAULTS) %}
 {%- set SLAVE_TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
 {#- List of keys which shall pass buildout before sending in request to parse ${...:...} #}
 {%- set NEED_BUILDOUT_PASS_REQUEST_KEY_LIST = [
@@ -41,10 +40,12 @@
     ]
 %}
 {%- set FRONTEND_NODE_SLAVE_PASSED_KEY_LIST = FRONTEND_NODE_SLAVE_PASSED_KEY_LIST_SCHEMA + FRONTEND_NODE_SLAVE_PASSED_KEY_LIST_INTERNAL %}
-{%- set AIKC_ENABLED = instance_parameter_dict['configuration'].get('automatic-internal-kedifa-caucase-csr', CLUSTER_DEFAULTS['automatic-internal-kedifa-caucase-csr']) %}
-{%- set AIBCC_ENABLED = instance_parameter_dict['configuration'].get('automatic-internal-backend-client-caucase-csr', CLUSTER_DEFAULTS['automatic-internal-backend-client-caucase-csr']) %}
+{%- set AIKC_ENABLED = configuration['automatic-internal-kedifa-caucase-csr'] %}
+{%- set AIBCC_ENABLED = configuration['automatic-internal-backend-client-caucase-csr'] %}
+# AIKC_ENABLED {{ AIKC_ENABLED }}
+# AIBCC_ENABLED {{ AIBCC_ENABLED }}
 {#- Ports 8401, 8402 and 8410+1..N are reserved for monitor ports on various partitions #}
-{%- set CAUCASE_NETLOC = '[' ~ instance_parameter_dict['ipv6-random'] ~ ']' ~ ':' ~ instance_parameter_dict['configuration']['expert'].get('caucase-port', CLUSTER_DEFAULTS['expert']['caucase-port']) %}
+{%- set CAUCASE_NETLOC = '[' ~ instance_parameter_dict['ipv6-random'] ~ ']' ~ ':' ~ configuration['expert']['caucase-port'] %}
 {%- set CAUCASE_URL = 'http://' ~ CAUCASE_NETLOC %}
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
@@ -60,11 +61,11 @@ context =
 {%- set FRONTEND_LIST = [] %}
 {%- set FRONTEND_SECTION_LIST = [] %}
 {#- # XXX Dirty hack, not possible to define default value before #}
-{%- if not '-sla-1-computer_guid' in instance_parameter_dict['configuration'] %}
-{%-   do instance_parameter_dict['configuration'].__setitem__('-sla-1-computer_guid', '${slap-connection:computer-id}') %}
+{%- if not '-sla-1-computer_guid' in configuration %}
+{%-   do configuration.__setitem__('-sla-1-computer_guid', '${slap-connection:computer-id}') %}
 {%- endif %}
-{%- if not '-sla-kedifa-computer_guid' in instance_parameter_dict['configuration'] %}
-{%-   do instance_parameter_dict['configuration'].__setitem__('-sla-kedifa-computer_guid', '${slap-connection:computer-id}') %}
+{%- if not '-sla-kedifa-computer_guid' in configuration %}
+{%-   do configuration.__setitem__('-sla-kedifa-computer_guid', '${slap-connection:computer-id}') %}
 {%- endif %}
 
 {#- Here we request individually each frontend.
@@ -269,7 +270,7 @@ context =
 {#- END: Slave processing #}
 
 [monitor-instance-parameter]
-monitor-httpd-port = {{ instance_parameter_dict['configuration']['expert'].get('monitor-httpd-port', CLUSTER_DEFAULTS['expert']['monitor-httpd-port']) }}
+monitor-httpd-port = {{ configuration['expert']['monitor-httpd-port'] }}
 
 [replicate]
 <= slap-connection
@@ -280,7 +281,7 @@ return = slave-instance-information-list monitor-base-url backend-client-csr-url
 
 {#- BEGIN: Frontend node request #}
 {%- set DEFAULT_FRONTEND = {
-        'domain': instance_parameter_dict['configuration'].get('domain', CLUSTER_DEFAULTS['domain']),
+        'domain': configuration['domain'],
         'monitor-username': '${monitor-instance-parameter:username}',
         'monitor-password': '${monitor-htpasswd:passwd}',
         'backend-client-caucase-url': CAUCASE_URL,
@@ -290,7 +291,7 @@ return = slave-instance-information-list monitor-base-url backend-client-csr-url
         'kedifa-caucase-url': '${request-kedifa:connection-caucase-url}',
         'master-key-download-url': '${request-kedifa:connection-master-key-download-url}',
 } %}
-{%- for frontend_node_name, frontend_node_dict in instance_parameter_dict['configuration'].get('frontend-node-dict', {}).items() %}
+{%- for frontend_node_name, frontend_node_dict in configuration.get('frontend-node-dict', {}).items() %}
 {%-   set section_name = 'request-' ~ frontend_node_name %}
 [{{ section_name }}]
 {%-   do PART_LIST.append(section_name) %}
@@ -323,7 +324,7 @@ return =
 [publish-information]
 <= monitor-publish
 recipe = slapos.cookbook:publish
-domain = {{ instance_parameter_dict['configuration'].get('domain') }}
+domain = {{ configuration.get('domain') }}
 slave-amount = {{ instance_parameter_dict['slave-instance-list'] | length }}
 accepted-slave-amount = {{ AUTHORIZED_SLAVE_LIST | length }}
 rejected-slave-amount = {{ REJECTED_SLAVE_DICT | length }}
@@ -337,7 +338,7 @@ master-key-generate-auth-url = ${request-kedifa:connection-master-key-generate-a
 kedifa-caucase-url = ${request-kedifa:connection-caucase-url}
 {%-  set warning_list = [] %}
 {%-  for key in ['certificate-chain'] %}
-{%-    if key in instance_parameter_dict['configuration'] %}
+{%-    if key in configuration %}
 {%-      do warning_list.append('%s is obsolete, please use master-key-upload-url' % (key, )) %}
 {%-    endif %}
 {%-  endfor %}
@@ -399,23 +400,23 @@ custom-group = instance-publish-slave-information:output
 [request-kedifa]
 <= slap-connection
 recipe = slapos.cookbook:requestoptional.serialised
-config-monitor-cors-domains = {{ instance_parameter_dict['configuration'].get('monitor-cors-domains', CLUSTER_DEFAULTS['kedifa-node']['expert']['monitor-cors-domains']) }}
+config-monitor-cors-domains = {{ configuration['kedifa-node']['expert']['monitor-cors-domains'] }}
 config-monitor-username = ${monitor-instance-parameter:username}
 config-monitor-password = ${monitor-htpasswd:passwd}
-config-monitor-httpd-port = {{ dumps(instance_parameter_dict['configuration']['kedifa-node']['expert'].get('monitor-httpd-port', CLUSTER_DEFAULTS['kedifa-node']['expert']['monitor-httpd-port'])) }}
-config-caucase_port = {{ dumps(instance_parameter_dict['configuration']['kedifa-node']['expert'].get('caucase-port', CLUSTER_DEFAULTS['kedifa-node']['expert']['caucase-port'])) }}
-config-kedifa_port = {{ dumps(instance_parameter_dict['configuration']['kedifa-node']['expert'].get('kedifa-port', CLUSTER_DEFAULTS['kedifa-node']['expert']['kedifa-port'])) }}
-config-rotate-num = {{ dumps(instance_parameter_dict['configuration']['kedifa-node']['expert'].get('rotate-num', CLUSTER_DEFAULTS['kedifa-node']['expert']['rotate-num'])) }}
+config-monitor-httpd-port = {{ dumps(configuration['kedifa-node']['expert']['monitor-httpd-port']) }}
+config-caucase_port = {{ dumps(configuration['kedifa-node']['expert']['caucase-port']) }}
+config-kedifa_port = {{ dumps(configuration['kedifa-node']['expert']['kedifa-port']) }}
+config-rotate-num = {{ dumps(configuration['kedifa-node']['expert']['rotate-num']) }}
 {%- for key in ['kedifa_port', 'caucase_port'] -%}
-{%-   if key in instance_parameter_dict['configuration'] %}
-config-{{ key }} = {{ dumps(instance_parameter_dict['configuration'][key]) }}
+{%-   if key in configuration %}
+config-{{ key }} = {{ dumps(configuration[key]) }}
 {%-   endif %}
 {%- endfor %}
 config-slave-list = {{ dumps(AUTHORIZED_SLAVE_LIST) }}
 config-cluster-identification = {{ instance_parameter_dict['root-instance-title'] }}
 
-{%- if '-kedifa-software-release-url' in instance_parameter_dict['configuration'] %}
-software-url = {{ instance_parameter_dict['configuration'].pop('-kedifa-software-release-url') }}
+{%- if '-kedifa-software-release-url' in configuration %}
+software-url = {{ configuration.pop('-kedifa-software-release-url') }}
 {%- else %}
 software-url = ${slap-connection:software-release-url}
 {%- endif %}
@@ -424,9 +425,9 @@ name = kedifa
 return = slave-kedifa-information master-key-generate-auth-url master-key-upload-url master-key-download-url caucase-url kedifa-csr-url csr-certificate  monitor-base-url
 {%- set sla_kedifa_key = "-sla-kedifa-" %}
 {%- set sla_kedifa_key_length = sla_kedifa_key | length %}
-{%- for key in list(instance_parameter_dict['configuration'].keys()) %}
+{%- for key in list(configuration.keys()) %}
 {%-   if key.startswith(sla_kedifa_key) %}
-sla-{{ key[sla_kedifa_key_length:] }} = {{ instance_parameter_dict['configuration'].pop(key) }}
+sla-{{ key[sla_kedifa_key_length:] }} = {{ configuration.pop(key) }}
 {%-   endif %}
 {%- endfor %}
 

software/rapid-cdn/instance.cfg.in

@@ -95,6 +95,7 @@ url = {{ software_parameter_dict['profile_frontend'] }}
 filename = instance-frontend.cfg
 extra-context =
   import furl_module furl
+  import software_module software
   raw software_type single-custom-personal
   key SLAVE_DEFAULTS instance-slave-default:defaults
   key FRONTEND_USER_DEFAULTS instance-frontend-user-default:defaults

software/rapid-cdn/software.py

@@ -6,6 +6,7 @@ import sys
 import urllib.error
 import urllib.parse
 import urllib.request
+import collections.abc
 from cryptography import x509
 from cryptography.hazmat.primitives import serialization
 
@@ -121,3 +122,14 @@ def caucase_csr_sign_check():
     sys.exit(1)
   else:
     print('OK No CSR to sign on %s' % (ca_url,))
+
+
+def merge_dict(d, u):
+  # inspired https://stackoverflow.com/a/3233356
+  for k, v in u.items():
+    if isinstance(v, collections.abc.Mapping):
+      d[k] = merge_dict(d.get(k, {}), v)
+    else:
+      if k not in d:
+        d[k] = v
+  return d