apache-frontend: introduce frontend replication

This commit introduce the "replicate" software type. The main instance of type replicate will request frontend and transmit to them all the needed configuration so that it seems transparent.

software/apache-frontend/common.cfg

@@ -14,10 +14,12 @@ extends =
   ../../component/rdiff-backup/buildout.cfg
   ../../component/squid/buildout.cfg
 
-parts =
+parts +=
   slapos-cookbook
   slapos-toolbox
   template
+  template-apache-frontend
+  template-apache-replicate
   binutils
   apache-2.2
   apache-antiloris-apache-2.2
@@ -65,17 +67,29 @@ command =
   grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
 
 [template]
-# Default template for apache instance.
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
-md5sum = 81ab3daac0b6d404744596a2d4f8d8f0
+#md5sum = 81ab3daac0b6d404744596a2d4f8d8f0
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
+[template-apache-frontend]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-apache-frontend.cfg
+#md5sum = 81ab3daac0b6d404744596a2d4f8d8f0
+output = ${buildout:directory}/template-apache-frontend.cfg
+mode = 0644
+
+[template-apache-replicate]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
+#md5sum = 81ab3daac0b6d404744596a2d4f8d8f0
+mode = 0644
+
 [template-slave-list]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/apache-custom-slave-list.cfg.in
-md5sum = e0ea94cbd15284464e4f73bac699b996
+#md5sum = e0ea94cbd15284464e4f73bac699b996
 mode = 640
 
 [template-slave-configuration]
@@ -84,6 +98,11 @@ url = ${:_profile_base_location_}/templates/slave-virtualhost.conf.in
 md5sum = a7ad2e83b7f919fc45a7ef1e64344dcb
 mode = 640
 
+[template-replicate-publish-slave-information]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/templates/replicate-publish-slave-information.cfg.in
+mode = 640
+
 [template-apache-frontend-configuration]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/apache.conf.in
@@ -105,7 +124,7 @@ mode = 640
 [template-custom-slave-list]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/apache-default-slave-list.cfg.in
-md5sum = 62b327008e08940f6f7eda8466976d13
+#md5sum = 62b327008e08940f6f7eda8466976d13
 mode = 640
 
 [template-not-found-html]

software/apache-frontend/instance-apache-replicate.cfg.in

+{% if software_type == slap_software_type -%}
+
+[jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/${:filename}
+extra-context =
+context =
+    import json_module json
+    key eggs_directory buildout:eggs-directory
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key slap_software_type slap-parameter:slap_software_type
+    key slave_instance_list slap-parameter:slave_instance_list
+    ${:extra-context}
+
+{% set part_list = [] -%}
+{% set frontend_type = slapparameter_dict.pop('-frontend-type', 'default') -%}
+{% set frontend_quantity = slapparameter_dict.pop('-frontend-quantity', '2') | int -%}
+{% set slave_list_name = 'extra_slave_instance_list' -%}
+{% set frontend_list = [] %}
+{% set frontend_section_list = [] %}
+{% set namebase = 'apache-frontend' -%}
+
+[replicate]
+<= slap-connection
+recipe = slapos.cookbook:request
+software-url = ${slap-connection:software-release-url}
+software-type = {{frontend_type}}
+return = private-ipv4 public-ipv4 slave-instance-information-list
+config = {{ ' '.join(slapparameter_dict.keys()) + ' ' + slave_list_name }}
+{% for parameter, value in slapparameter_dict.iteritems() -%}
+config-{{parameter}} = {{ value }}
+{% endfor -%}
+config-{{ slave_list_name }} = {{ json_module.dumps(slave_instance_list) }}
+
+{% for i in range(1,frontend_quantity + 1) -%}
+{%   set frontend_name = "%s-%s" % (namebase, i) -%}
+{%   set request_section_title = 'request-%s' % frontend_name -%}
+{%   do frontend_list.append(frontend_name) -%}
+{%   do frontend_section_list.append(request_section_title) -%}
+{%   do part_list.append(request_section_title) -%}
+[{{request_section_title}}]
+<= replicate
+name = {{frontend_name}}
+{% endfor -%}
+
+[publish-information]
+recipe = slapos.cookbook:publish
+domain = {{ slapparameter_dict.get('domain') }}
+slave-amount = {{ slave_instance_list | length }}
+{% for frontend in frontend_list -%}
+#{{frontend}}-private-ipv4 = ${request-{{frontend}}:private-ipv4}
+{% endfor -%}
+
+
+#----------------------------
+#--
+#-- Publish slave information
+[publish-slave-information]
+recipe = slapos.cookbook:softwaretype
+default = ${dynamic-publish-slave-information:rendered}
+replicate = ${dynamic-publish-slave-information:rendered}
+
+[slave-information]
+{% for frontend_section in frontend_section_list -%}
+{{ frontend_section }} = {{ "${%s:connection-slave-instance-information-list}" % frontend_section }}
+{% endfor -%}
+
+[dynamic-publish-slave-information]
+< = jinja2-template-base
+template = {{ template_publish_slave_information }}
+filename = dynamic-publish-slave-information.cfg
+extensions = jinja2.ext.do
+extra-context =
+    section slave_information slave-information
+
+[buildout]
+parts =
+  publish-slave-information
+  publish-information
+{% for part in part_list -%}
+{{ '  %s' % part }}
+{% endfor -%}
+#      publish-information
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+
+[slap_connection]
+# Kept for backward compatiblity
+computer_id = ${slap-connection:computer-id}
+partition_id = ${slap-connection:partition-id}
+server_url = ${slap-connection:server-url}
+software_release_url = ${slap-connection:software-release-url}
+key_file = ${slap-connection:key-file}
+cert_file = ${slap-connection:cert-file}
+
+
+{%- endif %}

software/apache-frontend/templates/apache-custom-slave-list.cfg.in

+{% if software_type == slap_software_type -%}
+
 {% set cached_server_dict = {} -%}
 {% set part_list = [] -%}
 {% set cache_access = "http://%s:%s" % (local_ipv4, cache_port) -%}
 {% set generic_instance_parameter_dict = {'cache_access': cache_access,} -%}
+{% if extra_slave_instance_list -%}
+{%   set slave_instance_information_list = []-%}
+{%   set slave_instance_list = slave_instance_list + json_module.loads(extra_slave_instance_list) -%}
+{% endif -%}
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
 rendered = {{ apache_configuration_directory }}/${:filename}
@@ -66,14 +72,29 @@ apache_custom_https = {{ dumps(apache_custom_https) }}
 {%   endif -%}
 
 # Publish slave information
-{%   set publish_section_title = 'publish-%s-connection-information' % slave_instance.get('slave_reference') -%}
-{%   do part_list.append(publish_section_title) -%}
+{%   if not extra_slave_instance_list -%}
+{%     set publish_section_title = 'publish-%s-connection-information' % slave_instance.get('slave_reference') -%}
+{%     do part_list.append(publish_section_title) -%}
 [{{ publish_section_title }}]
 recipe = slapos.cookbook:publish
 public-ipv4 = {{ public_ipv4 }}
 slave-reference = {{ slave_instance.get('slave_reference') }}
+{%   else -%}
+{%     do slave_instance_information_list.append({'slave-reference':slave_instance.get('slave_reference'), 'public-ipv4':public_ipv4, 'domain':slave_instance.get('domain'), 'url':"http://%s" % slave_instance.get('domain'), 'site_url':"http://%s" % slave_instance.get('domain')}) -%}
+{%   endif -%}
 {% endfor -%}
 
+# Publish information for the instance
+{% set publish_section_title = 'publish-apache-information' -%}
+{% do part_list.append(publish_section_title) -%}
+[{{ publish_section_title }}]
+recipe = slapos.cookbook:publish
+public-ipv4 = {{ public_ipv4 }}
+private-ipv4 = {{ local_ipv4 }}
+domain = {{ domain }}
+{% if extra_slave_instance_list -%}
+slave-instance-information-list = {{ json_module.dumps(slave_instance_information_list) }}
+{% endif -%}
 
 {% do part_list.append('cached-rewrite-rules') -%}
 [cached-rewrite-rules]
@@ -96,4 +117,6 @@ parts +=
 eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
-cache-access = {{ cache_access }}
\ No newline at end of file
+cache-access = {{ cache_access }}
+
+{% endif -%}

software/apache-frontend/templates/apache-default-slave-list.cfg.in

+{% if software_type == slap_software_type -%}
+
 {% set cached_server_dict = {} -%}
 {% set part_list = [] -%}
 {% set cache_access = "http://%s:%s" % (local_ipv4, cache_port) -%}
 {% set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
 {% set generic_instance_parameter_dict = {'cache_access': cache_access,} -%}
+{% if extra_slave_instance_list -%}
+{%   set slave_instance_information_list = []-%}
+{%   set slave_instance_list = slave_instance_list + json_module.loads(extra_slave_instance_list) -%}
+{% endif -%}
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
 rendered = {{ apache_configuration_directory }}/${:filename}
@@ -79,8 +85,9 @@ value = {{ dumps(slave_instance.get(cert_name)) }}
 {%   endfor %}
 
 # Publish slave information
-{%   set publish_section_title = 'publish-%s-connection-information' % slave_instance.get('slave_reference') -%}
-{%   do part_list.append(publish_section_title) -%}
+{%   if not extra_slave_instance_list -%}
+{%     set publish_section_title = 'publish-%s-connection-information' % slave_instance.get('slave_reference') -%}
+{%     do part_list.append(publish_section_title) -%}
 [{{ publish_section_title }}]
 recipe = slapos.cookbook:publish
 slave-reference = {{ slave_instance.get('slave_reference') }}
@@ -89,8 +96,23 @@ domain = {{ slave_instance.get('domain') }}
 url = http://{{ slave_instance.get('domain') }}
 # Backward compatibility
 site_url = ${:url}
+{%   else -%}
+{%     do slave_instance_information_list.append({'slave-reference':slave_instance.get('slave_reference'), 'public-ipv4':public_ipv4, 'domain':slave_instance.get('domain'), 'url':"http://%s" % slave_instance.get('domain'), 'site_url':"http://%s" % slave_instance.get('domain')}) -%}
+{%   endif -%}
 {% endfor -%}
 
+# Publish information for the instance
+{% set publish_section_title = 'publish-apache-information' -%}
+{% do part_list.append(publish_section_title) -%}
+[{{ publish_section_title }}]
+recipe = slapos.cookbook:publish
+public-ipv4 = {{ public_ipv4 }}
+private-ipv4 = {{ local_ipv4 }}
+domain = {{ domain }}
+{% if extra_slave_instance_list -%}
+slave-instance-information-list = {{ json_module.dumps(slave_instance_information_list) }}
+{% endif -%}
+
 
 {% do part_list.append('cached-rewrite-rules') -%}
 # Set rewrite rules for second apache
@@ -116,4 +138,6 @@ parts +=
 eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
-cache-access = {{ cache_access }}
\ No newline at end of file
+cache-access = {{ cache_access }}
+
+{% endif -%}

software/apache-frontend/templates/replicate-publish-slave-information.cfg.in

+{% set part_list = [] -%}
+{% set slave_information_dict = {} -%}
+
+# regroup slave information from all frontends
+{%- for frontend, slave_list_raw in slave_information.iteritems() -%}
+{%   set slave_list = json_module.loads(slave_list_raw) -%}
+{%   for slave_dict in slave_list -%}
+{%     set slave_reference = slave_dict.pop('slave-reference') %}
+{%     set current_slave_dict = slave_information_dict.get(slave_reference, {}) %}
+{%     do current_slave_dict.update(slave_dict) -%}
+{%     do current_slave_dict.__setitem__(
+            'replication_number',
+            current_slave_dict.get('replication_number', 0) + 1
+            ) -%}
+{%     do slave_information_dict.__setitem__(slave_reference, current_slave_dict) -%}
+{%   endfor -%}
+{% endfor %}
+
+# Publish information for each slave
+{% for slave_reference, slave_information in slave_information_dict.iteritems() %}
+{%   set publish_section_title = 'publish-%s' % slave_reference -%}
+{%   do part_list.append(publish_section_title) -%}
+[{{ publish_section_title }}]
+recipe = slapos.cookbook:publish
+slave-reference = {{ slave_reference }}
+{%   for key, value in slave_information.iteritems() -%}
+{{ key }} = {{ value }}
+{%   endfor -%}
+{% endfor %}
+
+[buildout]
+parts =
+{% for part in part_list %}
+{{ '  %s' % part }}
+{% endfor %}
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
\ No newline at end of file