software/theia: Add resilient framework

software/theia/buildout.hash.cfg

@@ -19,7 +19,19 @@ md5sum = 510f8893ab631f5feade6f6aeee98e61
 
 [instance]
 filename = instance.cfg.in
-md5sum = ff5a6d3ef22524f85054e8d0ac0e11d7
+md5sum = 3542f65aaa2c6a4e6db59cab2702ea1c
+
+[instance-import]
+filename = instance-import.cfg.in
+md5sum = 0398a33f33dc99679da0728b2cfbcbf9
+
+[instance-export]
+filename = instance-export.cfg.in
+md5sum = 5cf7154b3731795156972e3e35a8be24
+
+[instance-resilient]
+filename = instance-resilient.cfg.jinja
+md5sum = 01a1bd2a0202d33433e8f08103fcba42
 
 [yarn.lock]
 filename = yarn.lock

software/theia/instance-export.cfg.in

+[buildout]
+extends = ${theia:rendered}
+          ${template-pbsready-export:rendered}
+
+# The resilient stack makes the 'resilient' instance
+# request the 'export' instance with a 'namebase' parameter.
+# The export template then expects to receive it in
+# slap-parameter:namebase
+parts +=
+  monitor-base
+  $${:theia-parts}
+  $${:theia-environment-parts}
+  publish-connection-parameter
+
+# The resilient stack makes the 'resilient' instance
+# request the 'export' instance with a 'namebase' parameter.
+# The export template then expects to receive it in
+# slap-parameter:namebase
+[slap-parameter]
+namebase = ${slap-configuration:configuration.namebase}
+
+# Change frontend name to help disambiguation
+[apache-frontend]
+name = Theia Export Frontend
+
+# The resilient export stack periodically calls exporter:wrapper
+# and then notifies the pull-backup instance that data is ready
+# to be pulled from the export instance.
+# All it expects is that a script be available in exporter:wrapper.
+[exporter]
+recipe = slapos.cookbook:wrapper
+command-line = echo "Export Not Implemented Yet"
+wrapper-path = $${directory:bin}/$${slap-parameter:namebase}-exporter
+wrapper = $${:wrapper-path}
+
+# Extends publish section with resilient parameters
+[publish-connection-parameter]
+<= resilient-publish-connection-parameter

software/theia/instance-import.cfg.in

+[buildout]
+extends = ${theia:rendered}
+          ${template-pbsready-import:rendered}
+
+parts +=
+  monitor-base
+  $${:theia-parts}
+  $${:theia-environment-parts}
+
+# The resilient stack makes the 'resilient' instance
+# request the 'import' instance with a 'namebase' parameter.
+# The import template then expects to receive it in
+# slap-parameter:namebase
+[slap-parameter]
+namebase = ${slap-configuration:configuration.namebase}
+
+# Change frontend name to help disambiguation
+[apache-frontend]
+name = Theia Import Frontend
+
+# The resilient stack calls importer:wrapper when the import instance
+# is notified that the backup files have just been pushed to it.
+# All it expects is that a script be available in importer:wrapper.
+[importer]
+recipe = slapos.cookbook:wrapper
+command-line = echo "Import Not Implemented Yet"
+wrapper-path = $${directory:bin}/$${slap-parameter:namebase}-exporter
+wrapper = $${:wrapper-path}
+
+# Resilient connection parameters of import instance are published
+# through the resilient stack.
+# Extend resilient parameters with normal theia connection parameters
+[resilient-publish-connection-parameter]
+<= publish-connection-parameter

software/theia/instance-resilient.cfg.jinja

+{% import 'parts' as parts %}
+{% import 'replicated' as replicated with context %}
+
+{% set number_of_instances = slapparameter_dict.get('resilient-clone-number', 1)|int %}
+
+[buildout]
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+
+extends =
+  {{ monitor_template }}
+
+parts +=
+# Generate the parts to request theia-export, pull-backup and theia-import
+# See stack/resilient/template-parts.cfg.in and stack/resilient/template-replicated.cfg.in
+# See below for the generation of the sections corresponding to the parts generated here
+  {{ parts.replicate("theia", number_of_instances + 1) }}
+# Also publish some connection parameters
+  publish-connection-parameter
+
+[ArgLeader]
+
+[ArgBackup]
+
+# Generate sections to request theia-export, pull-backup and theia-import
+# See stack/resilient/template-replicated.cfg.in
+# In particular:
+#
+# [request-theia]
+# <= ArgLeader
+# software-type = export
+# ...
+#
+# [request-theia-pseudo-replicating-1]
+# <= ArgBackup
+# software-type = import
+# ...
+#
+# [request-pbs-theia-1]
+# software-type = pull-backup
+# ...
+#
+{{ replicated.replicate("theia", number_of_instances + 1,
+                        "export", "import",
+                        "ArgLeader", "ArgBackup",
+                        slapparameter_dict=slapparameter_dict) }}
+
+# Extend the list of return parameters for the export request
+[request-theia]
+return += url username password
+
+# Publish some parameters from the export instance
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+url = ${request-theia:connection-url}
+username = ${request-theia:connection-username}
+password = ${request-theia:connection-password}
+
+# Publish resiliency parameters fetched by the resilient stack
+[publish-connection-parameter]
+<= publish-connection-information

software/theia/instance.cfg.in

@@ -2,6 +2,9 @@
 parts =
   switch_softwaretype
 
+extends =
+  ${template-resilient-templates:output}
+
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 
@@ -21,6 +24,11 @@ recipe = slapos.cookbook:switch-softwaretype
 RootSoftwareInstance = $${:default}
 default = $${:theia}
 theia = theia:rendered
+export = export:output
+import = import:output
+resilient = resilient:rendered
+frozen = instance-frozen:rendered
+pull-backup = template-pull-backup:rendered
 
 [theia]
 recipe = slapos.recipe.template:jinja2
@@ -29,3 +37,32 @@ rendered = $${buildout:directory}/instance-theia.cfg
 mode = 0644
 context =
   section slapconfiguration_section slap-configuration
+
+[import]
+recipe = slapos.recipe.template
+url = ${instance-import:output}
+output = $${buildout:directory}/instance-import.cfg
+mode = 0644
+
+[export]
+recipe = slapos.recipe.template
+url = ${instance-export:output}
+output = $${buildout:directory}/instance-export.cfg
+mode = 0644
+
+[resilient]
+recipe = slapos.recipe.template:jinja2
+template = ${instance-resilient:output}
+rendered = $${buildout:directory}/instance-resilient.cfg
+mode = 0644
+extensions = jinja2.ext.do
+context =
+  key buildout buildout:bin-directory
+  key develop_eggs_directory buildout:develop-eggs-directory
+  key eggs_directory buildout:eggs-directory
+  key slapparameter_dict slap-configuration:configuration
+  raw monitor_template ${monitor-template:rendered}
+template-parts-destination = ${template-parts:target}
+template-replicated-destination = ${template-replicated:target}
+import-list = file parts :template-parts-destination
+              file replicated :template-replicated-destination

software/theia/software.cfg

@@ -15,6 +15,7 @@ extends =
    ../../stack/nodejs.cfg
    ../../stack/slapos.cfg
    ../../stack/monitor/buildout.cfg
+   ../../stack/resilient/buildout.cfg
    ../../component/defaults.cfg
    ./download-plugins.cfg
    ./buildout.hash.cfg
@@ -24,6 +25,11 @@ common-parts =
   slapos-cookbook
   instance-theia
   instance
+  instance-import
+  instance-export
+  instance-resilient
+# XXX: we have to manually add this for resilience
+  rdiff-backup
 
 parts =
   ${:common-parts}
@@ -155,6 +161,15 @@ url = ${:_profile_base_location_}/${:filename}
 output = ${buildout:parts-directory}/${:_buildout_section_name_}
 mode = 0644
 
+[download-base]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/${:filename}
+destination = ${buildout:directory}
+download-only = true
+ignore-existing = true
+output = ${:destination}/${:filename}
+mode = 0644
+
 [python-language-server]
 version = 0.19.0
 recipe = plone.recipe.command
@@ -331,3 +346,12 @@ output = ${buildout:directory}/instance-theia.cfg.jinja
 [instance]
 <= template-base
 output = ${buildout:directory}/instance.cfg
+
+[instance-import]
+<= download-base
+
+[instance-export]
+<= download-base
+
+[instance-resilient]
+<= download-base