software/theia: Simplify and improve embedded SR

See merge request nexedi/slapos!1139

software/theia/buildout.hash.cfg

@@ -15,24 +15,28 @@
 
 [instance-theia]
 _update_hash_filename_ = instance-theia.cfg.jinja.in
-md5sum = 7ea72f2300dd2019c2d27549e7128dec
+md5sum = 7d146ca31d7e44e909386e21e0a562ec
 
 [instance]
 _update_hash_filename_ = instance.cfg.in
-md5sum = af4d92a95dc25c2f64ddbd634996e46c
+md5sum = e211c439571e2900f9f35482c9638d06
 
 [instance-import]
 _update_hash_filename_ = instance-import.cfg.jinja.in
-md5sum = 29bc1c5b76d20fd46430dff5c72d192b
+md5sum = 23c3df4a889ebfa9f0a94e873e95ad3b
 
 [instance-export]
 _update_hash_filename_ = instance-export.cfg.jinja.in
-md5sum = e2630148998c3cdf6d03b5e884d7f464
+md5sum = 84ceb4c9ee0f07fce8518ef7517ce1d4
 
 [instance-resilient]
 _update_hash_filename_ = instance-resilient.cfg.jinja
 md5sum = ad9499e7355ded4975ad313442cecb7a
 
+[slapos-standalone-script]
+_update_hash_filename_ = slapos_standalone_script.py.jinja
+md5sum = ef5b73648513caf46573f3302953790f
+
 [theia-common]
 _update_hash_filename_ = theia_common.py
 md5sum = 6a25c6a7f1beb27232a3c9acd8a76500

software/theia/instance-export.cfg.jinja.in

@@ -92,14 +92,14 @@ inline =
   then
     echo "ERROR export script last ran on " $(date -r {{ repr(exitcodefile) }})
     exit 1
-  elif [ $( cat {{ repr(exitcodefile) }}) -ne 0 ]
+  elif [ "$(cat {{ repr(exitcodefile) }})" = 0 ]
   then
+    echo "OK export script last ran on " $(date -r {{ repr(exitcodefile) }})
+    exit 0
+  else
     echo "ERROR export script failed on " $(date -r {{ repr(exitcodefile) }})
     cat {{ repr(errorfile) }}
     exit 1
-  else
-    echo "OK export script last ran on " $(date -r {{ repr(exitcodefile) }})
-    exit 0
   fi
 {%- endraw %}
 

software/theia/instance-import.cfg.jinja.in

@@ -154,14 +154,14 @@ inline =
   then
     echo "ERROR import script last ran on " $(date -r {{ repr(exitcodefile) }})
     exit 1
-  elif [ $( cat {{ repr(exitcodefile) }}) -ne 0 ]
+  elif [ "$(cat {{ repr(exitcodefile) }})" = 0 ]
   then
+    echo "OK import script last ran on " $(date -r {{ repr(exitcodefile) }})
+    exit 0
+  else
     echo "ERROR import script failed on " $(date -r {{ repr(exitcodefile) }})
     cat {{ repr(errorfile) }}
     exit 1
-  else
-    echo "OK import script last ran on " $(date -r {{ repr(exitcodefile) }})
-    exit 0
   fi
 {%- endraw %}
 

software/theia/instance-input-schema.json

@@ -15,20 +15,14 @@
       ],
       "default": "running"
     },
-    "embedded-sr": {
-      "title": "Embedded Software URL",
-      "description": "Optional URL of a software to be embedded",
-      "type": "string"
-    },
-    "embedded-sr-type": {
-      "title": "Embedded Software Type",
-      "description": "Type of the optional embedded software",
-      "type": "string"
-    },
-    "embedded-instance-parameters": {
-      "title": "Embedded Instance Parameters",
-      "description": "Parameters for the embedded instance, as a JSON dict",
-      "type": "string"
+    "initial-embedded-instance": {
+      "title": "Initial Embedded Instance Configuration",
+      "description": "One-shot optional JSON preconfiguration for an embedded instance. Only applied once when Theia is instantiated. Changing this option afterward will have no effect.",
+      "type": "string",
+      "examples": [
+        "{\"software-url\": \"~/srv/project/slapos/software/html5as-base/software.cfg\"}",
+        "{\"software-url\": \"~/srv/project/slapos/software/html5as/software.cfg\", \"software-type\": \"replicate\", \"instance-parameters\": {\"replicate-quantity\": 3}}"
+      ]
     },
     "frontend-guid": {
       "title": "Frontend Instance ID",

software/theia/instance.cfg.in

@@ -31,6 +31,7 @@ pull-backup = template-pull-backup:output
 recipe = slapos.recipe.template:jinja2
 url = ${instance-theia:output}
 output = $${buildout:directory}/instance-theia.cfg
+extensions = jinja2.ext.do
 context =
   jsonkey default_parameter_dict :default-parameters
   key parameter_dict slap-configuration:configuration
@@ -38,14 +39,12 @@ context =
   key partition_root_path buildout:directory
   key ipv6_random slap-configuration:ipv6-random
   key ipv4_random slap-configuration:ipv4-random
-  import os_module os
-  import hashlib_module hashlib
+  import os os
+  import json json
 default-parameters =
   {
     "autorun": "running",
-    "embedded-sr": null,
-    "embedded-sr-type": null,
-    "embedded-instance-parameters": null,
+    "initial-embedded-instance": null,
     "frontend-name": "Theia Frontend",
     "frontend-sr": "$${:frontend-sr}",
     "frontend-sr-type": "RootSoftwareInstance",

software/theia/slapos_standalone_script.py.jinja

+#!{{ python_for_standalone }}
+import contextlib
+import glob
+import json
+import logging
+import os
+import re
+import signal
+import socket
+import subprocess
+import sys
+import time
+
+import slapos.slap.standalone
+
+from slapos.util import unicode2str
+
+
+logging.basicConfig(format="[%(asctime)s] %(message)s", level=logging.DEBUG)
+
+
+REQUEST_SCRIPT_RAW_TEMPLATE = """{{ request_script_template }}"""
+
+{%  raw -%}
+REQUEST_SCRIPT_TEMPLATE = re.sub(
+  r"{{\s*(\w+)\s*}}",
+  r"{\1}",
+  REQUEST_SCRIPT_RAW_TEMPLATE,
+)
+{%- endraw %}
+
+REQUEST_SCRIPT_HEADER_TEXT = """
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# This script is generated once by your Theia's SlapOS Standalone service.  #
+#                                                                           #
+# It was used to pre-request an instance inside Theia's embedded SlapOS     #
+# according to the preconfiguration parameters given to your Theia.         #
+#                                                                           #
+# It will not be overwritten or recreated.                                  #
+# It will not be launched automatically.                                    #
+#                                                                           #
+# You can edit it and run it to modify your embedded instance.              #
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+""".strip()
+
+
+def signal_handler(signum, frame):
+  logging.info("Signal %d received", signum)
+  sys.exit()
+
+
+@contextlib.contextmanager
+def setupStandalone():
+  partition_forward_configuration = (
+    slapos.slap.standalone.PartitionForwardAsPartitionConfiguration(
+      master_url={{ repr(slap_connection['server-url']) }},
+      computer={{ repr(slap_connection['computer-id']) }},
+      partition={{ repr(slap_connection['partition-id']) }},
+      cert={{ repr(slap_connection['cert-file']) }},
+      key={{ repr(slap_connection['key-file']) }},
+      software_release_list=(
+        'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg',
+      ),
+    ),
+  )
+  shared_parts = {{ repr(shared_part_list) }}
+  shared_part_list = [x.strip() for x in shared_parts.splitlines() if x.strip()]
+  standalone = slapos.slap.standalone.StandaloneSlapOS(
+    {{ repr(slapos_standalone_config['base-directory']) }},
+    {{ repr(slapos_standalone_config['ipv4']) }},
+    {{ slapos_standalone_config['port'] }},
+    computer_id={{ repr(slapos_standalone_config['computer-id']) }},
+    shared_part_list=shared_part_list,
+    software_root={{ repr(slapos_standalone_config['software-root']) }},
+    instance_root={{ repr(slapos_standalone_config['instance-root']) }},
+    partition_forward_configuration=partition_forward_configuration,
+    slapos_bin={{ repr(slapos_standalone_config['slapos-bin']) }},
+    local_software_release_root={{ repr(slapos_standalone_config['local-software-release-root']) }},
+  )
+  standalone.start()
+  try:
+    partition_count = 20
+    logging.info("Standalone SlapOS: Formatting %d partitions", partition_count)
+    standalone.format(partition_count, {{ repr(slapos_standalone_config['ipv4']) }}, {{ repr(slapos_standalone_config['ipv6']) }})
+    logging.info("Standalone SlapOS for computer `%s` started", {{ repr(slapos_standalone_config['computer-id']) }})
+    # Run instance at least once, to start the supervisor managing instances.
+    try:
+      standalone.waitForInstance(max_retry=0)
+    except slapos.slap.standalone.SlapOSNodeCommandError as e:
+      logging.info("Error instanciating: {}".format(e))
+
+    yield standalone
+
+  finally:
+    logging.info("Stopping standalone subsystem")
+    standalone.stop()
+    logging.info("Exiting")
+
+
+def parseEmbeddedInstanceConfig(config_json_file):
+  with open(config_json_file) as f:
+    try:
+      config = json.load(f)
+    except json.JSONDecodeError:
+      logging.error("%s is not valid JSON", config_json_file)
+      raise
+  assert(isinstance(config, dict))
+  if config:
+    software_url = unicode2str(config['software-url'])
+    software_type = config.get('software-type')
+    instance_parameters = config.get('instance-parameters')
+    assert(isinstance(software_url, str))
+    if software_type:
+      software_type = unicode2str(software_type)
+      assert(isinstance(software_type, str))
+    if instance_parameters:
+      assert(isinstance(instance_parameters, dict))
+    return software_url, software_type, instance_parameters
+
+
+def createRequestScript(software_url, software_type, instance_parameters):
+  # Generate request script
+  request_script_path = {{ repr(request_script_path) }}
+  parameters_file_path = {{ repr(parameters_file_path) }}
+
+  request_options = "embedded_instance " + software_url
+  if software_type:
+    request_options += ' --type ' + software_type
+  if instance_parameters:
+    with open(parameters_file_path, 'w') as f:
+      json.dump(instance_parameters, f)
+    request_options += ' --parameters-file ' + parameters_file_path
+
+  with open(request_script_path, 'w') as f:
+    f.write(REQUEST_SCRIPT_TEMPLATE.format(
+      header_text = REQUEST_SCRIPT_HEADER_TEXT,
+      software_url = software_url,
+      request_options = request_options,
+    ))
+    f.write("\n")
+    os.fchmod(f.fileno(), 0o755)
+
+  return request_script_path
+
+
+def main():
+  signal.signal(signal.SIGTERM, signal_handler)
+
+  with setupStandalone() as standalone:
+
+    config_json_file = {{ repr(embedded_instance_config) }}
+    done_file = config_json_file + '.done'
+
+    if not os.path.exists(done_file):
+      with open(done_file, 'w'): pass
+      try:
+        config = parseEmbeddedInstanceConfig(config_json_file)
+      except Exception:
+        logging.info("Failed to parse embedded instance config", exc_info=True)
+        config = None
+      if config:
+        try:
+          request_script_path = createRequestScript(*config)
+          logging.info("Calling %s", request_script_path)
+          exitcode = subprocess.call((request_script_path,), env={
+            'HOME': {{ repr(home_path) }},
+            'PATH': os.path.dirname(standalone._slapos_bin),
+            'SLAPOS_CONFIGURATION': standalone._slapos_config,
+            'SLAPOS_CLIENT_CONFIGURATION': standalone._slapos_config,
+          })
+          with open({{ repr(embedded_request_exitcode_file) }}, 'w') as f:
+            f.write(str(exitcode))
+        except Exception:
+          logging.info("Failed to request embedded instance", exc_info=True)
+
+    s = socket.socket(socket.AF_UNIX)
+
+    s.bind({{ repr('\0' + slapos_standalone_config['abstract-socket-path']) }})
+    s.listen(5)
+    logging.info("Standalone SlapOS ready")
+
+    while True:
+      s.accept()[0].close()
+
+
+main()

software/theia/software.cfg

@@ -19,17 +19,9 @@ extends =
   ./buildout.hash.cfg
 
 parts =
-  theia-wrapper
   slapos-cookbook
-  python-for-resiliency
-  python-for-standalone
-  instance-theia
   instance
-  instance-import
-  instance-export
-  instance-resilient
-  theia-common
-  theia-export
+
 
 # default for slapos-standalone
 shared-part-list =
@@ -95,6 +87,10 @@ output = ${buildout:directory}/instance-export.cfg.jinja
 [instance-resilient]
 <= download-base
 
+[slapos-standalone-script]
+<= download-base
+destination = ${buildout:directory}/slapos_standalone_script.py.jinja
+
 [theia-common]
 <= download-base
 destination = ${buildout:directory}/theia_common.py
@@ -102,10 +98,12 @@ destination = ${buildout:directory}/theia_common.py
 [theia-export]
 <= download-base
 destination = ${buildout:directory}/theia_export.py
+depends = ${theia-common:recipe}
 
 [theia-import]
 <= download-base
 destination = ${buildout:directory}/theia_import.py
+depends = ${theia-common:recipe}
 
 
 # Utilities

software/theia/test/project_tests.py

@@ -62,10 +62,9 @@ class ERP5Mixin(object):
   _test_software_url = erp5_software_release_url
   _connexion_parameters_regex = re.compile(r"{.*}", re.DOTALL)
 
-  def _getERP5ConnexionParameters(self, software_type='export'):
-    slapos = self._getSlapos(software_type)
-    out = subprocess.check_output(
-      (slapos, 'request', 'test_instance', self._test_software_url),
+  def _getERP5ConnexionParameters(self, instance_type='export'):
+    out = self.captureSlapos(
+      'request', 'test_instance', self._test_software_url,
       stderr=subprocess.STDOUT,
     )
     print(out)
@@ -110,10 +109,10 @@ class ERP5Mixin(object):
       raise Exception("Found several partitions for ERP5 %s" % servicename)
     return found.pop()
 
-  def _getERP5PartitionPath(self, software_type, servicename, *paths):
+  def _getERP5PartitionPath(self, instance_type, servicename, *paths):
     partition = self._getERP5Partition(servicename)
-    return self._getPartitionPath(
-      software_type, 'srv', 'runner', 'instance', partition, *paths)
+    return self.getPartitionPath(
+      instance_type, 'srv', 'runner', 'instance', partition, *paths)
 
 
 class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
@@ -161,16 +160,15 @@ class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
 
     # Update ERP5 parameters
     print('Requesting ERP5 with parameters %s' % params)
-    slapos = self._getSlapos()
-    subprocess.check_call((slapos, 'request', 'test_instance', self._test_software_url, '--parameters', params))
+    self.checkSlapos('request', 'test_instance', self._test_software_url, '--parameters', params)
 
     # Process twice to propagate parameter changes
     for _ in range(2):
-      subprocess.check_call((slapos, 'node', 'instance'))
+      self.checkSlapos('node', 'instance')
 
     # Restart cron (actually all) services to let them take the new date into account
     # XXX this should not be required, updating ERP5 parameters should be enough
-    subprocess.call((slapos, 'node', 'restart', 'all'))
+    self.callSlapos('node', 'restart', 'all')
 
     # Wait until after the programmed backup date, and a bit more
     t = (soon - datetime.now()).total_seconds()
@@ -213,9 +211,8 @@ class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
     self.assertNotIn(self._erp5_new_title, out)
 
     # Stop all services
-    slapos = self._getSlapos()
     print("Stop all services")
-    subprocess.call((slapos, 'node', 'stop', 'all'))
+    self.callSlapos('node', 'stop', 'all')
 
     # Manually restore mariadb from backup
     mariadb_restore_script = os.path.join(mariadb_partition, 'bin', 'restore-from-backup')