software/theia: Generate supply/request script for embedded instance

See merge request !1105

software/theia/buildout.hash.cfg

@@ -15,19 +15,19 @@
 
 [instance-theia]
 _update_hash_filename_ = instance-theia.cfg.jinja.in
-md5sum = 78c99ef4799063dc5b67e76b8dba2112
+md5sum = 4df9f0d76a134a8abec9060a0c1be50b
 
 [instance]
 _update_hash_filename_ = instance.cfg.in
-md5sum = 94703df1104405a5a73aa1bc980ea370
+md5sum = f2f01a47d98a980177dc1755e618bbb7
 
 [instance-import]
 _update_hash_filename_ = instance-import.cfg.jinja.in
-md5sum = 57b707cf0ed83be1959d26a88c131906
+md5sum = b0a2c2b3d59fd6c8ba76c634b83a1ba2
 
 [instance-export]
 _update_hash_filename_ = instance-export.cfg.jinja.in
-md5sum = 190a736471f0e0cffcb2838968e01d84
+md5sum = b3f1dd83033d6a45def0bd26e70d5a9c
 
 [instance-resilient]
 _update_hash_filename_ = instance-resilient.cfg.jinja

software/theia/instance-export.cfg.jinja.in

@@ -33,7 +33,7 @@ mode = 0700
 exitcode-file = $${directory:srv}/export-exitcode-file
 error-file = $${directory:srv}/export-errormessage-file
 context =
-  raw python ${software-info:python-with-eggs}
+  raw python ${software-info:python-for-resiliency}
   raw theia_export ${software-info:theia-export}
   raw bash ${software-info:bash}
   raw rsync ${software-info:rsync}

software/theia/instance-import.cfg.jinja.in

@@ -29,6 +29,11 @@ name = Import {{ parameter_dict['additional-frontend-name'] }}
 {%- endif %}
 
 
+# Change standalone socket path to avoid collisions
+[slapos-standalone-config]
+abstract-socket-path = $${directory:home}/standalone-import-ready
+
+
 # Change port ranges to avoid race conditions on port allocation
 [frontend-instance-port]
 minimum = 3200
@@ -79,7 +84,7 @@ mode = 0700
 exitcode-file = $${directory:srv}/import-exitcode-file
 error-file = $${directory:srv}/import-errormessage-file
 context =
-  raw python ${software-info:python-with-eggs}
+  raw python ${software-info:python-for-resiliency}
   raw theia_import ${software-info:theia-import}
   raw bash ${software-info:bash}
   raw rsync ${software-info:rsync}

software/theia/instance-theia.cfg.jinja.in

@@ -151,7 +151,7 @@ config-port = $${slapos-standalone-instance:port}
 <= monitor-promise-base
 promise = check_socket_listening
 name = standalone-ready-promise.py
-config-abstract = $${directory:runner}/standalone_ready
+config-abstract = $${slapos-standalone-config:abstract-socket-path}
 
 [slapos-autorun-promise]
 <= monitor-promise-base
@@ -464,7 +464,7 @@ template =
 
   slapos supply {{ embedded_sr }} slaprunner
 
-  slapos request "Embedded Instance" {{ embedded_sr }}
+  slapos request "embedded_instance" {{ embedded_sr }}
 {%-   if embedded_sr_type %} --type {{ embedded_sr_type }} {%- endif %}
 {%-   if embedded_instance_parameters %} --parameters-file $${embedded-instance-parameters:rendered}
 
@@ -476,6 +476,9 @@ template =
 {%-   endif %}
 {%- endif %}
 
+{%- set embedded_digest = str(embedded_sr) + str(embedded_sr_type) + str(embedded_instance_parameters) %}
+{%- set embedded_digest_hash = hashlib_module.md5(embedded_digest.encode()).hexdigest() %}
+
 
 # SlapOS Standalone
 # -----------------
@@ -493,6 +496,7 @@ port = $${slapos-standalone-port:port}
 local-software-release-root = $${directory:home}
 slapos-configuration = $${directory:runner}/etc/slapos.cfg
 computer-id = slaprunner
+abstract-socket-path = $${directory:home}/standalone-{{ embedded_digest_hash[:16] }}
 
 [slapos-standalone-activate]
 recipe = slapos.recipe.template:jinja2
@@ -504,6 +508,100 @@ template =
   export SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION
   echo 'Standalone SlapOS for computer `$${slapos-standalone-config:computer-id}` activated'
 
+[slapos-standalone-script]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:bin}/$${:_buildout_section_name_}
+template =
+  inline:#!${python-for-standalone:executable}
+  import glob
+  import json
+  import os
+  import signal
+  import socket
+  import subprocess
+  import sys
+  import time
+  import traceback
+
+  import slapos.slap.standalone
+
+  # Include this hash, so that if it changes the standalone service will be restarted
+  # {{ embedded_digest_hash }}
+
+  shared_parts = """{{ '''${buildout:shared-part-list}''' | indent(2) }}"""
+  shared_part_list = [x.strip() for x in shared_parts.splitlines() if x.strip()]
+  partition_forward_configuration = (
+    slapos.slap.standalone.PartitionForwardAsPartitionConfiguration(
+      master_url="$${slap-connection:server-url}",
+      computer="$${slap-connection:computer-id}",
+      partition="$${slap-connection:partition-id}",
+      cert="$${slap-connection:cert-file}",
+      key="$${slap-connection:key-file}",
+      software_release_list=(
+        'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg',
+      ),
+    ),
+  )
+  standalone = slapos.slap.standalone.StandaloneSlapOS(
+    "$${directory:runner}",
+    "$${slapos-standalone-config:ipv4}",
+    $${slapos-standalone-config:port},
+    computer_id="$${slapos-standalone-config:computer-id}",
+    shared_part_list=shared_part_list,
+    software_root="$${directory:runner}/software",
+    instance_root="$${directory:runner}/instance",
+    partition_forward_configuration=partition_forward_configuration,
+    slapos_bin="${buildout:bin-directory}/slapos",
+    local_software_release_root="$${slapos-standalone-config:local-software-release-root}",
+  )
+
+  def signal_handler(signum, frame):
+    print("Signal {signum} received".format(signum=signum))
+    sys.exit()
+  signal.signal(signal.SIGTERM, signal_handler)
+
+  standalone.start()
+  try:
+    partition_count = 20
+    print("Standalone SlapOS: Formatting %d partitions" % partition_count)
+    standalone.format(partition_count, '$${slapos-standalone-config:ipv4}', '$${slapos-standalone-config:ipv6}')
+    print("Standalone SlapOS for computer `$${slapos-standalone-config:computer-id}` started")
+    # Run instance at least once, to start the supervisor managing instances.
+    try:
+      standalone.waitForInstance(max_retry=0)
+    except slapos.slap.standalone.SlapOSNodeCommandError as e:
+      print("Error instanciating: {}".format(e))
+{%- if embedded_sr %}
+    # Compatibility layer
+    try:
+      for cp in standalone.computer.getComputerPartitionList():
+        if cp.getInstanceParameterDict().get("instance_title") == "Embedded Instance":
+          print("Renaming 'Embedded Instance' into 'embedded_instance'")
+          cp.rename(new_name="embedded_instance")
+          break
+    except Exception:
+      print("Exception in compatibility layer, printing and moving on")
+      traceback.print_exc()
+    # Run request script
+    print("Running SlapOS script $${request-embedded-instance-script:rendered}")
+    slapos_env = {
+      'PATH': os.path.dirname(standalone._slapos_bin),
+      'SLAPOS_CONFIGURATION': standalone._slapos_config,
+      'SLAPOS_CLIENT_CONFIGURATION': standalone._slapos_config
+    }
+    subprocess.call(("$${request-embedded-instance-script:rendered}",), env=slapos_env)
+{%- endif %}
+    s = socket.socket(socket.AF_UNIX)
+    s.bind("\0$${slapos-standalone-config:abstract-socket-path}")
+    s.listen(5)
+    print("Standalone SlapOS ready")
+    while True:
+      s.accept()[0].close()
+  finally:
+    print("Stopping standalone subsystem")
+    standalone.stop()
+    print("Exiting")
+
 [slapos-standalone]
 recipe = slapos.recipe.template:jinja2
 rendered = $${directory:bin}/$${:_buildout_section_name_}
@@ -514,21 +612,7 @@ template =
   #XXX find out where the extra nodejs in theia's PATH comes from
   export PATH=${nodejs:location}/bin/:$PATH
   . $${slapos-standalone-activate:rendered}
-  exec ${slapos-standalone:script-path} \
-      $${directory:runner} \
-      $${slapos-standalone-config:ipv4} \
-      $${slapos-standalone-config:ipv6} \
-      $${slapos-standalone-config:port} \
-      $${slapos-standalone-config:local-software-release-root} \
-      $${slapos-standalone-config:computer-id} \
-      {%- if embedded_sr %}
-      --slapos_script=$${request-embedded-instance-script:rendered} \
-      {%- endif %}
-      $${slap-connection:server-url} \
-      $${slap-connection:computer-id} \
-      $${slap-connection:partition-id} \
-      --key='$${slap-connection:key-file}' \
-      --cert='$${slap-connection:cert-file}'
+  exec $${slapos-standalone-script:rendered}
 
 [slapos-standalone-instance]
 recipe = slapos.cookbook:wrapper
@@ -536,6 +620,7 @@ wrapper-path = $${directory:services}/$${:_buildout_section_name_}
 command-line = $${slapos-standalone:rendered}
 hash-files =
   $${slapos-standalone:rendered}
+  $${slapos-standalone-script:rendered}
 hostname = $${slapos-standalone-config:ipv4}
 port = $${slapos-standalone-config:port}
 

software/theia/instance.cfg.in

@@ -40,6 +40,7 @@ context =
   key ipv6_random slap-configuration:ipv6-random
   key ipv4_random slap-configuration:ipv4-random
   import os_module os
+  import hashlib_module hashlib
 default-parameters =
   {
     "autorun": "running",

software/theia/software.cfg

@@ -21,7 +21,8 @@ extends =
 parts =
   theia-wrapper
   slapos-cookbook
-  python-with-eggs
+  python-for-resiliency
+  python-for-standalone
   instance-theia
   instance
   instance-import
@@ -33,133 +34,28 @@ parts =
 # default for slapos-standalone
 shared-part-list =
 
+
+# Versions
+# --------
+
+[gcc]
 # We keep the gcc part in sync with the one from erp5 software, so that when we install
 # erp5 inside theia's slapos parts can be shared.
-[gcc]
 max_version = 0
 
-
 [nodejs]
 <= nodejs-14.16.0
 
 [yarn]
 <= yarn-1.17.3
 
-[slapos-standalone]
-recipe = zc.recipe.egg
-eggs =
-  slapos.core
-scripts = ${:_buildout_section_name_}
-script-path = ${buildout:bin-directory}/${:scripts}
-# XXX generate a fake entry point for a non existant module, that will not
-# be used because we exit in initialization step
-entry-points =
-  ${:scripts}=not_used:main
-initialization =
-  import argparse
-  import glob
-  import json
-  import os
-  import signal
-  import socket
-  import subprocess
-  import sys
-  import time
-
-  import slapos.slap.standalone
-
-  parser = argparse.ArgumentParser()
-  parser.add_argument('base_directory')
-  parser.add_argument('ipv4')
-  parser.add_argument('ipv6')
-  parser.add_argument('server_port', type=int)
-  parser.add_argument('local_software_release_root')
-  parser.add_argument('computer_id')
-  parser.add_argument('--slapos_script')
-  forwarded_arguments = parser.add_argument_group('forwarded')
-  forwarded_arguments.add_argument('master_url')
-  forwarded_arguments.add_argument('computer')
-  forwarded_arguments.add_argument('partition')
-  # cert and key are optional
-  forwarded_arguments.add_argument('--cert')
-  forwarded_arguments.add_argument('--key')
-  args = parser.parse_args()
-  shared_part_list = [x.strip() for x in '''${buildout:shared-part-list}'''.splitlines() if x.strip()]
-  partition_forward_configuration = (
-      slapos.slap.standalone.PartitionForwardAsPartitionConfiguration(
-          master_url=args.master_url,
-          computer=args.computer,
-          partition=args.partition,
-          cert=args.cert,
-          key=args.key,
-          software_release_list=(
-            'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg',
-          ),
-      ),
-  )
-  standalone = slapos.slap.standalone.StandaloneSlapOS(
-      args.base_directory,
-      args.ipv4,
-      args.server_port,
-      computer_id=args.computer_id,
-      shared_part_list=shared_part_list,
-      software_root="%s/software" % args.base_directory,
-      instance_root="%s/instance" % args.base_directory,
-      partition_forward_configuration=partition_forward_configuration,
-      slapos_bin="${buildout:bin-directory}/slapos",
-      local_software_release_root=args.local_software_release_root,
-  )
-
-  def signal_handler(signum, frame):
-    print("Signal {signum} received".format(signum=signum))
-    sys.exit()
-  signal.signal(signal.SIGTERM, signal_handler)
-
-  standalone.start()
-  try:
-    partition_count = 20
-    print("Standalone SlapOS: Formatting {partition_count} partitions".format(
-        partition_count=partition_count))
-    standalone.format(partition_count, args.ipv4, args.ipv6)
-
-    print("Standalone SlapOS for computer `{}` started".format(args.computer_id))
-    # Run instance at least once, to start the supervisor managing instances.
-    try:
-      standalone.waitForInstance(max_retry=0)
-    except slapos.slap.standalone.SlapOSNodeCommandError as e:
-      print("Error instanciating: {}".format(e))
-
-    if args.slapos_script:
-      print("Running SlapOS script {}".format(args.slapos_script))
-      slapos_env = {
-        'PATH': os.path.dirname(standalone._slapos_bin),
-        'SLAPOS_CONFIGURATION': standalone._slapos_config,
-        'SLAPOS_CLIENT_CONFIGURATION': standalone._slapos_config
-      }
-      subprocess.call((args.slapos_script,), env=slapos_env)
-
-    s = socket.socket(socket.AF_UNIX)
-    s.bind('\0' + os.path.join(args.base_directory, 'standalone_ready'))
-    s.listen(5)
-    print("Standalone SlapOS ready")
-    while True:
-      s.accept()[0].close()
-
-  finally:
-    print("Stopping standalone subsystem")
-    standalone.stop()
-    print("Exiting")
+[gowork]
+install +=
+  golang.org/x/tools/gopls@v0.6.6
 
 
-needs-these-eggs-scripts-in-path =
-  ${supervisor:recipe}
-  ${slapos-command:recipe}
-
-[supervisor]
-recipe = zc.recipe.egg
-eggs =
-  supervisor
-  setuptools
+# Downloads and templates
+# -----------------------
 
 [template-base]
 recipe = slapos.recipe.template
@@ -172,16 +68,6 @@ url = ${:_profile_base_location_}/${:_update_hash_filename_}
 destination = ${buildout:directory}/${:_buildout_section_name_}
 output = ${:destination}
 
-[python-language-server]
-version = 0.19.0
-recipe = plone.recipe.command
-command =
-  PATH=${git:location}/bin/:$PATH  bash -c "${python3:executable} -m venv --clear ${:location} && \
-    . ${:location}/bin/activate && \
-    pip install -r ${python-language-server-requirements.txt:output}"
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-stop-on-error = true
-
 [python-language-server-requirements.txt]
 <= download-base
 
@@ -191,27 +77,6 @@ stop-on-error = true
 [logo.png]
 <= download-base
 
-[gowork]
-install +=
-   golang.org/x/tools/gopls@v0.6.6
-
-
-[cli-utilities]
-PATH = ${nodejs:location}/bin:${bash:location}/bin:${fish-shell:location}/bin:${tig:location}/bin:${vim:location}/bin:${tmux:location}/bin:${git:location}/bin:${curl:location}/bin:${python:location}/bin:${buildout:bin-directory}
-
-
-[python-with-eggs]
-recipe = zc.recipe.egg
-interpreter = ${:_buildout_section_name_}
-eggs =
-  ${slapos-toolbox:eggs}
-  six
-  zc.buildout
-# Only generate the interpreter script to avoid conflicts with scripts
-# for eggs that are also generated by another section, like slapos.toolbox
-scripts = ${:interpreter}
-
-
 [instance-theia]
 <= template-base
 output = ${buildout:directory}/instance-theia.cfg.jinja
@@ -243,9 +108,57 @@ destination = ${buildout:directory}/theia_export.py
 <= download-base
 destination = ${buildout:directory}/theia_import.py
 
+
+# Utilities
+# ---------
+
+[supervisor]
+recipe = zc.recipe.egg
+eggs =
+  supervisor
+  setuptools
+
+[python-language-server]
+version = 0.19.0
+recipe = plone.recipe.command
+command =
+  PATH=${git:location}/bin/:$PATH  bash -c "${python3:executable} -m venv --clear ${:location} && \
+    . ${:location}/bin/activate && \
+    pip install -r ${python-language-server-requirements.txt:output}"
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+stop-on-error = true
+
+[cli-utilities]
+PATH = ${nodejs:location}/bin:${bash:location}/bin:${fish-shell:location}/bin:${tig:location}/bin:${vim:location}/bin:${tmux:location}/bin:${git:location}/bin:${curl:location}/bin:${python:location}/bin:${buildout:bin-directory}
+
+
+[python-with-eggs]
+recipe = zc.recipe.egg
+interpreter = ${:_buildout_section_name_}
+# Only generate the interpreter script to avoid conflicts with scripts
+# for eggs that are also generated by another section, like slapos.toolbox
+scripts = ${:interpreter}
+executable = ${buildout:bin-directory}/${:interpreter}
+
+[python-for-resiliency]
+<= python-with-eggs
+eggs =
+  ${slapos-toolbox:eggs}
+  six
+  zc.buildout
+
+[python-for-standalone]
+<= python-with-eggs
+eggs =
+  slapos.core
+needs-these-eggs-scripts-in-path =
+  ${supervisor:recipe}
+  ${slapos-command:recipe}
+
+
 [software-info]
 slapos = ${buildout:bin-directory}/slapos
-python-with-eggs = ${buildout:bin-directory}/${python-with-eggs:interpreter}
+python-for-resiliency = ${python-for-resiliency:executable}
 python = ${python:location}/bin/python
 rsync = ${rsync:location}/bin/rsync
 sqlite3 = ${sqlite3:location}/bin/sqlite3

software/theia/test/test.py

@@ -40,7 +40,7 @@ import six
 
 from six.moves.urllib.parse import urlparse, urljoin
 
-from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass, SlapOSNodeCommandError
 from slapos.grid.svcbackend import getSupervisorRPC, _getSupervisordSocketPath
 
 
@@ -229,26 +229,76 @@ class TestTheiaEmbeddedSlapOSShutdown(TheiaTestCase):
     self.assertFalse(embedded_slapos_process.is_running())
 
 
-class TestTheiaWithSR(TheiaTestCase):
+class ReRequestMixin(object):
+  def rerequest(self, parameter_dict=None, state='started'):
+    software_url = self.getSoftwareURL()
+    software_type = self.getInstanceSoftwareType()
+    name = self.default_partition_reference
+    self.slap.request(
+      software_release=software_url,
+      software_type=software_type,
+      partition_reference=name,
+      partition_parameter_kw=parameter_dict,
+      state=state)
+
+  def reinstantiate(self):
+    # Process at least twice to propagate parameter changes
+    try:
+      self.slap.waitForInstance()
+    except SlapOSNodeCommandError:
+      pass
+    self.slap.waitForInstance(self.instance_max_retry)
+
+
+class TestTheiaWithSR(TheiaTestCase, ReRequestMixin):
   sr_url = '~/bogus/software.cfg'
   sr_type = 'bogus_type'
   instance_parameters = '{\n"bogus_param": "bogus_value",\n"bogus_param2": "bogus_value2"\n}'
 
-  @classmethod
-  def getInstanceParameterDict(cls):
-    return {
-      'embedded-sr': cls.sr_url,
-      'embedded-sr-type': cls.sr_type,
-      'embedded-instance-parameters': cls.instance_parameters
-    }
+  def proxy_show(self, slapos):
+    return subprocess.check_output((slapos, 'proxy', 'show'), universal_newlines=True)
 
   def test(self):
+    slapos = self._getSlapos()
     home = self.computer_partition_root_path
-    bogus_sr = os.path.join(home, self.sr_url[2:])
 
-    slapos = self._getSlapos()
-    info = subprocess.check_output((slapos, 'proxy', 'show'), universal_newlines=True)
-    instance_name = "Embedded Instance"
+    # Check that no request script was generated
+    request_script = os.path.join(home, 'srv', 'project', 'request_embedded.sh')
+    self.assertFalse(os.path.exists(request_script))
+
+    # Manually request old-name 'Embedded Instance'
+    old_instance_name = "Embedded Instance"
+    subprocess.check_call((slapos, 'request', old_instance_name, 'bogus_url'))
+    self.assertIn(old_instance_name, self.proxy_show(slapos))
+
+    # Update Theia instance parameters
+    embedded_request_parameters = {
+      'embedded-sr': self.sr_url,
+      'embedded-sr-type': self.sr_type,
+      'embedded-instance-parameters': self.instance_parameters
+    }
+    self.rerequest(embedded_request_parameters)
+    self.reinstantiate()
+
+    # Check that embedded instance was requested
+    instance_name = "embedded_instance"
+    info = self.proxy_show(slapos)
+    try:
+      self.assertIn(instance_name, info)
+    except AssertionError:
+      for filename in os.listdir(home):
+        if 'standalone' in filename and '.log' in filename:
+          filepath = os.path.join(home, filename)
+          with open(filepath) as f:
+            print("Contents of filepath: " + filepath)
+            print(f.read())
+      raise
+
+    # Check that old-name instance was renamed
+    self.assertNotIn(old_instance_name, info)
+
+    # Check embedded instance parameters
+    bogus_sr = os.path.join(home, self.sr_url[2:])
 
     self.assertIsNotNone(re.search(r"%s\s+slaprunner\s+available" % (bogus_sr,), info), info)
     self.assertIsNotNone(re.search(r"%s\s+%s\s+%s" % (bogus_sr, self.sr_type, instance_name), info), info)