Theia: run embedded slapos in a service + bash completions + upate to 1.8.0

See merge request !866

component/bash-completion/buildout.cfg

+[buildout]
+extends = ../xz-utils/buildout.cfg
+parts = bash-completion
+
+[bash-completion]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://github.com/scop/bash-completion/releases/download/2.11/bash-completion-2.11.tar.xz
+md5sum = 2514c6772d0de6254758b98c53f91861
+environment =
+  PATH=${xz-utils:location}/bin:%(PATH)s

software/theia/buildout.hash.cfg

@@ -15,11 +15,11 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = 2ceb9389281c00261abd864fc8ed566f
+md5sum = 0a9b4eb0234339a7ab6098ca4b5caddf
 
 [yarn.lock]
 filename = yarn.lock
-md5sum = c7aa84922a1b80fd8a4c3d96f6ac7e25
+md5sum = 5a89742266a9f9d4115efa6d641fd5bb
 
 [python-language-server-requirements.txt]
 filename = python-language-server-requirements.txt

software/theia/instance.cfg.in

@@ -173,7 +173,7 @@ template =
         ],
         "options": {
           "env": {
-            "SLAPOS_CONFIGURATION": "$${slapos-standalone-activate:slapos-configuration}",
+            "SLAPOS_CONFIGURATION": "$${slapos-standalone-config:slapos-configuration}",
             "GIT_EXEC_PATH": ""
           }
         },
@@ -197,7 +197,7 @@ template =
         ],
         "options": {
           "env": {
-            "SLAPOS_CONFIGURATION": "$${slapos-standalone-activate:slapos-configuration}",
+            "SLAPOS_CONFIGURATION": "$${slapos-standalone-config:slapos-configuration}",
             "GIT_EXEC_PATH": ""
           }
         },
@@ -227,6 +227,7 @@ template =
   export TMP=$${directory:tmp}
   export TEMP=$TMP
   export LC_ALL=C.UTF-8
+  export TERMINFO=${ncurses:location}/lib/terminfo/
   export EDITOR="${python-language-server:location}/bin/python -m theia_open --wait"
   exec ${theia-wrapper:rendered} $@
 ip =  $${instance-parameter:ipv4-random}
@@ -241,7 +242,7 @@ hash-existing-files =
   ${yarn.lock:output}
   ${theia-wrapper:rendered}
 ip =  $${instance-parameter:ipv4-random}
-hostname =  $${:ip}
+hostname = $${:ip}
 port = $${theia-service:port}
 base-url = $${theia-service:base-url}
 
@@ -256,36 +257,76 @@ template = inline:
   import time
   args = sys.argv[1:]
   # when running interactively, activate slapos configuration and reset GIT_EXEC_PATH to workaround https://github.com/eclipse-theia/theia/issues/7555
-  if not args: args = ["-c", ". $${slapos-standalone-activate:rendered} && exec env GIT_EXEC_PATH= ${bash:location}/bin/bash", ]
+  if not args: args = ["-c", ". $${slapos-standalone-activate:rendered} && exec env GIT_EXEC_PATH= ${bash:location}/bin/bash --rcfile $${theia-bashrc:rendered}", ]
   # otherwise, assume this shell is running task and add an artificial delay to workaround https://github.com/eclipse-theia/theia/issues/2961
   else: time.sleep(1)
   os.execv('${bash:location}/bin/bash', ['${bash:location}/bin/bash']  + args)
 
+[theia-bashrc]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:etc}/$${:_buildout_section_name_}
+template =
+  inline:
+  # enable bash completion
+  . ${bash-completion:location}/etc/profile.d/bash_completion.sh
+  # source user's .bashrc
+  [ -f ~/.bashrc ] && . ~/.bashrc
+depends =
+  $${shell-setup-completion:recipe}
+
+[shell-setup-completion]
+recipe = plone.recipe.command
+stop-on-error = true
+command =
+  ${buildout:bin-directory}/slapos complete > $${directory:bash-completions}/slapos
+  ${buildout:bin-directory}/slapos complete --shell fish > $${directory:fish-completions}/slapos.fish
+
+
+[slapos-standalone-config]
+ipv4 = $${instance-parameter:ipv4-random}
+ipv6 = $${instance-parameter:ipv6-random}
+port = 4000
+slapos-configuration = $${directory:slapos}/etc/slapos.cfg
+computer-id = local
+
 [slapos-standalone-activate]
 recipe = slapos.recipe.template:jinja2
 rendered = $${directory:bin}/$${:_buildout_section_name_}
 mode = 0700
-# XXX maybe standalone slapos should provide an activate script like virtualenv is doing?
 template =
   inline:#!/bin/sh
   export PATH=${buildout:bin-directory}:$PATH
-  ${slapos-standalone:script-path} \
+  export SLAPOS_CONFIGURATION=$${slapos-standalone-config:slapos-configuration}
+  export SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION
+  echo 'Standalone SlapOS for computer `$${slapos-standalone-config:computer-id}` activated'
+
+[slapos-standalone]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:bin}/$${:_buildout_section_name_}
+mode = 0700
+template =
+  inline:#!/bin/sh
+  export PATH=${buildout:bin-directory}:$PATH
+  exec ${slapos-standalone:script-path} \
       $${directory:slapos} \
-      $${:ipv4} \
-      $${:ipv6} \
-      $${:port} \
+      $${slapos-standalone-config:ipv4} \
+      $${slapos-standalone-config:ipv6} \
+      $${slapos-standalone-config:port} \
+      $${slapos-standalone-config:computer-id} \
       $${slap-connection:server-url} \
       $${slap-connection:computer-id} \
       $${slap-connection:partition-id} \
       --key='$${slap-connection:key-file}' \
       --cert='$${slap-connection:cert-file}'
-  export SLAPOS_CONFIGURATION=$${:slapos-configuration}
-  export SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION
 
-ipv4 = $${instance-parameter:ipv4-random}
-ipv6 = $${instance-parameter:ipv6-random}
-port = 4000
-slapos-configuration = $${directory:slapos}/etc/slapos.cfg
+[slapos-standalone-instance]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:services}/$${:_buildout_section_name_}
+command-line = $${slapos-standalone:rendered}
+hash-files =
+  $${slapos-standalone:rendered}
+hostname = $${slapos-standalone-config:ipv4}
+port = $${slapos-standalone-config:port}
 
 [promises]
 recipe =
@@ -293,6 +334,7 @@ instance-promises =
   $${theia-listen-promise:name}
   $${frontend-listen-promise:name}
   $${apache-frontend-url-available-promise:name}
+  $${slapos-standalone-listen-promise:name}
 
 [theia-listen-promise]
 <= monitor-promise-base
@@ -315,6 +357,14 @@ name = $${:_buildout_section_name_}.py
 config-url = $${apache-frontend:connection-secure_access}
 config-check-secure = 1
 
+[slapos-standalone-listen-promise]
+<= monitor-promise-base
+module = check_port_listening
+# XXX promise plugins can not contain "slapos" in their names
+name = standalone-listen-promise.py
+config-hostname = $${slapos-standalone-instance:hostname}
+config-port = $${slapos-standalone-instance:port}
+
 [apache-frontend]
 <= slap-connection
 recipe = slapos.cookbook:requestoptional
@@ -358,3 +408,6 @@ slapos = $${:srv}/slapos
 frontend-static = $${:srv}/frontend-static
 frontend-static-public = $${:frontend-static}/public
 frontend-static-css = $${:frontend-static}/css
+
+bash-completions = $${buildout:directory}/.local/share/bash-completion/completions/
+fish-completions = $${buildout:directory}/.config/fish/completions/

software/theia/software.cfg

@@ -4,6 +4,7 @@ extends =
    ../../component/caddy/buildout.cfg
    ../../component/git/buildout.cfg
    ../../component/bash/buildout.cfg
+   ../../component/bash-completion/buildout.cfg
    ../../component/fish-shell/buildout.cfg
    ../../component/tmux/buildout.cfg
    ../../component/tig/buildout.cfg
@@ -55,9 +56,11 @@ entry-points =
   ${:scripts}=not_used:main
 initialization =
   import argparse
+  import glob
   import os.path
   import sys
-  import glob
+  import signal
+  import time
 
   import slapos.slap.standalone
 
@@ -66,6 +69,7 @@ initialization =
   parser.add_argument('ipv4')
   parser.add_argument('ipv6')
   parser.add_argument('server_port', type=int)
+  parser.add_argument('computer_id')
   forwarded_arguments = parser.add_argument_group('forwarded')
   forwarded_arguments.add_argument('master_url')
   forwarded_arguments.add_argument('computer')
@@ -91,6 +95,7 @@ initialization =
       args.base_directory,
       args.ipv4,
       args.server_port,
+      computer_id=args.computer_id,
       shared_part_list=shared_part_list,
       partition_forward_configuration=partition_forward_configuration,
   )
@@ -104,7 +109,26 @@ initialization =
         args.ipv4,
         args.ipv6
     )
-  print ("Standalone SlapOS for computer `{}` activated".format(standalone.computer._computer_id))
+  print("Standalone SlapOS for computer `{}` started".format(args.computer_id))
+  # Run instance at least once, to start the supervisor managing instances.
+  try:
+    standalone.waitForInstance(max_retry=0)
+  except slapos.slap.standalone.SlapOSNodeCommandError as e:
+    print("Error instanciating: {}".format(e))
+
+  quit_requested = []
+  def signal_handler(signum, frame):
+    print("Signal {signum} received".format(signum=signum))
+    quit_requested.append(True)
+  signal.signal(signal.SIGTERM, signal_handler)
+
+  print("Standalone SlapOS ready")
+  while not quit_requested:
+    time.sleep(.1)
+
+  print("Stopping standalone subsystem")
+  standalone.stop()
+  print("Exiting")
   sys.exit(0)
 
 needs-these-eggs-scripts-in-path =
@@ -343,7 +367,7 @@ command =
 
 
 [cli-utilities]
-PATH = ${nodejs:location}/bin/:${bash:location}/bin/:${fish-shell:location}/bin/:${tig:location}/bin/:${vim:location}/bin/:${tmux:location}/bin/:${git:location}/bin/:${curl:location}/bin:${python2.7:location}/bin/
+PATH = ${nodejs:location}/bin/:${bash:location}/bin/:${fish-shell:location}/bin/:${tig:location}/bin/:${vim:location}/bin/:${tmux:location}/bin/:${git:location}/bin/:${curl:location}/bin:${python2.7:location}/bin/:${buildout:bin-directory}
 
 [theia-wrapper]
 recipe = slapos.recipe.template:jinja2

software/theia/test/test.py

@@ -36,9 +36,13 @@ import re
 from six.moves.urllib.parse import urlparse, urljoin
 
 import pexpect
+import psutil
 import requests
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+from slapos.grid.svcbackend import getSupervisorRPC
+from slapos.grid.svcbackend import _getSupervisordSocketPath
+
 
 setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
@@ -110,7 +114,6 @@ class TestTheia(SlapOSInstanceTestCase):
         pass
     process.logfile = DebugLogFile()
 
-    process.expect_exact('Standalone SlapOS: Formatting 20 partitions')
     process.expect_exact('Standalone SlapOS for computer `local` activated')
 
     # try to supply and install a software to check that this slapos is usable
@@ -135,12 +138,6 @@ class TestTheia(SlapOSInstanceTestCase):
     # interrupt this, we don't want to actually wait for software installation
     process.sendcontrol('c')
 
-    # shutdown this slapos
-    process.sendline(
-        'supervisorctl -c {}/srv/slapos/etc/supervisord.conf shutdown'.format(
-            self.computer_partition_root_path))
-    process.expect('Shut down')
-
     process.terminate()
     process.wait()
 
@@ -153,3 +150,37 @@ class TestTheia(SlapOSInstanceTestCase):
         'touch "{}"'.format(test_file)
     ])
     self.assertTrue(os.path.exists(test_file))
+
+
+class TestTheiaEmbeddedSlapOSShutdown(SlapOSInstanceTestCase):
+  __partition_reference__ = 'T' # for sockets in included slapos
+
+  def test_stopping_instance_stops_embedded_slapos(self):
+    embedded_slapos_supervisord_socket = _getSupervisordSocketPath(
+        os.path.join(
+            self.computer_partition_root_path,
+            'srv',
+            'slapos',
+            'inst',
+        ), self.logger)
+
+    # Wait a bit for this supervisor to be started.
+    for _ in range(20):
+      if os.path.exists(embedded_slapos_supervisord_socket):
+        break
+      time.sleep(1)
+
+    # get the pid of the supervisor used to manage instances
+    with getSupervisorRPC(embedded_slapos_supervisord_socket) as embedded_slapos_supervisor:
+      embedded_slapos_process = psutil.Process(embedded_slapos_supervisor.getPID())
+
+    # Stop theia's services
+    with self.slap.instance_supervisor_rpc as instance_supervisor:
+      process_info, = [
+          p for p in instance_supervisor.getAllProcessInfo()
+          if p['name'].startswith('slapos-standalone-instance-')
+      ]
+      instance_supervisor.stopProcessGroup(process_info['group'])
+
+    # the supervisor controlling instances is also stopped
+    self.assertFalse(embedded_slapos_process.is_running())