Merge remote-tracking branch 'origin/master' into t

component/fonts/buildout.cfg

@@ -64,6 +64,16 @@ environment =
 url = https://github.com/dejavu-fonts/dejavu-fonts/releases/download/version_2_37/dejavu-fonts-ttf-2.37.tar.bz2
 md5sum = d0efec10b9f110a32e9b8f796e21782c
 
+[source-code-pro-fonts]
+<= fonts-base
+url = https://codeload.github.com/adobe-fonts/source-code-pro/zip/235b72fc43a46cacf36e7c9b45d8d4fc0d121099
+md5sum = b25e165c3fb984bbf6d847adfbd9d694
+
+[jetbrains-mono-fonts]
+<= fonts-base
+url = https://download.jetbrains.com/fonts/JetBrainsMono-2.001.zip
+md5sum = 62f02985bfef43a27dbdd17641fec210
+
 # Microsoft's TrueType core fonts
 # non-free so not enabled by default
 [msttcore-fonts]

component/gcc/buildout.cfg

@@ -42,13 +42,11 @@ post-install =
   done
   cat <<EOF >ld
   #!/bin/sh -e
-  case \$#:\$1 in 0:|1:-*) ;; *)
-    if [ "\$LD_RUN_PATH" ]
-    then LD_RUN_PATH=\$LD_RUN_PATH$1
-    else set -- "\$@" $2
-    fi
+  for x; do case \$x in -rpath|-rpath=*)
+    set -- "\$@" $2
+    ! break
     ;;
-  esac
+  esac; done && export LD_RUN_PATH=\$${LD_RUN_PATH:+\$LD_RUN_PATH:}$${1#:}
   exec ${binutils:location}/bin/ld "\$@"
   EOF
   chmod +x ld

component/pure-ftpd/buildout.cfg

@@ -3,8 +3,8 @@ parts = pure-ftpd
 
 [pure-ftpd]
 recipe = slapos.recipe.cmmi
-url = https://download.pureftpd.org/pub/pure-ftpd/releases/pure-ftpd-1.0.46.tar.bz2
-md5sum = efce5529c1f0a39dafdd532c619503f1 
+url = https://download.pureftpd.org/pub/pure-ftpd/releases/pure-ftpd-1.0.49.tar.bz2
+md5sum = b7025f469711d88bd84a3518f67c1470 
 
 # See https://download.pureftpd.org/pub/pure-ftpd/doc/README for more configurations
 # We need the trick about UPLOAD_PIPE_FILE and UPLOAD_PIPE_LOCK so that the files are created inside the $CWD/var/run

component/quickjs/buildout.cfg

+[buildout]
+extends =
+  ../xz-utils/buildout.cfg
+parts = quickjs
+
+[quickjs]
+recipe = slapos.recipe.cmmi
+configure-command = true
+url = https://bellard.org/quickjs/quickjs-2020-09-06.tar.xz
+md5sum = 47cb6def3263d6a631b647b268b1c554
+environment =
+  PATH=${xz-utils:location}/bin:%(PATH)s

software/caddy-frontend/test/test.py

@@ -6764,6 +6764,16 @@ class TestPassedRequestParameter(HttpFrontendTestCase):
     cls.slap.supply(cls.frontend_3_sr, cls.slap._computer_id)
     cls.slap.supply(cls.kedifa_sr, cls.slap._computer_id)
 
+  @classmethod
+  def tearDownClass(cls):
+    cls.slap.supply(
+      cls.frontend_2_sr, cls.slap._computer_id, state="destroyed")
+    cls.slap.supply(
+      cls.frontend_3_sr, cls.slap._computer_id, state="destroyed")
+    cls.slap.supply(
+      cls.kedifa_sr, cls.slap._computer_id, state="destroyed")
+    super(TestPassedRequestParameter, cls).tearDownClass()
+
   instance_parameter_dict = {
       'port': HTTPS_PORT,
       'plain_http_port': HTTP_PORT,

software/pureftpd/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [instance-profile]
 filename = instance.cfg.in
-md5sum = c352c6f11b7a00dca0a544f7ecddeb52
+md5sum = 134ed1cf1f6de63b14425031eb5c9043

software/pureftpd/instance.cfg.in

@@ -32,12 +32,9 @@ plugin = ${:etc}/plugin
 pureftpd-dir = ${:srv}/pureftpd/
 
 [check-port-listening-promise]
-recipe = slapos.cookbook:promise.plugin
-eggs =
-  slapos.toolbox
-output = ${directory:plugin}/${:_buildout_section_name_}
-content =
-  from slapos.promise.plugin.check_port_listening import RunPromise
+<= monitor-promise-base
+module = check_port_listening
+name = check_nginx_port.py
 
 [pureftpd-listen-promise]
 <= check-port-listening-promise

software/pureftpd/software.cfg

@@ -9,13 +9,16 @@ extends =
 parts =
   slapos-cookbook
   instance-profile
+  
+  
+[python]
+part = python3
 
 
-# force to install plone.recipe.command and slapos.toolbox as it will be used during instanciation 
+# force to install plone.recipe.command as it will be used during instanciation 
 [slapos-cookbook]
 eggs +=
   plone.recipe.command
-  slapos.toolbox
 
 
 [instance-profile]

software/slaprunner/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 8d6878ff1d2e75010c50a1a2b0c13b24
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = 376ae851bb13bd88b02ecd72249a64bd
+md5sum = 6e279c46b07bf56b7b037a8ee2c6587e
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2

software/slaprunner/instance-runner.cfg

@@ -20,6 +20,7 @@ common-runner-parts =
   runtestsuite
   shellinabox
   shellinabox-service
+  template-slapuser-script
   slapos-cfg
   slapformat-definition.cfg
   cron-entry-prepare-software
@@ -30,6 +31,8 @@ common-runner-parts =
   bash-profile
   supervisord-wrapper
   supervisord-promise
+  slapos-supervisord-promise
+  slapos-proxy-promise
   logrotate-entry-supervisord
   logrotate-entry-slapgrid
   httpd-graceful-wrapper
@@ -213,8 +216,7 @@ default_repository_branch = $${slap-parameter:slapos-reference}
 #-- supervisord managing slaprunner instance processes
 [slaprunner-supervisord-wrapper]
 recipe = slapos.cookbook:wrapper
-# XXX hardcoded locations
-command-line = $${template-slapuser-script:rendered} node supervisord -n
+command-line = ${buildout:bin-directory}/slapos node supervisord --cfg=$${slaprunner:slapos.cfg} --nodaemon
 wrapper-path = $${directory:services}/slaprunner-supervisord
 hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
 
@@ -648,7 +650,13 @@ monitor-interface-url =
 monitor-httpd-port = 8386
 buildout-shared-folder = $${runnerdirectory:home}/shared
 {% for k, v in slapparameter_dict.items() -%}
+{% if k == 'user-authorized-key' and v -%}
+{% set key_list =  v.split('\n') -%}
+{{ k }} =
+  {{ key_list | join('\n  ') }}
+{% else -%}
 {{ k }} = {{ v }}
+{% endif -%}
 {% endfor -%}
 
 [slapos-cfg]
@@ -858,6 +866,19 @@ name = supervisord.py
 config-hostname = $${slaprunner:ipv4}
 config-port = $${supervisord:port}
 
+[slapos-supervisord-promise]
+<= monitor-promise-base
+module = check_command_execute
+name = instance_supervisord.py
+config-command = ${buildout:bin-directory}/slapos node supervisorctl --cfg=$${slaprunner:slapos.cfg} pid
+
+[slapos-proxy-promise]
+<= monitor-promise-base
+module = check_port_listening
+name = slaproxy.py
+config-hostname = $${slaprunner:ipv4}
+config-port = $${slaprunner:proxy_port}
+
 # XXX Monitor
 [monitor-instance-parameter]
 monitor-httpd-port = $${slap-parameter:monitor-httpd-port}

software/slaprunner/test/test.py

@@ -321,9 +321,11 @@ class TestWeb(SlaprunnerTestCase):
 class TestSSH(SlaprunnerTestCase):
   @classmethod
   def getInstanceParameterDict(cls):
-    cls.ssh_key = paramiko.RSAKey.generate(1024)
+    cls.ssh_key_list = [paramiko.RSAKey.generate(1024) for i in range(2)]
     return {
-        'user-authorized-key': 'ssh-rsa {}'.format(cls.ssh_key.get_base64())
+        'user-authorized-key': 'ssh-rsa {}\nssh-rsa {}'.format(
+          *[key.get_base64() for key in cls.ssh_key_list]
+          )
     }
 
   def test_connect(self):
@@ -355,43 +357,44 @@ class TestSSH(SlaprunnerTestCase):
     key_policy = KeyPolicy()
     client.set_missing_host_key_policy(key_policy)
 
-    with contextlib.closing(client):
-      client.connect(
-          username=username,
-          hostname=parsed.hostname,
-          port=parsed.port,
-          pkey=self.ssh_key,
-      )
-      # Check fingerprint from server matches the published one.
-      # Paramiko does not allow to get the fingerprint as SHA256 easily yet
-      # https://github.com/paramiko/paramiko/pull/1103
-      self.assertEqual(
-          fingerprint_from_url,
-          quote(
-              # base64 encoded fingerprint adds an extra = at the end
-              base64.b64encode(
-                  hashlib.sha256(key_policy.key.asbytes()).digest())[:-1],
-              # also encode /
-              safe=''))
-
-      # Check shell is usable
-      channel = client.invoke_shell()
-      channel.settimeout(30)
-      received = ''
-      while True:
-        r = bytes2str(channel.recv(1024))
-        self.logger.debug("received >%s<", r)
-        if not r:
-          break
-        received += r
-        if 'slaprunner shell' in received:
-          break
-      self.assertIn("Welcome to SlapOS slaprunner shell", received)
-
-      # simple commands can also be executed ( this would be like `ssh bash -c 'pwd'` )
-      self.assertEqual(
-          self.computer_partition_root_path,
-          bytes2str(client.exec_command("pwd")[1].read(1000)).strip())
+    for ssh_key in self.ssh_key_list:
+      with contextlib.closing(client):
+        client.connect(
+            username=username,
+            hostname=parsed.hostname,
+            port=parsed.port,
+            pkey=ssh_key,
+        )
+        # Check fingerprint from server matches the published one.
+        # Paramiko does not allow to get the fingerprint as SHA256 easily yet
+        # https://github.com/paramiko/paramiko/pull/1103
+        self.assertEqual(
+            fingerprint_from_url,
+            quote(
+                # base64 encoded fingerprint adds an extra = at the end
+                base64.b64encode(
+                    hashlib.sha256(key_policy.key.asbytes()).digest())[:-1],
+                # also encode /
+                safe=''))
+
+        # Check shell is usable
+        channel = client.invoke_shell()
+        channel.settimeout(30)
+        received = ''
+        while True:
+          r = bytes2str(channel.recv(1024))
+          self.logger.debug("received >%s<", r)
+          if not r:
+            break
+          received += r
+          if 'slaprunner shell' in received:
+            break
+        self.assertIn("Welcome to SlapOS slaprunner shell", received)
+
+        # simple commands can also be executed ( this would be like `ssh bash -c 'pwd'` )
+        self.assertEqual(
+            self.computer_partition_root_path,
+            bytes2str(client.exec_command("pwd")[1].read(1000)).strip())
 
 
 class TestSlapOS(SlaprunnerTestCase):

software/theia/buildout.hash.cfg

@@ -15,12 +15,24 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = f95354d4af4a78ad7fd11ebc9281ed19
+md5sum = 397fcb3279029af3055b23525d147445
 
 [yarn.lock]
 filename = yarn.lock
-md5sum = 07835b4acfbd8f4a96ac15ffecd238b0
+md5sum = d058e73c3d90ac3da44734c2d47eac95
 
 [python-language-server-requirements.txt]
 filename = python-language-server-requirements.txt
-md5sum = d2ce161244ce9ebce5295302a1b2a7df
+md5sum = 0883a40ebcb33d8d7c520490b21bd16c
+
+[preloadTemplate.html]
+filename = preloadTemplate.html
+md5sum = 8157c22134200bd862a07c6521ebf799
+
+[slapos.css.in]
+filename = slapos.css.in
+md5sum = d2930ec3ef973b7908f0fa896033fd64
+
+[logo.png]
+filename = logo.png
+md5sum = 97bd0f828ffbac2681af0f4bd72cba27

software/theia/instance.cfg.in

@@ -54,7 +54,7 @@ template = inline:
     root $${directory:frontend-static}
     browse
     proxy / $${theia-instance:base-url} {
-      except public $${favicon.ico:filename}
+      except $${frontend-instance-fonts:folder-name} $${frontend-instance-slapos.css:folder-name} public $${favicon.ico:filename} $${frontend-instance-logo:filename}
     }
     proxy /services $${theia-instance:base-url} {
       websocket
@@ -83,6 +83,33 @@ port = $${frontend-instance-config:port}
 pidfile = $${directory:pidfiles}/$${:_buildout_section_name_}.pid
 url = https://$${:hostname}:$${:port}/
 
+[frontend-instance-fonts]
+; XXX caddy 1 does not seem to serve different folders at different locations
+; so we link fonts in static folder
+recipe = plone.recipe.command
+location = $${directory:frontend-static}/$${:folder-name}
+folder-name = fonts
+command =
+  mkdir $${:location}
+  ln -s ${source-code-pro-fonts:location} $${:location}/source-code-pro
+  ln -s ${jetbrains-mono-fonts:location} $${:location}/jetbrains-mono
+stop-on-error = true
+
+[frontend-instance-logo]
+recipe = plone.recipe.command
+filename = logo.png
+command =
+  ln -s ${logo.png:output} $${directory:frontend-static}/$${:filename}
+stop-on-error = true
+
+[frontend-instance-slapos.css]
+recipe = slapos.recipe.template:jinja2
+template = ${slapos.css.in:output}
+rendered = $${directory:frontend-static}/$${:folder-name}/slapos.css
+folder-name = css
+context =
+  key logo_image frontend-instance-logo:filename
+
 [frontend-reload]
 recipe = slapos.cookbook:wrapper
 wrapper-path = $${directory:services}/$${:_buildout_section_name_}
@@ -220,11 +247,14 @@ rendered = $${directory:bin}/$${:_buildout_section_name_}
 mode = 0700
 template = inline:
   #!${python:location}/bin/python
-  import sys
   import os
+  import sys
+  import time
   args = sys.argv[1:]
   # when running interactively, activate slapos configuration and reset GIT_EXEC_PATH to workaround https://github.com/eclipse-theia/theia/issues/7555
   if not args: args = ["-c", ". $${slapos-standalone-activate:rendered} && exec env GIT_EXEC_PATH= ${bash:location}/bin/bash", ]
+  # otherwise, assume this shell is running task and add an artificial delay to workaround https://github.com/eclipse-theia/theia/issues/2961
+  else: time.sleep(1)
   os.execv('${bash:location}/bin/bash', ['${bash:location}/bin/bash']  + args)
 
 [slapos-standalone-activate]
@@ -323,3 +353,4 @@ project = $${:srv}/project
 slapos = $${:srv}/slapos
 frontend-static = $${:srv}/frontend-static
 frontend-static-public = $${:frontend-static}/public
+frontend-static-css = $${:frontend-static}/css

software/theia/preloadTemplate.html

+<script>
+let link = document.createElement('link');
+link.rel = "stylesheet";
+link.href = "/css/slapos.css";
+document.head.appendChild(link);
+</script>
\ No newline at end of file

software/theia/python-language-server-requirements.txt

@@ -14,7 +14,7 @@ parso==0.3.2
 pluggy==0.8.1
 pydocstyle==3.0.0
 pyflakes==2.1.0
-pygls==0.8.1
+pygls==0.9.1
 pylint==2.4.4
 python-jsonrpc-server==0.1.2
 -e git+https://github.com/palantir/python-language-server@50d03d5931d564e9908292ccfa21dd629ee817ba#egg=python_language_server
@@ -26,5 +26,5 @@ typed-ast==1.4.1
 typing-extensions==3.7.4.2
 wrapt==1.11.2
 yapf==0.29.0
-zc.buildout.languageserver==0.2.1
-theia-open==0.1.2
+zc.buildout.languageserver==0.4.0
+theia-open==0.3.0

software/theia/software.cfg

@@ -11,6 +11,7 @@ extends =
    ../../component/curl/buildout.cfg
    ../../component/coreutils/buildout.cfg
    ../../component/java-jdk/buildout.cfg
+   ../../component/fonts/buildout.cfg
    ../../stack/slapos.cfg
    ../../stack/monitor/buildout.cfg
    ../../component/defaults.cfg
@@ -116,6 +117,12 @@ eggs =
   supervisor
   setuptools
 
+[template-base]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:parts-directory}/${:_buildout_section_name_}
+mode = 0644
+
 [python-language-server]
 version = 0.19.0
 recipe = plone.recipe.command
@@ -127,11 +134,7 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
 stop-on-error = true
 
 [python-language-server-requirements.txt]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/${:filename}
-output = ${buildout:parts-directory}/${:_buildout_section_name_}
-mode = 0644
-
+<= template-base
 
 [theia]
 recipe = plone.recipe.command
@@ -152,10 +155,16 @@ uses = ${yarn.lock:recipe}
 THEIA_DEFAULT_PLUGINS = ${:location}/plugins/
 
 [yarn.lock]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/${:filename}
-output = ${buildout:parts-directory}/${:_buildout_section_name_}
-mode = 0644
+<= template-base
+
+[preloadTemplate.html]
+<= template-base
+
+[slapos.css.in]
+<= template-base
+
+[logo.png]
+<= template-base
 
 [package.json]
 recipe = slapos.recipe.template:jinja2
@@ -193,6 +202,11 @@ template =
                       "java.home": "${java-jdk:location}"
                   }
               }
+          },
+          "generator": {
+              "config": {
+                  "preloadTemplate": "${preloadTemplate.html:output}"
+              }
           }
       },
       "dependencies": {
@@ -309,8 +323,11 @@ template =
           "vscode-java-test": "https://github.com/microsoft/vscode-java-test/releases/download/0.22.0/vscjava.vscode-java-test-0.22.0.vsix",
           "vscode-python": "https://github.com/microsoft/vscode-python/releases/download/2020.1.58038/ms-python-release.vsix",
           "vscode-ruby": "https://github.com/rubyide/vscode-ruby/releases/download/v0.25.0/ruby-0.25.0.vsix",
-          "vscode-zc-buildout": "https://github.com/perrinjerome/vscode-zc-buildout/releases/download/v0.2.0/vscode-zc-buildout-0.2.0.vsix",
-          "plantuml": "https://open-vsx.org/api/jebbs/plantuml/2.13.12/file/jebbs.plantuml-2.13.12.vsix"
+          "vscode-zc-buildout": "https://github.com/perrinjerome/vscode-zc-buildout/releases/download/v0.4.0/vscode-zc-buildout-0.4.0.vsix",
+          "plantuml": "https://open-vsx.org/api/jebbs/plantuml/2.13.12/file/jebbs.plantuml-2.13.12.vsix",
+          "diff": "https://open-vsx.org/api/rafaelmaiolla/diff/0.0.1/file/rafaelmaiolla.diff-0.0.1.vsix",
+          "git-commit-syntax": "https://github.com/perrinjerome/git-commit-syntax/releases/download/v0.0.1/git-commit-syntax-0.0.1.vsix",
+          "git-rebase-syntax": "https://github.com/perrinjerome/git-rebase-syntax/releases/download/v0.0.1/git-rebase-syntax-0.0.1.vsix"
       }
     }
 rendered = ${buildout:directory}/${:_buildout_section_name_}
@@ -345,9 +362,7 @@ template =
 
 
 [instance]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/${:filename}
-mode = 0644
+<= template-base
 output = ${buildout:directory}/instance.cfg
 
 [versions]

software/theia/test/test.py

@@ -32,6 +32,7 @@ import logging
 import subprocess
 import tempfile
 import time
+import re
 from six.moves.urllib.parse import urlparse, urljoin
 
 import pexpect
@@ -81,6 +82,16 @@ class TestTheia(SlapOSInstanceTestCase):
     self.assertEqual(requests.codes.ok, resp.status_code)
     self.assertTrue(resp.raw)
 
+    # there is a CSS referencing fonts
+    css_text = requests.get(urljoin(authenticated_url, '/css/slapos.css'), verify=False).text
+    css_urls = re.findall(r'url\([\'"]+([^\)]+)[\'"]+\)', css_text)
+    self.assertTrue(css_urls)
+    # and fonts are served
+    for url in css_urls:
+      resp = requests.get(urljoin(authenticated_url, url), verify=False)
+      self.assertEqual(requests.codes.ok, resp.status_code)
+      self.assertTrue(resp.raw)
+
   def test_theia_slapos(self):
     # Make sure we can use the shell and the integrated slapos command
     process = pexpect.spawnu(
@@ -134,7 +145,7 @@ class TestTheia(SlapOSInstanceTestCase):
     process.wait()
 
   def test_theia_shell_execute_tasks(self):
-    # shell needs to understand -c "comamnd" arguments for theia tasks feature
+    # shell needs to understand -c "command" arguments for theia tasks feature
     test_file = '{}/test file'.format(self.computer_partition_root_path)
     subprocess.check_call([
         '{}/bin/theia-shell'.format(self.computer_partition_root_path),