Update Release Candidate

component/ZEO/buildout.cfg

@@ -65,7 +65,7 @@ egg     = ZEO
 eggs    = ${:egg}
 
 egg-versions =
-  ZEO = 5.2.3
+  ZEO = 5.4.0
   trollius = 2.2.1
   futures = 3.3.0
 

component/ZODB/buildout.cfg

@@ -58,7 +58,7 @@ egg-versions =
 [ZODB5]
 <= _ZODB
 egg-versions =
-  ZODB = 5.6.0
+  ZODB = 5.8.0
   transaction = 2.4.0
 
 

component/haproxy/buildout.cfg

@@ -13,8 +13,8 @@ parts = haproxy
 [haproxy]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://www.haproxy.org/download/2.6/src/haproxy-2.6.8.tar.gz
-md5sum = 0b506378f1ade633d5b921f5806a5e82
+url = http://www.haproxy.org/download/2.6/src/haproxy-2.6.7.tar.gz
+md5sum = cfa36413f2bc5187ab34ffcdf71914d4
 configure-command = true
 # for Linux kernel 2.6.28 and above, we use "linux-glibc" as the TARGET,
 # otherwise use "generic".

slapos/test/recipe/test_wrapper.py

@@ -135,6 +135,8 @@ class TestPidFile(WrapperTestCase):
       time.sleep(0.1)
       if os.path.exists(self.pidfile):
         break
+    else:
+      self.fail(process.stdout.read())
 
     with open(self.pidfile) as f:
       pid = int(f.read())
@@ -169,7 +171,7 @@ class TestPidFile(WrapperTestCase):
       if process.pid == pid:
         break
     else:
-      self.fail('pidfile not updated', process.stdout.read())
+      self.fail('pidfile not updated: %s' % process.stdout.read())
 
 
 class TestWaitForFiles(WrapperTestCase):

software/slapos-sr-testing/software.cfg

@@ -117,6 +117,11 @@ setup = ${slapos-repository:location}/software/restic-rest-server/test/
 egg = slapos.test.seleniumserver
 setup = ${slapos-repository:location}/software/seleniumserver/test/
 
+[slapos.test.ssh-setup]
+<= setup-develop-egg
+egg = slapos.test.ssh
+setup = ${slapos-repository:location}/software/ssh/test/
+
 [slapos.test.metabase-setup]
 <= setup-develop-egg
 egg = slapos.test.metabase
@@ -332,6 +337,7 @@ eggs +=
   ${slapos.test.restic_rest_server-setup:egg}
   ${slapos.test.seleniumserver-setup:egg}
   ${slapos.test.slapos-master-setup:egg}
+  ${slapos.test.ssh-setup:egg}
   ${slapos.test.theia-setup:egg}
   ${slapos.test.turnserver-setup:egg}
   ${slapos.test.upgrade_erp5-setup:egg}
@@ -422,6 +428,7 @@ tests =
   restic-rest-server ${slapos.test.restic_rest_server-setup:setup}
   seleniumserver ${slapos.test.seleniumserver-setup:setup}
   slapos-master ${slapos.test.slapos-master-setup:setup}
+  ssh ${slapos.test.ssh-setup:setup}
   theia ${slapos.test.theia-setup:setup}
   turnserver ${slapos.test.turnserver-setup:setup}
   upgrade_erp5 ${slapos.test.upgrade_erp5-setup:setup}

software/ssh/README.md

+# OpenSSH server
+
+An OpenSSH server instance in a SlapOS instance. This can be used to access a shell on  this partition or to do ports redirections.
+
+## Known issues
+
+If we create an instance of this SSH Software Release in theia, there will be a theia instance, and the SSH instance is "inside" the theia. In this case, when user login through ssh, they will stay at the outer HOME directory(e.g: /srv/slapgrid/slappart1/), aka the HOME directory of theia. Even if we set a customized HOME directory in the authorized key file(/srv/slapgrid/slappart1/srv/runner/instance/slappart2).
+
+This is because the OpenSSH server reads the home directory from `/etc/passwd` file:
+https://github.com/openssh/openssh-portable/blob/2d1ff2b9431393ad99ef496d5e3b9dd0d4f5ac8c/session.c#L1027-L1029
+
+So setting the `HOME` variable in the key file or exporting `HOME` in the sshd service won't work. If we want ssh instance to spawn a shell in the actual `HOME` directory of the partition in case of nested partition, we would need further efforts, for example something like what was done in slaprunner ( [here](https://lab.nexedi.com/nexedi/slapos/blob/686adec3a2526fc54111866cd64de74fb8a4bd29/software/slaprunner/instance-runner.cfg#L270) and [](https://lab.nexedi.com/nexedi/slapos/blob/686adec3a2526fc54111866cd64de74fb8a4bd29/software/slaprunner/template/bash_profile.in#L6-8) )
+

software/ssh/buildout.hash.cfg

+[instance.cfg]
+filename = instance.cfg
+md5sum = 66c1a6971c8503dcaaa5f40c22cb6aaf

software/ssh/instance-ssh-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object",
+  "properties": {
+    "user-authorized-key": {
+      "title": "User Authorized Key",
+      "description": "SSH public key in order to connect to the SSH server of this instance.",
+      "textarea": true,
+      "type": "string"
+    }
+  }
+}

software/ssh/instance-ssh-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Values returned by instanciation",
+  "properties": {
+    "ssh_command": {
+      "description": "SSH command used to access your instance in ssh when you provided a ssh public key",
+      "type": "string"
+    }
+  },
+  "type": "object"
+}

software/ssh/instance.cfg

+[buildout]
+parts =
+  sshd-service
+  sshd-add-authorized-key
+  sshd-promise
+  publish-connection-parameter
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+extends = ${monitor2-template:output}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = $${buildout:directory}
+etc = $${:home}/etc/
+var = $${:home}/var/
+run = $${:var}/run/
+bin = $${:home}/bin/
+services = $${:etc}/service/
+
+[instance-parameter]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[user-info]
+recipe = slapos.cookbook:userinfo
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[slap-network-information]
+local-ipv4 = $${slap-configuration:ipv4-random}
+global-ipv6 = $${slap-configuration:ipv6-random}
+
+[slap-parameter]
+recipe = slapos.recipe.build
+slapparameter-dict = $${slap-configuration:configuration}
+home = $${buildout:directory}
+init =
+  import json
+  default_parameters = options.get('slapparameter-dict')
+  # Default value if no ssh key is specified
+  options['user-authorized-key'] = default_parameters.get('user-authorized-key', '')
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+ssh-command = ssh $${user-info:pw-name}@$${slap-network-information:global-ipv6} -p $${sshd-port:port}
+ssh-url = ssh://$${user-info:pw-name}@[$${slap-network-information:global-ipv6}]:$${sshd-port:port}
+
+# Deploy openssh-server
+[sshd-port]
+recipe = slapos.cookbook:free_port
+minimum = 22222
+maximum = 22231
+ip = $${instance-parameter:ipv6-random}
+
+[sshd-config]
+recipe = slapos.recipe.template:jinja2
+output = $${directory:etc}/sshd.conf
+path_pid = $${directory:run}/sshd.pid
+inline =
+  PidFile $${:path_pid}
+  Port $${sshd-port:port}
+  ListenAddress $${instance-parameter:ipv6-random}
+  Protocol 2
+  HostKey $${sshd-ssh-host-rsa-key:output}
+  HostKey $${sshd-ssh-host-ecdsa-key:output}
+  PasswordAuthentication no
+  PubkeyAuthentication yes
+  AuthorizedKeysFile $${buildout:directory}/.ssh/authorized_keys
+  Subsystem sftp ${openssh:location}/libexec/sftp-server
+
+[sshd-service]
+recipe = slapos.cookbook:wrapper
+command-line = ${openssh:location}/sbin/sshd -D -e -f $${sshd-config:output}
+hash-existing-files = $${buildout:directory}/buildout.cfg
+wrapper-path = $${directory:services}/sshd
+environment =
+  HOME=$${directory:home}
+
+[sshd-add-authorized-key]
+recipe = slapos.cookbook:dropbear.add_authorized_key
+home = $${buildout:directory}
+key = $${slap-parameter:user-authorized-key}
+
+[sshd-ssh-keygen-base]
+recipe = plone.recipe.command
+output = $${directory:etc}/$${:_buildout_section_name_}
+command = ${openssh-output:keygen} -f $${:output} -N '' $${:extra-args}
+
+[sshd-ssh-host-rsa-key]
+<=sshd-ssh-keygen-base
+extra-args=-t rsa
+[sshd-ssh-host-ecdsa-key]
+<=sshd-ssh-keygen-base
+extra-args=-t ecdsa -b 521
+
+[sshd-promise]
+<= monitor-promise-base
+promise = check_socket_listening
+name = sshd.py
+config-host = $${slap-network-information:global-ipv6}
+config-port = $${sshd-port:port}

software/ssh/software.cfg

+[buildout]
+extends =
+  buildout.hash.cfg
+  ../../stack/slapos.cfg
+  ../../component/openssh/buildout.cfg
+  ../../stack/monitor/buildout.cfg
+
+parts =
+  instance.cfg
+  slapos-cookbook
+
+[instance.cfg]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/instance.cfg
+

software/ssh/software.cfg.json

+{
+  "name": "SSH",
+  "description": "SSH software release which provide the SSH service",
+  "serialisation": "json-in-xml",
+  "software-type": {
+    "default": {
+      "title": "Default",
+      "software-type": "default",
+      "description": "Default",
+      "request": "instance-ssh-input-schema.json",
+      "response": "instance-ssh-output-schema.json",
+      "index": 1
+    }
+  }
+}

software/ssh/test/README.md

+Tests for ssh Software Release

software/ssh/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.ssh'
+with open("README.md") as f:
+  long_description = f.read()
+
+setup(name=name,
+      version=version,
+      description="Test for SSH' ssh",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+          'slapos.core',
+          'slapos.cookbook',
+          'slapos.libnetworkcache',
+          'erp5.util',
+          'supervisor',
+          'psutil',
+          'paramiko',
+          'requests',
+      ],
+      zip_safe=True,
+      test_suite='test',
+      )

software/ssh/test/test.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import contextlib
+import paramiko
+import subprocess
+
+from urllib.parse import urlparse
+import socket
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+from slapos.util import bytes2str
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class TestSSH(SlapOSInstanceTestCase):
+  @classmethod
+  def getInstanceParameterDict(cls):
+    cls.ssh_key_list = [paramiko.ECDSAKey.generate(bits=384) for i in range(2)]
+    return {
+        'user-authorized-key': 'ecdsa-sha2-nistp384 {}\necdsa-sha2-nistp384 {}'.format(
+          *[key.get_base64() for key in cls.ssh_key_list]
+          )
+    }
+
+  def test_connect(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    ssh_url = parameter_dict['ssh-url']
+    parsed = urlparse(ssh_url)
+    self.assertEqual('ssh', parsed.scheme)
+
+    client = paramiko.SSHClient()
+
+    client.set_missing_host_key_policy(paramiko.client.AutoAddPolicy())
+    for ssh_key in self.ssh_key_list:
+      with contextlib.closing(client):
+        client.connect(
+            username=parsed.username,
+            hostname=parsed.hostname,
+            port=parsed.port,
+            pkey=ssh_key,
+        )
+        self.assertTrue(client.get_transport().is_active())
+        # simple commands can also be executed ( this would be like `ssh bash -c 'pwd'` )
+        # exec_command means `ssh user@host command`
+        current_path = bytes2str(client.exec_command("pwd")[1].read(1000)).strip()
+        self.assertIn(current_path, self.computer_partition_root_path)

software/theia/software.cfg

@@ -42,23 +42,6 @@ max_version = 0
 install +=
   golang.org/x/tools/gopls@v0.6.6
 
-[versions]
-aiohttp = 3.8.1:whl
-aiosignal = 1.2.0
-async-timeout = 4.0.2
-cachetools = 5.2.0
-frozenlist = 1.3.0
-idna = 3.3
-multidict = 6.0.2
-packaging = 21.3
-pydantic = 1.9.1
-pygls = 0.12:whl
-typeguard = 2.13.3:whl
-typing-extensions = 4.3.0:whl
-yarl = 1.7.2
-zc.buildout.languageserver = 0.8.3
-
-
 # Downloads and templates
 # -----------------------
 

stack/slapos.cfg

@@ -136,11 +136,14 @@ zc.buildout = 2.7.1+slapos019
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
 zc.recipe.egg = 2.0.3+slapos003
 
+aiohttp = 3.8.1:whl
+aiosignal = 1.2.0
 apache-libcloud = 2.4.0
 argon2-cffi = 20.1.0
 asn1crypto = 1.3.0
 astor = 0.5
 async-generator = 1.10
+async-timeout = 4.0.2
 atomicwrites = 1.4.0
 atomize = 0.2.0
 attrs = 22.1.0
@@ -152,6 +155,7 @@ backports.shutil-get-terminal-size = 1.0.0
 bcrypt = 3.1.4
 bleach = 5.0.1
 CacheControl = 0.12.6:whl
+cachetools = 5.2.0
 certifi = 2022.12.7
 cffi = 1.15.0
 chardet = 3.0.4
@@ -180,6 +184,7 @@ et-xmlfile = 1.0.1
 exceptiongroup = 1.0.0:whl
 feedparser = 6.0.10
 Flask = 1.1.2
+frozenlist = 1.3.0
 funcsigs = 1.0.2
 functools32 = 3.2.3.post2
 gevent = 20.9.0
@@ -187,9 +192,10 @@ geventmp = 0.0.1
 gitdb = 4.0.10
 GitPython = 3.1.30
 greenlet = 0.4.17
-h5py = 2.7.1
 h11 = 0.14.0
+h5py = 2.7.1
 idna = 2.9
+idna = 3.3
 igmp = 1.0.4
 Importing = 1.10
 importlib-metadata = 1.7.0:whl
@@ -211,8 +217,8 @@ jupyter-core = 4.9.2
 jupyterlab = 0.26.3
 jupyterlab-launcher = 0.3.1
 jupyterlab-pygments = 0.1.2
-lock-file = 2.0
 lockfile = 0.12.2:whl
+lock-file = 2.0
 lxml = 4.9.1
 MarkupSafe = 2.0.1
 matplotlib = 2.1.2
@@ -222,6 +228,7 @@ mock = 3.0.5
 more-itertools = 5.0.0
 mpmath = 1.0.0
 msgpack = 0.6.2
+multidict = 6.0.2
 nbclient = 0.5.1
 nbconvert = 6.0.7
 nbformat = 5.0.8
@@ -232,6 +239,7 @@ notebook = 6.1.5
 openpyxl = 2.5.2
 outcome = 1.2.0
 packaging = 16.8
+packaging = 21.3
 pandocfilters = 1.4.3
 paramiko = 2.11.0
 parso = 0.7.1
@@ -252,11 +260,13 @@ prompt-toolkit = 3.0.19
 psutil = 5.8.0
 ptyprocess = 0.5.1
 py = 1.11.0:whl
-py-mld = 1.0.3
 pyasn1 = 0.4.5
 pycparser = 2.20
 pycurl = 7.43.0
+pydantic = 1.9.1
+pygls = 0.12:whl
 Pygments = 2.9.0
+py-mld = 1.0.3
 PyNaCl = 1.3.0
 pyOpenSSL = 19.1.0
 pyparsing = 3.0.9:whl
@@ -293,10 +303,10 @@ slapos.recipe.build = 0.56
 slapos.recipe.cmmi = 0.19
 slapos.recipe.template = 5.0
 slapos.toolbox = 0.129
-statsmodels = 0.11.1
 smmap = 5.0.0
 sniffio = 1.3.0
 sortedcontainers = 2.4.0
+statsmodels = 0.11.1
 stevedore = 1.21.0:whl
 subprocess32 = 3.5.4
 supervisor = 4.1.0
@@ -307,6 +317,8 @@ tornado = 6.1
 traitlets = 5.0.5
 trio = 0.22.0
 trio-websocket = 0.9.2
+typeguard = 2.13.3:whl
+typing-extensions = 4.3.0:whl
 tzlocal = 1.5.1
 unicodecsv = 0.14.1
 uritemplate = 3.0.0
@@ -317,8 +329,10 @@ Werkzeug = 2.0.2
 wheel = 0.38.4:whl
 widgetsnbextension = 2.0.0
 wsproto = 1.2.0
-xml-marshaller = 1.0.2
 xlrd = 1.1.0
+xml-marshaller = 1.0.2
+yarl = 1.7.2
+zc.buildout.languageserver = 0.8.3
 zc.lockfile = 1.4
 ZConfig = 3.6.1
 zdaemon = 4.2.0