Update Release Candidate

component/apache-php/buildout.cfg

@@ -51,8 +51,8 @@ environment =
 
 [php-apcu]
 recipe = slapos.recipe.cmmi
-url = https://github.com/krakjoe/apcu/archive/refs/tags/v5.1.23.tar.gz
-md5sum = 6508bc04b80a1ef12a71100026cfee3f
+url = https://github.com/krakjoe/apcu/archive/refs/tags/v5.1.24.tar.gz
+md5sum = cf4b56ae18e5b031acbc486880dd880b
 configure-command =
   phpize && ./configure
 configure-options =
@@ -67,8 +67,8 @@ shared = false
 
 [apache-php]
 recipe = slapos.recipe.cmmi
-url = https://www.php.net/distributions/php-8.2.20.tar.xz
-md5sum = 9478294cb87946891f70d5aa5f959bc3
+url = https://www.php.net/distributions/php-8.2.24.tar.xz
+md5sum = 3263dbd4846871dd6fabe042f141eb19
 configure-options =
   --disable-static
   --disable-zend-test

component/mariadb/buildout.cfg

@@ -38,6 +38,8 @@ mariadb-client-test = mariadb-client-test
 pre-configure =
   d() { x=$1; shift; [ "`grep -c "$@"`" = $x ]; sed -i "/$1/d" "$2"; }
   d 1 '\bSET(PLUGIN_AUTH_PAM YES CACHE BOOL "")' cmake/build_configurations/mysql_release.cmake
+  ${:pre-configure-extra}
+pre-configure-extra =
   d 5 '^ *\b\(ADD_SUBDIRECTORY(\(mysql-test\|tests\)\|my_safe_process\|${:mariadb-client-test}\)\b' CMakeLists.txt
   d 1 '\bINSTALL_MYSQL_TEST\b' cmake/plugin.cmake
 configure-command = ${cmake:location}/bin/cmake
@@ -109,20 +111,15 @@ patches =
 patches +=
   ${:_profile_base_location_}/mariadb-10.4.32-openssl-3.0.patch#ec3e2b57eae5421e052ab89224b29b7e
 
-[mariadb-10.5]
-<= mariadb-common
-version = 10.5.25
-md5sum = 85d74d62d3aee1c3361cdfce2135141b
-
-[mariadb-10.6]
+[mariadb-10.11]
 <= mariadb-common
-version = 10.6.18
-md5sum = ff0084890e4b8164144edb662f318fcb
+version = 10.11.9
+md5sum = 5f2f97f72de7b97e8df34a55a9ca2140
 
-[mariadb-10.11]
+[mariadb-11.4]
 <= mariadb-common
-version = 10.11.8
-md5sum = ba7abfae7947893c5a5343180808b0cb
+version = 11.4.3
+md5sum = c19f97dc1ea3165fb282a8384155dc0a
 
 [mariadb]
 <= mariadb-10.4

component/mariadb/test.cfg

+[buildout]
+extends =
+  ../../stack/slapos.cfg
+  buildout.cfg
+
+parts =
+  mariadb
+
+[mariadb]
+shared = false
+pre-configure-extra =

component/mariadb/test/README.md

+Tests for MariaDB

component/mariadb/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2024 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation, either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.mariadb'
+long_description = open("README.md").read()
+
+setup(name=name,
+      version=version,
+      description="Test for SlapOS' MariaDB",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'erp5.util',
+        ],
+      zip_safe=True,
+      test_suite='test',
+    )

component/mariadb/test/test.py

+##############################################################################
+#
+# Copyright (c) 2024 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation, either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+##############################################################################
+
+import hashlib
+import os
+import shutil
+import subprocess
+import tempfile
+
+from slapos.testing.testcase import installSoftwareUrlList, makeModuleSetUpAndTestCaseClass
+
+software_release_url = os.path.abspath(
+  os.path.join(os.path.dirname(__file__), '..', 'test.cfg'))
+
+setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass(
+  os.path.abspath(software_release_url))
+
+class TestMariaDB(InstanceTestCase):
+  request_instance = False
+
+  def test(self):
+    mariadb_path = os.path.join(
+      self.slap.software_directory,
+      hashlib.md5(self.getSoftwareURL().encode()).hexdigest(),
+      'parts',
+      'mariadb',
+    )
+    with tempfile.TemporaryDirectory() as tmp_dir:
+      shutil.copytree(
+        mariadb_path,
+        tmp_dir,
+        symlinks=True,
+        dirs_exist_ok=True,
+      )
+      test_dir = os.path.join(tmp_dir, 'mariadb-test')
+      if not os.path.isdir(test_dir): # BBB MariaDB < 11
+        test_dir = os.path.join(tmp_dir, 'mysql-test')
+      os.chdir(test_dir)
+      skip_test_list = [
+        'spider',            # fail if 'failed to retrieve the MAC address'.
+        'mysqld--help-aria', # fail if 'failed to retrieve the MAC address'.
+        'mysqldump-header',  # fail with our -DDEFAULT_CHARSET=utf8 build.
+        'ssl_autoverify',    # fail without Net/SSLeay.pm.
+      ]
+      result = subprocess.run(
+        ['./mtr', '--parallel=10', '--force', '--max-test-fail=40', '--mem',
+         '--skip-test=(%s)' % ('|'.join(skip_test_list))],
+        capture_output=True,
+      )
+      self.assertEqual(result.returncode, 0, result.stdout)

component/openssl/buildout.cfg

@@ -51,8 +51,8 @@ md5sum = 08f458c00fff496a52ef931c481045cd
 
 [openssl-quictls]
 <= openssl-3.0
-url = https://github.com/quictls/openssl/archive/refs/tags/openssl-3.0.14-quic1.tar.gz
-md5sum = fa0ae39667afa8be6dbeadb3cc838e89
+url = https://github.com/quictls/openssl/archive/refs/tags/openssl-3.0.15-quic1.tar.gz
+md5sum = 00fd544d5ae53bf841dc2f155c2f0524
 
 [openssl-1.1]
 <= openssl-common

component/python-nemu3/buildout.cfg

+[buildout]
+extends =
+  ../../component/git/buildout.cfg
+
+[python-nemu3-repository]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+forbid-download-cache = true
+revision = 0301f87d0f7d3e96809cf3c9d38d21248d17515d
+repository = https://lab.nexedi.com/nexedi/nemu3.git
+
+[python-nemu3]
+recipe = zc.recipe.egg:develop
+egg = nemu3
+setup = ${python-nemu3-repository:location}
+
+

component/re6stnet/buildout.cfg

@@ -12,11 +12,8 @@ extends =
 parts =
   re6stnet
 
-[python]
-part = python2.7
-
-[openssl]
-<= openssl-1.1
+[python3]
+<= python3.11
 
 [gcc]
 min_version = 0
@@ -28,6 +25,7 @@ eggs =
   ${python-cffi:egg}
   ${python-cryptography:egg}
   ${python-PyYAML:egg}
+  ${python-pim-dm:egg}
   pyOpenSSL
   re6stnet[geoip]
 
@@ -36,54 +34,4 @@ eggs =
 initialization =
   import os
   os.environ['GEOIP2_MMDB'] = "${geolite2-country:location}/GeoLite2-Country.mmdb"
-  os.environ['PATH'] = "${openvpn:location}/sbin:${babeld:location}/bin:${openssl:location}/bin:${python:location}/bin:${pim-dm-buildout:location}/bin:" + os.environ.get('PATH', '/usr/sbin:/usr/bin:/sbin:/bin')
-
-[pim-dm-buildout]
-recipe = slapos.recipe.build
-base = ${:_profile_base_location_}
-python = ${python3:executable}
-install =
-  import os
-  from zc.buildout.buildout import _isurl, urljoin
-  from slapos.rebootstrap.bootstrap import setup_script
-  base = options['base']
-  if _isurl(base):
-    extends = lambda path: urljoin(base + '/', path)
-  else:
-    extends = lambda path: os.path.realpath(os.path.join(base, path))
-
-  buildout_extra = []
-  buildout_section = self.buildout['buildout']
-  for option_name in ('download-cache', 'newest', 'shared-part-list'):
-    option_value = buildout_section.get(option_name)
-    if option_value:
-      buildout_extra.append('%s = %s' % (option_name, option_value))
-  os.mkdir(location)
-  with open(os.path.join(location, 'buildout.cfg'), 'w') as buildout_conf:
-    buildout_conf.write(
-      options['template'] % {
-        'extends': '\n  '.join(map(extends, (
-          '../../stack/slapos.cfg',
-          '../python-pim-dm/buildout.cfg',
-          ))),
-        'buildout': '\n'.join(buildout_extra),
-      }
-    )
-  buildout = os.path.join(location, 'bin', 'buildout')
-  os.mkdir(os.path.dirname(buildout))
-  setup_script(buildout, options['python'])
-  call((buildout, '-U'), cwd=location)
-template =
-  [buildout]
-  extends = %(extends)s
-  extends-cache = ${buildout:extends-cache}
-  extensions -= slapos.rebootstrap
-  %(buildout)s
-  [libpcap]
-  recipe =
-  location = ${libpcap:location}
-  environment =
-  [libyaml]
-  recipe =
-  location = ${libyaml:location}
-  pre-configure =
+  os.environ['PATH'] = "${openvpn:location}/sbin:${babeld:location}/bin:${openssl:location}/bin:${python:location}/bin:" + os.environ.get('PATH', '/usr/sbin:/usr/bin:/sbin:/bin')

component/wendelin.core/buildout.cfg

@@ -50,7 +50,7 @@ CGO_LDFLAGS += -Wl,-rpath=${zlib:location}/lib
 recipe  = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/wendelin.core.git
 branch   = master
-revision = wendelin.core-2.0.alpha3-22-g07087ec8
+revision = wendelin.core-2.0.alpha4-0-gdb6fea3
 # dir is pretty name as top-level recipe
 location = ${buildout:parts-directory}/wendelin.core
 git-executable = ${git:location}/bin/git

software/erp5/test/plot_benchmark_result.py

+import argparse
+import glob
+import json
+import matplotlib.pyplot as plt
+import os
+
+parser = argparse.ArgumentParser(description='Generate a graph from benchmark results.')
+parser.add_argument('dir', nargs='+',
+                    help='a directory where mesurestest.*.jsonl exists.')
+parser.add_argument('-o', '--output', metavar='filename', required=True,
+                    help='an output filename (.png, .svg etc.)')
+args = parser.parse_args()
+
+duration_list_dict = {}
+conflict_list_dict = {}
+deadlock_list_dict = {}
+
+def get_deadlock_total_count(data):
+  return data.get('innodb_metrics', {}).get('lock_deadlocks') or data.get('deadlock_total_count') or 0
+
+for dir in args.dir:
+  dir = dir.rstrip('/')
+  path, = glob.glob(os.path.join(dir, 'measurestest.*.jsonl'))
+  duration_list = []
+  conflict_list = []
+  deadlock_list = []
+  last_data = {}
+  for l in open(path).readlines():
+    data = json.loads(l)
+    if 'iteration_' in l:
+      duration_list.append(data['step_duration_seconds'])
+      conflict_list.append(data['zeo_stats']['conflicts'] - last_data['zeo_stats']['conflicts'])
+      deadlock_total_count = get_deadlock_total_count(data)
+      if deadlock_total_count:
+        deadlock_list.append(deadlock_total_count - get_deadlock_total_count(last_data))
+    last_data = data
+  duration_list_dict[dir] = duration_list
+  conflict_list_dict[dir] = conflict_list
+  deadlock_list_dict[dir] = deadlock_list
+plt.figure(figsize=(7, 10))
+ax = plt.subplot(3, 1, 1)
+plt.title('Duration')
+for k, duration_list in duration_list_dict.items():
+  plt.plot(range(len(duration_list)), duration_list, '-', label=k)
+ax.set_ylim(ymin=0)
+plt.legend(loc='upper right', framealpha=0.7)
+ax = plt.subplot(3, 1, 2)
+plt.title('ZODB Conflicts')
+for k, conflict_list in conflict_list_dict.items():
+  plt.plot(range(len(conflict_list)), conflict_list, '-', label=k)
+ax.set_ylim(ymin=0)
+plt.legend(loc='upper right', framealpha=0.7)
+ax = plt.subplot(3, 1, 3)
+plt.title('RDB Deadlocks')
+for k, deadlock_list in deadlock_list_dict.items():
+  plt.plot(range(len(deadlock_list)), deadlock_list, '-', label=k)
+ax.set_ylim(ymin=0)
+plt.legend(loc='upper right', framealpha=0.7)
+plt.subplots_adjust(hspace=0.5)
+plt.savefig(args.output)

software/erp5/test/test/benchmarks.py

@@ -28,6 +28,7 @@
 import contextlib
 import datetime
 import json
+import os
 import pathlib
 import socket
 import struct
@@ -37,6 +38,7 @@ import urllib.parse
 
 import psutil
 import requests
+import MySQLdb
 
 from . import ERP5InstanceTestCase, default, matrix, setUpModule
 from .test_erp5 import ZopeSkinsMixin
@@ -141,6 +143,24 @@ class TestOrderBuildPackingListSimulation(
           text=True),
     })
 
+  def getDatabaseConnection(self):
+    connection_parameter_dict = json.loads(
+      self.computer_partition.getConnectionParameterDict()['_'])
+    db_url = urllib.parse.urlparse(connection_parameter_dict['mariadb-database-list'][0])
+    self.assertEqual('mysql', db_url.scheme)
+
+    self.assertTrue(db_url.path.startswith('/'))
+    database_name = db_url.path[1:]
+    return MySQLdb.connect(
+      user=db_url.username,
+      passwd=db_url.password,
+      host=db_url.hostname,
+      port=db_url.port,
+      db=database_name,
+      use_unicode=True,
+      charset='utf8mb4',
+    )
+
   def write_measurement(
       self, measurement: dict[str, typing.Union[str, float]]) -> None:
     json.dump(
@@ -184,6 +204,15 @@ class TestOrderBuildPackingListSimulation(
       zeo_root_stats = zeo_stats.pop('root')
       assert not zeo_stats
 
+    # InnoDB/MariaDB metrics
+    with contextlib.closing(self.getDatabaseConnection()) as cnx:
+      cursor = cnx.cursor(MySQLdb.cursors.DictCursor)
+      cursor.execute('SELECT * FROM information_schema.INNODB_METRICS')
+      innodb_metrics = {r['NAME']: r['COUNT'] for r in cursor.fetchall()
+                        if r.get('ENABLED') or r.get('STATUS') == 'enabled'}
+      cursor.execute('SHOW GLOBAL STATUS')
+      mariadb_metrics = {r['Variable_name']: r['Value'] for r in cursor.fetchall()}
+
     self.logger.info(
       "Measurements for %s (after %s): "
       "elapsed=%s zope_total_rss=%s / %s root_fs_size=%s",
@@ -203,6 +232,8 @@ class TestOrderBuildPackingListSimulation(
         'zope_count': zope_count,
         'root_fs_size': root_fs_size,
         'zeo_stats': zeo_root_stats,
+        'innodb_metrics': innodb_metrics,
+        'mariadb_metrics': mariadb_metrics,
         'now': str(now),
       })
 
@@ -282,9 +313,21 @@ class TestOrderBuildPackingListSimulation(
       mariadb_slowquery_log = pathlib.Path(
         self.getComputerPartitionPath(
           'mariadb')) / 'var' / 'log' / 'mariadb_slowquery.log'
+      mariadb_data = pathlib.Path(
+        self.getComputerPartitionPath(
+          'mariadb')) / 'srv' / 'mariadb'
+      mariadb_data_size = sum(sum(os.path.getsize(os.path.join(root, f)) for f in files) \
+        for root, dirs, files in os.walk(mariadb_data))
+      mariadb_binlog_data = pathlib.Path(
+        self.getComputerPartitionPath(
+          'mariadb')) / 'srv' / 'backup' / 'mariadb-incremental'
+      mariadb_binlog_data_size = sum(sum(os.path.getsize(os.path.join(root, f)) for f in files) \
+        for root, dirs, files in os.walk(mariadb_binlog_data))
       self.write_measurement(
         {
           'pt-query-digest':
           subprocess.check_output(
-            (pt_query_digest, mariadb_slowquery_log), text=True)
+            (pt_query_digest, mariadb_slowquery_log), text=True),
+          'data-size': mariadb_data_size,
+          'binlog-data-size': mariadb_binlog_data_size,
         })

software/kvm/instance-kvm-cluster-input-schema.json

@@ -458,15 +458,15 @@
               "oneOf": [
                 {
                   "const": [
-                    "https://shacache.nxdcdn.com/02257c3ec27e45d9f022c181a69b59da67e5c72871cdb4f9a69db323a1fad58093f2e69702d29aa98f5f65e920e0b970d816475a5a936e1f3bf33832257b7e92#b710c178eb434d79ce40ce703d30a5f0"
+                    "https://shacache.nxdcdn.com/33c08e56c83d13007e4a5511b9bf2c4926c4aa12fd5dd56d493c0653aecbab380988c5bf1671dbaea75c582827797d98c4a611f7fb2b131fbde2c677d5258ec9#326b7737c4262e8eb09cd26773f3356a"
                   ],
-                  "title": "Debian Bullseye 11.1 netinst x86_64"
+                  "title": "Debian Bookworm 12 netinst x86_64"
                 },
                 {
                   "const": [
-                    "https://shacache.nxdcdn.com/bc469019b9057073d36ff8f5402c95ff0a0363657358336dc9a05fc6af66276229aa727ec46cf17b84d308f44b825de7f24ea1a256062a14e0f605cd70bae02f#2dcf188877075d7b58ca46e27ca31bb9"
+                    "https://shacache.nxdcdn.com/02257c3ec27e45d9f022c181a69b59da67e5c72871cdb4f9a69db323a1fad58093f2e69702d29aa98f5f65e920e0b970d816475a5a936e1f3bf33832257b7e92#b710c178eb434d79ce40ce703d30a5f0"
                   ],
-                  "title": "Debian Buster 10.12 netinst x86_64"
+                  "title": "Debian Bookworm 11 netinst x86_64"
                 },
                 {
                   "const": [
@@ -474,6 +474,12 @@
                   ],
                   "title": "Centos 8.2004 Minimal x86_64"
                 },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/3d518612aabbdb77fd6b49cb55b824fed11e40540e4af52f5f26174257715c93740f83079ea618b4d933081f0b1bc69d32b7885b7c75bc90da5ad3fe1814cfd4#c53b2d7c3269c3b91a2d941ceaa8ab9b"
+                  ],
+                  "title": "Ubuntu Jammy 24.04.1 Live Server x86_64"
+                },
                 {
                   "const": [
                     "https://shacache.nxdcdn.com/8017c532ed74586b718662d8b11cf8c34fa638b0affd0413ed38623989b8f98ffd0bcb475246e279ea2f3c194a3e33c55e0f376a9727de13e4bfd87e75e47b5d#e8d2a77c51b599c10651608a5d8c286f"

software/kvm/instance-kvm-input-schema.json

@@ -324,9 +324,15 @@
           "oneOf": [
             {
               "const": [
-                "https://shacache.nxdcdn.com/bc469019b9057073d36ff8f5402c95ff0a0363657358336dc9a05fc6af66276229aa727ec46cf17b84d308f44b825de7f24ea1a256062a14e0f605cd70bae02f#2dcf188877075d7b58ca46e27ca31bb9"
+                "https://shacache.nxdcdn.com/33c08e56c83d13007e4a5511b9bf2c4926c4aa12fd5dd56d493c0653aecbab380988c5bf1671dbaea75c582827797d98c4a611f7fb2b131fbde2c677d5258ec9#326b7737c4262e8eb09cd26773f3356a"
               ],
-              "title": "Debian Buster 10.12 netinst x86_64"
+              "title": "Debian Bookworm 12 netinst x86_64"
+            },
+            {
+              "const": [
+                "https://shacache.nxdcdn.com/02257c3ec27e45d9f022c181a69b59da67e5c72871cdb4f9a69db323a1fad58093f2e69702d29aa98f5f65e920e0b970d816475a5a936e1f3bf33832257b7e92#b710c178eb434d79ce40ce703d30a5f0"
+              ],
+              "title": "Debian Bookworm 11 netinst x86_64"
             },
             {
               "const": [
@@ -334,6 +340,12 @@
               ],
               "title": "Centos 8.2004 Minimal x86_64"
             },
+            {
+              "const": [
+                "https://shacache.nxdcdn.com/3d518612aabbdb77fd6b49cb55b824fed11e40540e4af52f5f26174257715c93740f83079ea618b4d933081f0b1bc69d32b7885b7c75bc90da5ad3fe1814cfd4#c53b2d7c3269c3b91a2d941ceaa8ab9b"
+              ],
+              "title": "Ubuntu Jammy 24.04.1 Live Server x86_64"
+            },
             {
               "const": [
                 "https://shacache.nxdcdn.com/8017c532ed74586b718662d8b11cf8c34fa638b0affd0413ed38623989b8f98ffd0bcb475246e279ea2f3c194a3e33c55e0f376a9727de13e4bfd87e75e47b5d#e8d2a77c51b599c10651608a5d8c286f"

software/re6stnet/software.cfg

@@ -13,13 +13,13 @@ extends =
 # Monitoring stack (keep on bottom)
   ../../stack/monitor/buildout.cfg
 
-# Python2 versions for buildout (keep bottom-most)
-  ../../stack/slapos-py2.cfg
-
 parts +=
   slapos-cookbook
   template
 
+[python3]
+<= python3.11
+
 [re6stnet-repository]
 recipe = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/re6stnet.git
@@ -103,14 +103,3 @@ context =
 
 [template-re6st-registry-conf]
 < = download-base
-
-[versions]
-miniupnpc = 1.9
-
-
-# Test Suite: SlapOS.Eggs.UnitTest-Master.Python2 ran at 2024/09/30 07:50:36.080156 UTC
-#             0 failures, 0 errors, 1367 total, status: PASS
-
-[re6stnet]
-revision = f2fd7247b2b3fa1162072e1928a3ead890ad3369
-

software/slapos-sr-testing/software.cfg

@@ -87,6 +87,11 @@ setup = ${slapos-repository:location}/software/jstestnode/test/
 egg = slapos.test.kvm
 setup = ${slapos-repository:location}/software/kvm/test/
 
+[slapos.test.mariadb-setup]
+<= setup-develop-egg
+egg = slapos.test.mariadb
+setup = ${slapos-repository:location}/component/mariadb/test/
+
 [slapos.test.monitor-setup]
 <= setup-develop-egg
 egg = slapos.test.monitor
@@ -387,6 +392,7 @@ eggs +=
   ${slapos.test.jstestnode-setup:egg}
   ${slapos.test.jupyter-setup:egg}
   ${slapos.test.kvm-setup:egg}
+  ${slapos.test.mariadb-setup:egg}
   ${slapos.test.matomo-setup:egg}
   ${slapos.test.metabase-setup:egg}
   ${slapos.test.monitor-setup:egg}
@@ -480,6 +486,7 @@ tests =
   jstestnode ${slapos.test.jstestnode-setup:setup}
   jupyter ${slapos.test.jupyter-setup:setup}
   kvm ${slapos.test.kvm-setup:setup}
+  mariadb ${slapos.test.mariadb-setup:setup}
   matomo ${slapos.test.matomo-setup:setup}
   metabase ${slapos.test.metabase-setup:setup}
   monitor ${slapos.test.monitor-setup:setup}

software/slapos-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = 5f5378550470b551d280dd432878a0ba
+md5sum = fd49b60268a2f58f758c1f4cf81114c2

software/slapos-testing/instance.cfg

@@ -94,8 +94,9 @@ repository = ${re6stnet-repository:location}
 [re6stnet-test-runner]
 recipe = slapos.recipe.template:jinja2
 template = inline:#!/bin/sh
-  #change #!/usr/bin/python2 -> #!/real_python_path/bin/pythonVersion
-  sed '1s?/usr/bin/python2?${python:location}/bin/python${python:version}?' -i ${re6stnet-repository:location}/re6st/ovpn-*
+  # put python3 in PATH for ovpn-server and ovpn-client scripts
+  export PATH=${python:location}/bin:$$PATH
+
   # update files in /sys/class/net
   mount -t sysfs sysfs /sys
 
@@ -120,6 +121,7 @@ output = $${:workdir}/.nxdtest
 workdir = $${create-directory:nxdtest-working-dir}
 inline =
   import six
+  import sys
   TestCase(
       "kedifa",
       ['python', '-m', 'unittest', 'discover', '-v'],
@@ -215,7 +217,7 @@ inline =
       cwd="""$${rubygemsrecipe:location}""",
       summaryf=UnitTest.summary,
   )
-  if six.PY2:
+  if sys.version_info >= (3,11):
     TestCase(
         "re6stnet",
         ['unshare', '-Umnr', '$${re6stnet-test-runner:rendered}'],

software/slapos-testing/software-py2.cfg

@@ -5,10 +5,6 @@ extends =
 # Python2 versions for buildout (keep last)
     ../../stack/slapos-py2.cfg
 
-[extra-eggs]
-eggs +=
-  ${re6stnet-setup:egg}
-
 [slapos.toolbox-setup]
 recipe = zc.recipe.egg
 eggs = ${:egg}

software/slapos-testing/software-py3next.cfg

@@ -4,3 +4,8 @@ extends =
 
 [python3]
 <= python3.11
+
+[extra-eggs]
+eggs +=
+  ${python-nemu3:egg}
+  ${re6stnet-setup:egg}

software/slapos-testing/software.cfg

@@ -9,6 +9,7 @@ extends =
   ../../component/zlib/buildout.cfg
   ../../component/phantomjs/buildout.cfg
   ../../component/pycurl/buildout.cfg
+  ../../component/python-nemu3/buildout.cfg
   ../../component/coreutils/buildout.cfg
   ../../component/socat/buildout.cfg
   ../../component/lmsensors/buildout.cfg
@@ -243,6 +244,7 @@ repository = https://lab.nexedi.com/nexedi/rubygemsrecipe.git
 [re6stnet-repository]
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/re6stnet.git
+branch = re6st-py3
 
 [template]
 recipe = slapos.recipe.template
@@ -279,8 +281,8 @@ testfixtures = 6.11.0
 pyflakes = 2.0.0
 zope.testing = 4.6.2
 iniparse = 0.5
-miniupnpc = 1.9
 nemu = 0.3.1
+nemu3 = 0.4
 multiping = 1.1.2
 python-passfd = 0.2
 python-unshare = 0.2

stack/erp5-zope2/buildout.hash.cfg

@@ -70,7 +70,7 @@ md5sum = b95084ae9eed95a68eada45e28ef0c04
 
 [template]
 filename = instance.cfg.in
-md5sum = 1a81908e9716dedba4f397f37264b015
+md5sum = 6d9a926e07b674ffdaecd381d763c068
 
 [template-erp5]
 filename = instance-erp5.cfg.in
@@ -102,4 +102,4 @@ md5sum = 5cf0316fdd17a940031e4083bbededd8
 
 [instance-wcfs.cfg.in]
 filename = instance-wcfs.cfg.in
-md5sum = a495f84194a46c585b89947f8bc7d754
+md5sum = 29bff387a18b0c54a3868453114e10f0

stack/erp5-zope2/instance-wcfs.cfg.in

 {# instance that runs WCFS service associated with ZODB storage  #}
 {% from "instance_zodb_base" import zodb_dict  with context %}
-
-{# q(text) returns urllib.quote_plus(text) #}
-{% macro q(text) %}{{ urllib_parse.quote_plus(text) }}{% endmacro %}
+{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
+{% set part_list = [] -%}
+{% set ssl_dict = {"ca": 'ca.crt', "cert": 'neo.crt', "key": 'neo.key'} -%}
 
 {# build zurl to connect to configured ZODB #}
 {% if len(zodb_dict) != 1 -%}
@@ -10,35 +10,37 @@
 {% endif -%}
 
 {% set db_name, zodb = zodb_dict.popitem() -%}
-{% set z = zodb['storage-dict'] -%}
+{% set z = dict(zodb['storage-dict']) -%}
 {% if zodb['type'] == 'zeo' -%}
 {%   set zurl = ('zeo://%s?storage=%s' % (z['server'], z['storage'])) -%}
 {% elif zodb['type'] == 'neo' -%}
-{#   neo(s)://[credentials@]master1,master2,...,masterN/name?options #}
-{#   (see https://lab.nexedi.com/kirr/neo/blob/3e13fa06/go/neo/client.go#L417) #}
-{#   If 'ca' in storage-dict, ssl is true. #}
+{#   If 'ca' in storage-dict, SSL is true. #}
 {#   (see https://lab.nexedi.com/nexedi/slapos/blob/397726e1/stack/erp5/instance-zodb-base.cfg.in#L17-21) #}
 {%   if "ca" in z -%}
-{#     ca=ca.crt;cert=my.crt;key=my.key  (see https://lab.nexedi.com/kirr/neo/blob/3e13fa06/go/neo/client.go#L428) #}
-{%     set zurl = 'neos://ca=%s;cert=%s;key=%s@' % (q(z.pop("ca")), q(z.pop("cert")), q(z.pop("key"))) -%}
+{%     set zurlscheme = 'neos' -%}
 {%   else -%}
-{%     set zurl = 'neo://' -%}
+{%     set zurlscheme = 'neo' -%}
+{%   endif -%}
+{%   set zurl = ('%s://%s@%s' % (zurlscheme, z.pop('name'), z.pop('master_nodes')|replace(" ", ","))) -%}
+{%   set argv = [] -%}
+{%   set i = 0 -%}
+{%   for k,v in z|dictsort -%}
+{#     Parameters which neither set client nor server: -#}
+{#     they can be ignored as they are merely resulting from a convenience hack. -#}
+{#     (see https://lab.nexedi.com/nexedi/slapos/commit/706801f2) -#}
+{%     if k not in ["_ca", "_key", "_cert"] %}
+{%       set v = (ssl_dict.get(k) and "${directory:etc}/" + ssl_dict.get(k)) or v -%}
+{%       do argv.append('%s=%s' % (k,v)) -%}
+{%     endif %}
+{%   endfor -%}
+{%   if len(argv) > 0 -%}
+{%     set zurl = zurl + '?' + '&'.join(argv) -%}
 {%   endif -%}
-{%   set zurl = ('%s%s/%s' % (zurl, z.pop('master_nodes')|replace(" ", ","), z.pop('name'))) -%}
-{%   set zurl = zurl + '?' + (z | dictsort | urlencode) -%}
 {% else -%}
 {%   do assert(False, ("unsupported ZODB type", zodb)) -%}
 {% endif -%}
 
 
-[buildout]
-extends = {{ template_monitor }}
-parts +=
-  wcfs
-  wcfs-promise
-  publish
-
-
 [directory]
 recipe = slapos.cookbook:mkdirectory
 etc = ${buildout:directory}/etc
@@ -64,3 +66,31 @@ config-command = {{ bin_directory }}/wcfs status {{ zurl }}
 [publish]
 recipe = slapos.cookbook:publish.serialised
 serving-zurl = {{ zurl }}
+
+{# Hack to deploy SSL certs via instance parameters -#}
+{# XXX Duplication wrt https://lab.nexedi.com/nexedi/slapos/-/blob/5ae99219/stack/erp5/instance-zope.cfg.in -#}
+{% set storage_dict = zodb.setdefault('storage-dict', {}) -%}
+{% if zodb['type'] == 'neo' and storage_dict.get('ssl', 1) -%}
+{%   for k,v in ssl_dict|dictsort -%}
+{%     set k = "_{}".format(k) -%}
+{%     if k in storage_dict -%}
+[{{ section('neo-ssl-' + k[1:]) }}]
+recipe = slapos.recipe.template:jinja2
+output = ${directory:etc}/{{v}}
+inline = {{'{{'}}pem}}
+context = key pem :pem
+pem = {{dumps(storage_dict.pop(k))}}
+
+{%     endif -%}
+{%   endfor -%}
+{% endif -%}
+{# endhack -#}
+
+
+[buildout]
+extends = {{ template_monitor }}
+parts +=
+  wcfs
+  wcfs-promise
+  publish
+  {{ '\n  '.join(part_list) }}

stack/erp5-zope2/instance.cfg.in

@@ -219,7 +219,6 @@ url = {{ instance_wcfs_cfg_in }}
 filename = instance_wcfs.cfg
 extra-context =
     section parameter_dict dynamic-template-zope-parameters
-    import urllib_parse six.moves.urllib.parse
 import-list =
     file instance_zodb_base context:template-zodb-base
 

stack/erp5/buildout.hash.cfg

@@ -70,7 +70,7 @@ md5sum = b95084ae9eed95a68eada45e28ef0c04
 
 [template]
 filename = instance.cfg.in
-md5sum = 39e391372cc616ea5d1d6559621e6747
+md5sum = 8e452bd32fc0d4d858b275a2b3ee790b
 
 [template-erp5]
 filename = instance-erp5.cfg.in
@@ -106,4 +106,4 @@ md5sum = 5cf0316fdd17a940031e4083bbededd8
 
 [instance-wcfs.cfg.in]
 filename = instance-wcfs.cfg.in
-md5sum = a495f84194a46c585b89947f8bc7d754
+md5sum = 29bff387a18b0c54a3868453114e10f0

stack/erp5/instance-wcfs.cfg.in

 {# instance that runs WCFS service associated with ZODB storage  #}
 {% from "instance_zodb_base" import zodb_dict  with context %}
-
-{# q(text) returns urllib.quote_plus(text) #}
-{% macro q(text) %}{{ urllib_parse.quote_plus(text) }}{% endmacro %}
+{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
+{% set part_list = [] -%}
+{% set ssl_dict = {"ca": 'ca.crt', "cert": 'neo.crt', "key": 'neo.key'} -%}
 
 {# build zurl to connect to configured ZODB #}
 {% if len(zodb_dict) != 1 -%}
@@ -10,35 +10,37 @@
 {% endif -%}
 
 {% set db_name, zodb = zodb_dict.popitem() -%}
-{% set z = zodb['storage-dict'] -%}
+{% set z = dict(zodb['storage-dict']) -%}
 {% if zodb['type'] == 'zeo' -%}
 {%   set zurl = ('zeo://%s?storage=%s' % (z['server'], z['storage'])) -%}
 {% elif zodb['type'] == 'neo' -%}
-{#   neo(s)://[credentials@]master1,master2,...,masterN/name?options #}
-{#   (see https://lab.nexedi.com/kirr/neo/blob/3e13fa06/go/neo/client.go#L417) #}
-{#   If 'ca' in storage-dict, ssl is true. #}
+{#   If 'ca' in storage-dict, SSL is true. #}
 {#   (see https://lab.nexedi.com/nexedi/slapos/blob/397726e1/stack/erp5/instance-zodb-base.cfg.in#L17-21) #}
 {%   if "ca" in z -%}
-{#     ca=ca.crt;cert=my.crt;key=my.key  (see https://lab.nexedi.com/kirr/neo/blob/3e13fa06/go/neo/client.go#L428) #}
-{%     set zurl = 'neos://ca=%s;cert=%s;key=%s@' % (q(z.pop("ca")), q(z.pop("cert")), q(z.pop("key"))) -%}
+{%     set zurlscheme = 'neos' -%}
 {%   else -%}
-{%     set zurl = 'neo://' -%}
+{%     set zurlscheme = 'neo' -%}
+{%   endif -%}
+{%   set zurl = ('%s://%s@%s' % (zurlscheme, z.pop('name'), z.pop('master_nodes')|replace(" ", ","))) -%}
+{%   set argv = [] -%}
+{%   set i = 0 -%}
+{%   for k,v in z|dictsort -%}
+{#     Parameters which neither set client nor server: -#}
+{#     they can be ignored as they are merely resulting from a convenience hack. -#}
+{#     (see https://lab.nexedi.com/nexedi/slapos/commit/706801f2) -#}
+{%     if k not in ["_ca", "_key", "_cert"] %}
+{%       set v = (ssl_dict.get(k) and "${directory:etc}/" + ssl_dict.get(k)) or v -%}
+{%       do argv.append('%s=%s' % (k,v)) -%}
+{%     endif %}
+{%   endfor -%}
+{%   if len(argv) > 0 -%}
+{%     set zurl = zurl + '?' + '&'.join(argv) -%}
 {%   endif -%}
-{%   set zurl = ('%s%s/%s' % (zurl, z.pop('master_nodes')|replace(" ", ","), z.pop('name'))) -%}
-{%   set zurl = zurl + '?' + (z | dictsort | urlencode) -%}
 {% else -%}
 {%   do assert(False, ("unsupported ZODB type", zodb)) -%}
 {% endif -%}
 
 
-[buildout]
-extends = {{ template_monitor }}
-parts +=
-  wcfs
-  wcfs-promise
-  publish
-
-
 [directory]
 recipe = slapos.cookbook:mkdirectory
 etc = ${buildout:directory}/etc
@@ -64,3 +66,31 @@ config-command = {{ bin_directory }}/wcfs status {{ zurl }}
 [publish]
 recipe = slapos.cookbook:publish.serialised
 serving-zurl = {{ zurl }}
+
+{# Hack to deploy SSL certs via instance parameters -#}
+{# XXX Duplication wrt https://lab.nexedi.com/nexedi/slapos/-/blob/5ae99219/stack/erp5/instance-zope.cfg.in -#}
+{% set storage_dict = zodb.setdefault('storage-dict', {}) -%}
+{% if zodb['type'] == 'neo' and storage_dict.get('ssl', 1) -%}
+{%   for k,v in ssl_dict|dictsort -%}
+{%     set k = "_{}".format(k) -%}
+{%     if k in storage_dict -%}
+[{{ section('neo-ssl-' + k[1:]) }}]
+recipe = slapos.recipe.template:jinja2
+output = ${directory:etc}/{{v}}
+inline = {{'{{'}}pem}}
+context = key pem :pem
+pem = {{dumps(storage_dict.pop(k))}}
+
+{%     endif -%}
+{%   endfor -%}
+{% endif -%}
+{# endhack -#}
+
+
+[buildout]
+extends = {{ template_monitor }}
+parts +=
+  wcfs
+  wcfs-promise
+  publish
+  {{ '\n  '.join(part_list) }}

stack/erp5/instance.cfg.in

@@ -223,7 +223,6 @@ url = {{ instance_wcfs_cfg_in }}
 filename = instance_wcfs.cfg
 extra-context =
     section parameter_dict dynamic-template-zope-parameters
-    import urllib_parse six.moves.urllib.parse
 import-list =
     file instance_zodb_base context:template-zodb-base
 

stack/slapos.cfg

@@ -271,6 +271,7 @@ MarkupSafe = 2.1.3
 matplotlib = 2.1.2
 matplotlib-inline = 0.1.6:whl
 meld3 = 1.0.2
+miniupnpc = 2.2.8
 mistune = 0.8.4
 mock = 3.0.5
 more-itertools = 5.0.0
@@ -298,7 +299,7 @@ patsy = 0.5.3
 pbr = 5.9.0
 pexpect = 4.8.0
 pickleshare = 0.7.4
-pim-dm = 1.4.0nxd002
+pim-dm = 1.4.0+nxd002
 pkgconfig = 1.5.1:whl
 pkgutil-resolve-name = 1.3.10:whl
 platformdirs = 4.2.0:whl
@@ -332,7 +333,7 @@ pytest-runner = 5.2:whl
 python-dateutil = 2.8.2:whl
 python-json-logger = 2.0.7
 pytz = 2022.2.1
-PyYAML = 5.4.1
+PyYAML = 6.0.2
 pyzmq = 24.0.1
 qtconsole = 5.5.1
 qtpy = 2.4.1:whl
@@ -353,7 +354,7 @@ sgmllib3k = 1.0.0
 simplegeneric = 0.8.1
 singledispatch = 3.4.0.3
 six = 1.16.0
-slapos.cookbook = 1.0.367
+slapos.cookbook = 1.0.369
 slapos.core = 1.13.0
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.25
@@ -387,6 +388,7 @@ typeguard = 3.0.2:whl
 typing-extensions = 4.8.0:whl
 tzlocal = 1.5.1
 unicodecsv = 0.14.1
+unshare = 0.22
 uri-template = 1.2.0
 uritemplate = 4.1.1
 urllib3 = 1.26.12
@@ -427,6 +429,7 @@ importlib-metadata = 6.7.0:whl
 itsdangerous = 0.24
 Jinja2 = 2.11.3
 MarkupSafe = 2.0.1
+PyYAML = 5.4.1
 typing-extensions = 4.7.1:whl
 Werkzeug = 2.0.2