Update Release Candidate

component/ZODB/test-zodb4-wc2.cfg

@@ -23,6 +23,8 @@ parts =
 major = 4-wc2
 [ZODB4-wc2-repository]
 revision =
+[python]
+part = python2.7
 
 # we need persistent to be a git checkout because persistent tests want to
 # discover in-tree files that are not present in persistent egg when it is
@@ -76,7 +78,9 @@ inline =
         from os.path import dirname
         for m in "ZODB", "persistent", "BTrees", "transaction", "ZEO":
             print(dirname(__import__(m).__file__))
-  """])
+  """],
+    universal_newlines=True,
+  )
   ZODB, persistent, BTrees, transaction, ZEO = where.split()
 
   # run tests for whole ZODB stack

component/groonga/buildout.cfg

@@ -15,8 +15,8 @@ extends =
 [groonga]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://packages.groonga.org/source/groonga/groonga-12.0.4.tar.gz
-md5sum = a7c6416301e5b5899ef517193c510bd0
+url = https://packages.groonga.org/source/groonga/groonga-12.0.7.tar.gz
+md5sum = 5ef412a6941994e623cb50d76a8be261
 groonga-plugin-dir =  @@LOCATION@@/lib/groonga/plugins/
 # temporary patch to respect more tokens in natural language mode.
 patches =

component/mariadb/buildout.cfg

@@ -99,8 +99,8 @@ post-install =
 # as plugin-dir ( https://mariadb.com/kb/en/server-system-variables/#plugin_dir )
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://packages.groonga.org/source/mroonga/mroonga-12.04.tar.gz
-md5sum = 1d154e6cb4540f7be5791b5fb376a5de
+url = https://packages.groonga.org/source/mroonga/mroonga-12.07.tar.gz
+md5sum = bee7506e08deda9a607d85ed03b8b62e
 pre-configure =
   rm -rf fake_mariadb_source
   mkdir -p fake_mariadb_source
@@ -117,9 +117,8 @@ configure-options =
   --disable-document
 post-install =
   cp -rs ${mariadb:location}/lib/plugin/* ${:plugin-dir}
-  cat @@LOCATION@@/share/mroonga/install.sql @@LOCATION@@/share/mroonga/update.sql > ${:install-sql}
 plugin-dir = @@LOCATION@@/lib/plugin
-install-sql = @@LOCATION@@/share/mroonga/install-and-update.sql
+install-sql = @@LOCATION@@/share/mroonga/install.sql
 make-targets = plugindir=${:plugin-dir} install
 patch-options = -p1
 patches =

component/nxdtest/test-py3.cfg → component/nxdtest/test-py2.cfg

@@ -5,4 +5,4 @@ extends =
   test.cfg
 
 [python]
-part = python3
+part = python2.7

component/pygolang/test-py3.cfg → component/pygolang/test-py2.cfg

-# SlapOS software release to test pygolang/py3 on Nexedi testing infrastructure.
+# SlapOS software release to test pygolang/py2 on Nexedi testing infrastructure.
 
 [buildout]
 extends = test.cfg
 
 [python]
-part = python3
+part = python2.7

component/python3/buildout.cfg

@@ -9,7 +9,6 @@ extends =
   ../libpng/buildout.cfg
   ../ncurses/buildout.cfg
   ../openssl/buildout.cfg
-  ../patch/buildout.cfg
   ../readline/buildout.cfg
   ../sqlite3/buildout.cfg
   ../xz-utils/buildout.cfg
@@ -28,9 +27,6 @@ package_version_suffix =
 
 executable = @@LOCATION@@/bin/${:_buildout_section_name_}
 
-patch-options = -p1
-patches =
-  ${:_profile_base_location_}/default_encoding.patch#4ad9664e622d5556b4c32b1d9cb587ff
 url =
   https://www.python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz
 configure-options =
@@ -48,45 +44,30 @@ pre-install = mkdir profile-opt
 # the entry "-Wl,-rpath=${file:location}/lib" below is needed by python-magic,
 # which would otherwise load the system libmagic.so with ctypes
 environment =
-  PATH=${patch:location}/bin:${xz-utils:location}/bin:%(PATH)s
+  PATH=${xz-utils:location}/bin:%(PATH)s
   CPPFLAGS=-I${zlib:location}/include -I${xz-utils:location}/include -I${readline:location}/include -I${libexpat:location}/include -I${libffi:location}/include -I${ncurses:location}/include -I${ncurses:location}/include -I${bzip2:location}/include  -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
   LDFLAGS=-L${zlib:location}/lib -L${xz-utils:location}/lib -L${readline:location}/lib -L${libexpat:location}/lib -L${libffi:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${xz-utils:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
 
-[python3.6]
-<= python3-common
-version = 3.6
-package_version = 3.6.10
-md5sum = 986078f11b39074be22a199e56491d98
-
 [python3.7]
 <= python3-common
 version = 3.7
-package_version = 3.7.9
-md5sum = 389d3ed26b4d97c741d9e5423da1f43b
-patch-options =
-patches =
+package_version = 3.7.13
+md5sum = 10822726f75fd7efe05a94fbd6ac2258
 
 [python3.8]
 <= python3-common
 version = 3.8
-package_version = 3.8.9
-md5sum = 51b5bbf2ab447e66d15af4883db1c133
-patch-options =
-patches =
+package_version = 3.8.13
+md5sum = c4b7100dcaace9d33ab1fda9a3a038d6
 
 [python3.9]
 <= python3-common
 version = 3.9
 package_version = 3.9.13
 md5sum = 5e2411217b0060828d5f923eb422a3b8
-patch-options =
-patches =
-
 
 [python3.10]
 <= python3-common
 version = 3.10
-package_version = 3.10.5
-md5sum = f05727cb3489aa93cd57eb561c16747b
-patch-options =
-patches =
+package_version = 3.10.6
+md5sum = afc7e14f7118d10d1ba95ae8e2134bf0

component/python3/default_encoding.patch

---- Python-3.5.3/Programs/python.c.orig	2017-01-17 08:57:53.000000000 +0100
-+++ Python-3.5.3/Programs/python.c	2017-06-08 19:30:42.927077437 +0200
-@@ -24,6 +24,9 @@
-     int i, res;
-     char *oldloc;
- 
-+    if (!getenv("LC_CTYPE") && !getenv("LC_ALL"))
-+        setenv("LANG", "C.UTF-8", 0);
-+
-     argv_copy = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1));
-     argv_copy2 = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1));
-     if (!argv_copy || !argv_copy2) {

component/slapos/obs.cfg

@@ -31,6 +31,7 @@ init +=
   assert buildout['directory'] == buildout['destdir'] + buildout['rootdir'], (
     "Buildout MUST BE launched in destdir/rootdir (currently launched in %s but should be launched in %s)" %
     (buildout['directory'], buildout['destdir'] + buildout['rootdir']))
+part = python3.7
 
 [python3-common]
 configure-options +=
@@ -85,8 +86,8 @@ pre-configure =
   sed -i 's#/opt/slapos/parts/dbus/lib/libdbus-1.la#${dbus:location}/lib/libdbus-1.la#' ${dbus-glib:location}/lib/libdbus-glib-1.la
 environment +=
   LD_LIBRARY_PATH=${dbus:location}/lib
-  PYTHON_INCLUDES=-I${python:location}/include/python${python:version}
-  PYTHON_LIBS=-L${python:location}/lib -lpython${python:version} -lpthread -ldl -lutil -lm
+  PYTHON_INCLUDES=-I${python:location}/include/python${python:version}m
+  PYTHON_LIBS=-L${python:location}/lib -lpython${python:version}m -lpthread -ldl -lutil -lm
 post-install =
   sed -i 's#${dbus:location}/lib/libdbus-1.la#/opt/slapos/parts/dbus/lib/libdbus-1.la#' ${dbus-glib:location}/lib/libdbus-glib-1.la
 

software/caddy-frontend/test/test.py

@@ -302,7 +302,8 @@ class TestDataMixin(object):
       os.path.dirname(os.path.realpath(__file__)), 'test_data', filename)
 
     try:
-      test_data = open(test_data_file).read().strip()
+      with open(test_data_file) as fh:
+        test_data = fh.read().strip()
     except IOError:
       test_data = ''
 
@@ -498,6 +499,7 @@ def fakeHTTPSResult(domain, path, port=HTTPS_PORT,
   headers.setdefault('Via', 'http/1.1 clientvia')
 
   session = requests.Session()
+  with session:
     if source_ip is not None:
       new_source = source.SourceAddressAdapter(source_ip)
       session.mount('http://', new_source)
@@ -540,6 +542,7 @@ def fakeHTTPResult(domain, path, port=HTTP_PORT,
   headers.setdefault('Via', 'http/1.1 clientvia')
   headers['Host'] = '%s:%s' % (domain, port)
   session = requests.Session()
+  with session:
     if source_ip is not None:
       new_source = source.SourceAddressAdapter(source_ip)
       session.mount('http://', new_source)
@@ -742,12 +745,15 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     server_process = multiprocessing.Process(
       target=server.serve_forever, name='HTTPServer')
     server_process.start()
+    # from now on, socket is used by server subprocess, we can close it
+    server.socket.close()
     cls.logger.debug('Started process %s' % (server_process,))
 
     cls.backend_https_url = 'https://%s:%s/' % server_https.server_address
     server_https_process = multiprocessing.Process(
       target=server_https.serve_forever, name='HTTPSServer')
     server_https_process.start()
+    server_https.socket.close()
     cls.logger.debug('Started process %s' % (server_https_process,))
 
     class NetlocHandler(TestHandler):
@@ -759,6 +765,7 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     netloc_a_http_process = multiprocessing.Process(
       target=netloc_a_http.serve_forever, name='netloc-a-http')
     netloc_a_http_process.start()
+    netloc_a_http.socket.close()
 
     netloc_b_http = ThreadedHTTPServer(
       (cls._ipv4_address, cls._server_netloc_b_http_port),
@@ -766,6 +773,7 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     netloc_b_http_process = multiprocessing.Process(
       target=netloc_b_http.serve_forever, name='netloc-b-http')
     netloc_b_http_process.start()
+    netloc_b_http.socket.close()
 
     cls.server_process_list = [
       server_process,
@@ -821,6 +829,7 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     self.server_https_auth_process = multiprocessing.Process(
       target=server_https_auth.serve_forever, name='HTTPSServerAuth')
     self.server_https_auth_process.start()
+    server_https_auth.socket.close()
     self.logger.debug('Started process %s' % (self.server_https_auth_process,))
 
   def stopAuthenticatedServerProcess(self):
@@ -840,8 +849,8 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     assert ca_certificate.status_code == http.client.OK
     cls.kedifa_caucase_ca_certificate_file = os.path.join(
       cls.working_directory, 'kedifa-caucase.ca.crt.pem')
-    open(cls.kedifa_caucase_ca_certificate_file, 'w').write(
-        ca_certificate.text)
+    with open(cls.kedifa_caucase_ca_certificate_file, 'w') as fh:
+      fh.write(ca_certificate.text)
 
   @classmethod
   def _fetchBackendClientCaCertificateFile(cls, parameter_dict):
@@ -850,8 +859,8 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     assert ca_certificate.status_code == http.client.OK
     cls.backend_client_caucase_ca_certificate_file = os.path.join(
       cls.working_directory, 'backend-client-caucase.ca.crt.pem')
-    open(cls.backend_client_caucase_ca_certificate_file, 'w').write(
-      ca_certificate.text)
+    with open(cls.backend_client_caucase_ca_certificate_file, 'w') as fh:
+      fh.write(ca_certificate.text)
 
   @classmethod
   def setUpMaster(cls):
@@ -1391,8 +1400,9 @@ class SlaveHttpFrontendTestCase(HttpFrontendTestCase):
         self.instance_path, '*', 'var', 'log', 'httpd', log_name
       ))[0]
 
+    with open(log_file) as fh:
       self.assertRegex(
-      open(log_file, 'r').readlines()[-1],
+        fh.readlines()[-1],
         log_regexp)
 
 
@@ -2050,10 +2060,9 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     # check that monitor cors domains are correctly setup by file presence, as
     # we trust monitor stack being tested in proper place and it is too hard
     # to have working monitor with local proxy
-    self.assertTestData(
-      open(
-        os.path.join(
-          partition_path, 'etc', 'httpd-cors.cfg'), 'r').read().strip())
+    with open(os.path.join(
+          partition_path, 'etc', 'httpd-cors.cfg')) as fh:
+      self.assertTestData(fh.read().strip())
 
   def test_node_information_json(self):
     node_information_file_path = glob.glob(os.path.join(
@@ -2100,10 +2109,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
   def test_slave_partition_state(self):
     partition_path = self.getSlavePartitionPath()
-    self.assertTrue(
-      '-grace 2s' in
-      open(os.path.join(partition_path, 'bin', 'caddy-wrapper'), 'r').read()
-    )
+    with open(os.path.join(partition_path, 'bin', 'caddy-wrapper')) as fh:
+      self.assertIn('-grace 2s', fh.read())
 
   def test_monitor_conf(self):
     monitor_conf_list = glob.glob(
@@ -2112,8 +2119,11 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
       ))
     self.assertEqual(3, len(monitor_conf_list))
     expected = [(False, q) for q in monitor_conf_list]
-    got = [('!py!' in open(q).read(), q) for q in monitor_conf_list]
-    # check that no monitor.conf in generated configuratio has magic !py!
+    got = []
+    for q in monitor_conf_list:
+      with open(q) as fh:
+        got.append(('!py!' in fh.read(), q))
+    # check that no monitor.conf in generated configuration has magic !py!
     self.assertEqual(
       expected,
       got
@@ -2173,8 +2183,9 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
   def test_server_polluted_keys_removed(self):
     buildout_file = os.path.join(
       self.getMasterPartitionPath(), 'instance-caddy-replicate.cfg')
+    with open(buildout_file) as fh:
       for line in [
-      q for q in open(buildout_file).readlines()
+        q for q in fh.readlines()
         if q.startswith('config-slave-list') or q.startswith(
             'config-extra_slave_instance_list')]:
         self.assertFalse('slave_title' in line)
@@ -3680,10 +3691,10 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
       os.path.join(
         self.instance_path, '*', 'etc', 'caddy-slave-conf.d', '_ciphers.conf'
       ))[0]
-    self.assertTrue(
-      'ciphers RSA-3DES-EDE-CBC-SHA RSA-AES128-CBC-SHA'
-      in open(configuration_file).read()
-    )
+    with open(configuration_file) as fh:
+      self.assertIn(
+        'ciphers RSA-3DES-EDE-CBC-SHA RSA-AES128-CBC-SHA',
+        fh.read())
 
   def test_enable_cache_custom_domain(self):
     parameter_dict = self.assertSlaveBase(
@@ -5867,10 +5878,10 @@ class TestSlaveCiphers(SlaveHttpFrontendTestCase, TestDataMixin):
         self.instance_path, '*', 'etc', 'caddy-slave-conf.d',
         '_default_ciphers.conf'
       ))[0]
-    self.assertTrue(
-      'ciphers ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-RSA-AES256-GCM-SHA384'
-      in open(configuration_file).read()
-    )
+    with open(configuration_file) as fh:
+      self.assertIn(
+        'ciphers ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-RSA-AES256-GCM-SHA384',
+        fh.read())
 
   def test_own_ciphers(self):
     parameter_dict = self.assertSlaveBase('own_ciphers')
@@ -5893,10 +5904,10 @@ class TestSlaveCiphers(SlaveHttpFrontendTestCase, TestDataMixin):
         self.instance_path, '*', 'etc', 'caddy-slave-conf.d',
         '_own_ciphers.conf'
       ))[0]
-    self.assertTrue(
-      'ciphers ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-RSA-AES128-GCM-SHA256'
-      in open(configuration_file).read()
-    )
+    with open(configuration_file) as fh:
+      self.assertIn(
+        'ciphers ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-RSA-AES128-GCM-SHA256',
+        fh.read())
 
 
 class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):

software/html5as/buildout.hash.cfg

@@ -17,7 +17,7 @@
 
 [template-cfg]
 filename = instance.cfg.in
-md5sum = ef264514b64a4b2c77c9965c587c6d34
+md5sum = 23c15a579b66cef866b30a2f53b1b737
 
 [instance_html5as]
 _update_hash_filename_ = instance_html5as.cfg.in
@@ -45,4 +45,4 @@ md5sum = 1c0ee16966e1fcdb3fd11c09f12ee2b2
 
 [template_instance_replicate]
 _update_hash_filename_ = instance_replicate.cfg.in
-md5sum = 7ff7e11d05145115f53564ec1af205ef
+md5sum = d7071867625070c27dbd6456c761f9f0

software/html5as/instance.cfg.in

@@ -61,7 +61,7 @@ default = instance-html5as:output
 replicate = instance-replicate:output
 
 [slap-configuration]
-recipe = slapos.cookbook:slapconfiguration
+recipe = slapos.cookbook:slapconfiguration.serialised
 computer = ${slap-connection:computer-id}
 partition = ${slap-connection:partition-id}
 url = ${slap-connection:server-url}

software/html5as/instance_replicate.cfg.in

@@ -23,7 +23,7 @@ offline = true
 # Macro section sharing request parameters
 [instance-request-base]
 <= slap-connection
-recipe = slapos.cookbook:request
+recipe = slapos.cookbook:request.serialised
 # It is the same software as the current one
 software-url = ${slap-connection:software-release-url}
 # We want the default behaviour
@@ -51,7 +51,7 @@ sla-computer_guid = {{ parameter_dict["sla-%s-computer-guid" % i] }}
 
 # Publish information to connect to the two instances
 [publish-connection-information]
-recipe = slapos.cookbook:publish
+recipe = slapos.cookbook:publish.serialised
 {% for i in range(1, replicate_quantity + 1) %}
 instance-{{ i }}-server_url = ${instance-{{ i }}:connection-server_url}
 instance-{{ i }}-server-cdn-url = ${instance-{{ i }}:connection-server-cdn-url}

software/html5as/test/test.py

@@ -25,12 +25,14 @@
 #
 ##############################################################################
 
+import json
 import os
 import requests
-from urlparse import urlparse
+from urllib.parse import urlparse
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 
+
 setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
         os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
@@ -43,8 +45,8 @@ class HTML5ASTestCase(SlapOSInstanceTestCase):
     * Install the software release.
     * Checks it compile without issue.
     * Deploy the instance
-    * Check deployement works and promise pass
-  For testing the deployement a different testing class will need to be set up
+    * Check deployment works and promise pass
+  For testing the deployment a different testing class will need to be set up
   per each variation of parameters the instance needs to be given.
   """
 
@@ -63,11 +65,12 @@ class TestEmptyDeploy(HTML5ASTestCase):
   """
 
   def test_deploy_with_no_paramater(self):
-    url = self.requestDefaultInstance().getConnectionParameterDict()['server_url']
+    url = json.loads(self.computer_partition.getConnectionParameterDict()['_'])['server_url']
     response = self.checkUrlAndGetResponse(url)
     result = response.text
-    self.assertFalse("<h1>" in result)
-    self.assertTrue("<p>Hello World</p>" in result)
+    self.assertNotIn("<h1>", result)
+    self.assertIn("<p>Hello World</p>", result)
+
 
 class TestDeployWithTitle(HTML5ASTestCase):
   """
@@ -77,17 +80,21 @@ class TestDeployWithTitle(HTML5ASTestCase):
   @classmethod
   def getInstanceParameterDict(cls):
     return {
+      '_': json.dumps(
+        {
           'title': 'Test1',
         }
+      )
+    }
 
   def test_deploy_with_title_parameter(self):
-    connection_parameter_dict = self.computer_partition.getConnectionParameterDict()
+    connection_parameter_dict = json.loads(self.computer_partition.getConnectionParameterDict()['_'])
     self.assertEqual(connection_parameter_dict["title"], "Title Test1!")
     url = connection_parameter_dict['server_url']
     response = self.checkUrlAndGetResponse(url)
     result = response.text
-    self.assertTrue("<h1>Test1</h1>" in result)
-    self.assertTrue("<p>Hello World</p>" in result)
+    self.assertIn("<h1>Test1</h1>", result)
+    self.assertIn("<p>Hello World</p>", result)
 
 class TestGracefulWithPortChange(HTML5ASTestCase):
   """
@@ -95,7 +102,9 @@ class TestGracefulWithPortChange(HTML5ASTestCase):
   """
 
   instance_parameter_dict = {
+    '_': json.dumps({
       'port': 8087
+    })
   }
 
   @classmethod
@@ -107,23 +116,23 @@ class TestGracefulWithPortChange(HTML5ASTestCase):
     This test test port change and its application with graceful restart
     """
     # Check initial connection parameter match expected port
-    url = self.computer_partition.getConnectionParameterDict()['server_url']
+    url = json.loads(self.computer_partition.getConnectionParameterDict()['_'])['server_url']
     self.assertEqual(urlparse(url).port, 8087)
     # Check port is listening even thought it is duplicated with the promise:
     # "port-listening-promise"
     self.checkUrlAndGetResponse(url)
 
     # Update port parameter
-    self.instance_parameter_dict.update({
-      'port': 8086,
-
+    self.instance_parameter_dict['_'] = json.dumps({
+      'port': 8086
     })
+
     # Request instance with the new port parameter
     self.requestDefaultInstance()
     # Reprocess the instance to apply new port and run promises
     self.slap.waitForInstance(self.instance_max_retry)
-    # Rerequest instance to get update connection parameter
-    url = self.requestDefaultInstance().getConnectionParameterDict()['server_url']
+    # Re-request instance to get update connection parameter
+    url = json.loads(self.requestDefaultInstance().getConnectionParameterDict()['_'])['server_url']
     # Make sure the new port is the one being used
     self.assertEqual(urlparse(url).port, 8086)
 
@@ -138,8 +147,10 @@ class TestReplicateHTML5AS(HTML5ASTestCase):
   """
 
   instance_parameter_dict = {
+    '_': json.dumps({
       "port-1": 8088,
       "title-1": "Title 1",
+    })
   }
 
   @classmethod
@@ -152,31 +163,36 @@ class TestReplicateHTML5AS(HTML5ASTestCase):
 
   def test_replicate_instance(self):
     # Check First instance is deployed with proper parameters
-    connection_parameter_dict = self.computer_partition.getConnectionParameterDict()
+    connection_parameter_dict = json.loads(self.computer_partition.getConnectionParameterDict()['_'])
     url = connection_parameter_dict['instance-1-server_url']
     self.assertEqual(urlparse(url).port, 8088)
     response = self.checkUrlAndGetResponse(url)
     result = response.text
-    self.assertTrue("<h1>Title 1</h1>" in result)
+    self.assertIn("<h1>Title 1</h1>", result)
 
     # Check only one instance is deployed by default
-    self.assertTrue("instance-2-server_url" not in connection_parameter_dict)
+    self.assertNotIn("instance-2-server_url", connection_parameter_dict)
 
     # Update replicate quantity parameter
-    self.instance_parameter_dict.update({
+    self.instance_parameter_dict['_'] = json.dumps(
+      dict(
+        json.loads(self.instance_parameter_dict['_']),
+        **{
           'replicate-quantity': 2,
           'port-2': 8089,
           'sla-2-computer_guid': self.slap._computer_id,
           "title-2": "Title 314",
-    })
+        }
+      )
+    )
     # Request instance with the one more replicate
     self.requestDefaultInstance()
     self.slap.waitForInstance(self.instance_max_retry)
 
     # Check the second replicate
-    connection_parameter_dict = self.requestDefaultInstance().getConnectionParameterDict()
+    connection_parameter_dict = json.loads(self.requestDefaultInstance().getConnectionParameterDict()['_'])
     url = connection_parameter_dict['instance-2-server_url']
     self.assertEqual(urlparse(url).port, 8089)
     response = self.checkUrlAndGetResponse(url)
     result = response.text
-    self.assertTrue("<h1>Title 314</h1>" in result)
+    self.assertIn("<h1>Title 314</h1>", result)

software/htmlvalidatorserver/software.cfg

@@ -45,7 +45,5 @@ url = ${:_profile_base_location_}/${:filename}
 output = ${buildout:directory}/template.cfg
 
 [versions]
-# 1.3.4nxd2 is invalid version string, thus pached version string is not '1.3.4nxd2+SlapOSPatched001'
-# but '1.3.4nxd2-SlapOSPatched001'.
 gunicorn = 19.1.1
 inotifyx = 0.2.2

software/htmlvalidatorserver/test/test.py

@@ -41,7 +41,6 @@ class TestHtmlValidatorServer(InstanceTestCase):
   def test(self):
     parameter_dict = self.computer_partition.getConnectionParameterDict()
 
-    # Check that there is a RSS feed
     self.assertTrue('vnu-url' in parameter_dict)
     self.assertEqual(
       'https://[%s]:8899/' % (self._ipv6_address, ),

software/jstestnode/buildout.hash.cfg

@@ -27,4 +27,4 @@ md5sum = 98faa5ad8cfb23a11d97a459078a1d05
 
 [template-runTestSuite]
 filename = runTestSuite.in
-md5sum = 70d10e7005b98620ddbbf30030b08198
+md5sum = 54d585d50a3464100611774db68b72c0

software/jstestnode/runTestSuite.in

@@ -33,7 +33,7 @@ def main():
   parser = argparse.ArgumentParser(description='Run a test suite.')
   parser.add_argument('--test_suite', help='The test suite name',
                       default=parsed_parameters.get('test-suite', ''),
-                      required=not parsed_parameters.has_key('test-suite'))
+                      required='test-suite' not in parsed_parameters)
   parser.add_argument('--test_suite_title', help='The test suite title')
   parser.add_argument('--test_node_title', help='The test node title')
   parser.add_argument('--project_title', help='The project title')
@@ -96,7 +96,7 @@ def main():
             firefox_binary='${firefox-wrapper-68:location}',
             executable_path='${geckodriver:location}')
       else:
-        assert target == 'selenium-server', "Unsupported target {}".format(test_runner['target'])
+        assert target == 'selenium-server', f"Unsupported target {test_runner['target']}"
         # use a remote connection which optionally verifies TLS certificate
         # and optionally against a user provided CA.
         executor = RemoteConnection(test_runner['server-url'], keep_alive=True)
@@ -120,14 +120,14 @@ def main():
       remote_access_url = parsed_parameters.get('remote-access-url')
       if remote_access_url:
         if ('jio' in test_suite):
-          url = '{}/jio/test/tests.html'.format(remote_access_url)
+          url = f'{remote_access_url}/jio/test/tests.html'
         else:
-          url = '{}/renderjs/test/'.format(remote_access_url)
+          url = f'{remote_access_url}/renderjs/test'
 
       is_browser_running = True
       agent = browser.execute_script("return navigator.userAgent")
-      print agent
-      print url
+      print(agent)
+      print(url)
 
       browser.get(url)
       WebDriverWait(browser, 300).until(EC.presence_of_element_located((
@@ -137,7 +137,7 @@ def main():
       html_parser = etree.HTMLParser(recover=True)
       body = etree.fromstring(browser.page_source.encode('UTF-8'), html_parser)
 
-      print ' '.join(body.xpath('//*[@id="qunit-testresult"]//text()'))
+      print(' '.join(body.xpath('//*[@id="qunit-testresult"]//text()')))
 
       for elt in body.xpath('.//ol[@id="qunit-tests"]/li'):
         if (len(elt.xpath('.//span[@class="module-name"]'))):
@@ -149,7 +149,7 @@ def main():
         else:
           test_name = elt.xpath('.//span[@class="test-name"]')[0].text
 
-        print elt.get('class'), ''.join(elt.xpath('.//strong')[0].itertext())
+        print(elt.get('class'), ''.join(elt.xpath('.//strong')[0].itertext()))
         # print elt.find_element_by_tag_name('ol').get_attribute('innerHTML')
 
         failure = int(elt.xpath('.//b[@class="failed"]')[0].text)
@@ -176,7 +176,7 @@ def main():
     # Send results
     tool = taskdistribution.TaskDistributor(portal_url=args.master_url)
     test_result = tool.createTestResult(revision = revision,
-                                        test_name_list = test_line_dict.keys(),
+                                        test_name_list = list(test_line_dict.keys()),
                                         node_title = args.test_node_title,
                                         test_title = test_suite_title,
                                         project_title = args.project_title)
@@ -186,17 +186,17 @@ def main():
     while 1:
       test_result_line = test_result.start()
       if not test_result_line:
-        print 'No test result anymore.'
+        print('No test result anymore.')
         break
 
-      print 'Submitting: "%s"' % test_result_line.name
+      print('Submitting: "%s"' % test_result_line.name)
       # report status back to Nexedi ERP5
       test_result_line.stop(**test_line_dict[test_result_line.name])
 
   except:
     # Catch any exception here, to warn user instead of being silent,
     # by generating fake error result
-    print traceback.format_exc()
+    print(traceback.format_exc())
     result = dict(status_code=-1,
                   command=url,
                   stderr=traceback.format_exc(),

software/jstestnode/software.cfg

@@ -29,9 +29,6 @@ parts =
   openssl
   instance
 
-[python]
-part = python2.7
-
 [nodejs]
 <= nodejs-14.16.0
 

software/ors-amarisoft/buildout.hash.cfg

@@ -20,7 +20,7 @@ md5sum = fc7bccab64e6794b33766ca9925231c8
 
 [template-lte-enb-epc]
 _update_hash_filename_ = instance-enb-epc.jinja2.cfg
-md5sum = b10c10a9408a934052fd0960f31a75fc
+md5sum = c9994f2ef03edd7d6773307a6385c47c
 
 [template-lte-enb]
 _update_hash_filename_ = instance-enb.jinja2.cfg
@@ -28,11 +28,11 @@ md5sum = c450b8c307d20738a98bcb4f6ffeee16
 
 [template-lte-gnb-epc]
 _update_hash_filename_ = instance-gnb-epc.jinja2.cfg
-md5sum = 5f86177e7ef18a6046826f96397ded1d
+md5sum = 4b27f60eab8d71c2c47f21a845fddb47
 
 [template-lte-epc]
 _update_hash_filename_ = instance-epc.jinja2.cfg
-md5sum = 2cfe5679374559ab94478b987d646d83
+md5sum = b17674f523adce9b0dda942c1493674d
 
 [template-lte-gnb]
 _update_hash_filename_ = instance-gnb.jinja2.cfg

software/ors-amarisoft/instance-enb-epc.jinja2.cfg

@@ -51,7 +51,7 @@ config-monitor-password = ${monitor-htpasswd:passwd}
 name = MME
 software-type = mme
 config-name = mme
-{%- for key, value in slapparameter_dict.iteritems() %}
+{%- for key, value in slapparameter_dict.items() %}
 config-{{ key }} = {{ dumps(value) }}
 {% endfor -%}
 config-slave-list = {{ dumps(slave_instance_list) }}
@@ -62,7 +62,7 @@ return = monitor-base-url epc-ipv6
 name = eNB
 software-type = enb
 config-name = enb
-{%- for key, value in slapparameter_dict.iteritems() %}
+{%- for key, value in slapparameter_dict.items() %}
 config-{{ key }} = {{ dumps(value) }}
 {% endfor -%}
 return = monitor-base-url

software/ors-amarisoft/instance-epc.jinja2.cfg

@@ -51,7 +51,7 @@ return = monitor-base-url epc-ipv6
 name = MME
 software-type = mme
 config-name = mme
-{%- for key, value in slapparameter_dict.iteritems() %}
+{%- for key, value in slapparameter_dict.items() %}
 config-{{ key }} = {{ dumps(value) }}
 {% endfor -%}
 config-slave-list = {{ dumps(slave_instance_list) }}

software/ors-amarisoft/instance-gnb-epc.jinja2.cfg

@@ -51,7 +51,7 @@ config-monitor-password = ${monitor-htpasswd:passwd}
 name = MME
 software-type = mme
 config-name = mme
-{%- for key, value in slapparameter_dict.iteritems() %}
+{%- for key, value in slapparameter_dict.items() %}
 config-{{ key }} = {{ dumps(value) }}
 {% endfor -%}
 config-slave-list = {{ dumps(slave_instance_list) }}
@@ -62,7 +62,7 @@ return = monitor-base-url epc-ipv6
 name = gNB
 software-type = gnb
 config-name = gnb
-{%- for key, value in slapparameter_dict.iteritems() %}
+{%- for key, value in slapparameter_dict.items() %}
 config-{{ key }} = {{ dumps(value) }}
 {% endfor -%}
 return = monitor-base-url

software/ors-amarisoft/software.cfg

@@ -21,8 +21,6 @@ parts +=
 #  apache-php
 #  logrotate
 
-[python]
-part = python2.7
 
 [template]
 recipe = slapos.recipe.template

software/seleniumserver/buildout.hash.cfg

@@ -19,4 +19,4 @@ md5sum = 9b41238f67ea8e12f8ea9590fd552b33
 
 [template-selenium]
 filename = instance-selenium.cfg.in
-md5sum = 3e50d629307ca9f1381909102a52756d
+md5sum = 9b4742e8a249aef38c7c8c12d74b0605

software/seleniumserver/instance-selenium.cfg.in

@@ -316,10 +316,10 @@ recipe = slapos.recipe.template
 output = $${directory:bin}/$${:_buildout_section_name_}
 inline =
   #!${buildout:executable}
-  import json, urllib, sys
+  import json, urllib.request, sys
   api_url = sys.argv[1]
   expected_node_count = int(sys.argv[2])
-  actual_node_count = json.load(urllib.urlopen(api_url))['slotCounts']['total']
+  actual_node_count = json.load(urllib.request.urlopen(api_url))['slotCounts']['total']
   sys.exit(0 if expected_node_count == actual_node_count else 1)
 
 

software/seleniumserver/software.cfg

@@ -23,9 +23,6 @@ parts =
   collective.recipe.shelloutput
   template
 
-[python]
-part = python2.7
-
 [collective.recipe.shelloutput]
 recipe = zc.recipe.egg
 

software/seleniumserver/test/test.py

@@ -31,12 +31,12 @@ import multiprocessing
 import os
 import tempfile
 import unittest
-import urlparse
+import urllib.parse
 import base64
 import hashlib
 import logging
 import contextlib
-from BaseHTTPServer import BaseHTTPRequestHandler
+from http.server import BaseHTTPRequestHandler
 
 from io import BytesIO
 
@@ -72,7 +72,7 @@ class WebServer(ManagedHTTPServer):
       self.send_response(200)
       self.end_headers()
       self.wfile.write(
-          '''
+          b'''
         <html>
           <title>Test page</title>
           <body>
@@ -97,15 +97,15 @@ class WebServer(ManagedHTTPServer):
       self.send_response(200)
       self.end_headers()
       file_data = 'no file'
-      if form.has_key('f'):
-        file_data = form['f'].file.read()
+      if 'f' in form:
+        file_data = form['f'].file.read().decode()
       self.wfile.write(
-          '''
+          ('''
         <html>
           <title>%s</title>
           <div>%s</div>
         </html>
-      ''' % (form['q'].value, file_data))
+      ''' % (form['q'].value, file_data)).encode())
 
     log_message = logging.getLogger(__name__ + '.WebServer').info
 
@@ -148,7 +148,7 @@ class BrowserCompatibilityMixin(WebServerMixin):
     WebDriverWait(self.driver, 3).until(EC.title_is(self.id()))
 
   def test_upload_file(self):
-    f = tempfile.NamedTemporaryFile(delete=False)
+    f = tempfile.NamedTemporaryFile(delete=False, mode='w')
     f.write(self.id())
     f.close()
     self.addCleanup(lambda: os.remove(f.name))
@@ -307,7 +307,7 @@ class TestFrontend(WebServerMixin, SeleniumServerTestCase):
     parameter_dict = self.computer_partition.getConnectionParameterDict()
     admin_url = parameter_dict['admin-url']
 
-    parsed = urlparse.urlparse(admin_url)
+    parsed = urllib.parse.urlparse(admin_url)
     self.assertEqual('admin', parsed.username)
     self.assertTrue(parsed.password)
 
@@ -316,7 +316,7 @@ class TestFrontend(WebServerMixin, SeleniumServerTestCase):
   def test_browser_use_hub(self):
     parameter_dict = self.computer_partition.getConnectionParameterDict()
     webdriver_url = parameter_dict['url']
-    parsed = urlparse.urlparse(webdriver_url)
+    parsed = urllib.parse.urlparse(webdriver_url)
     self.assertEqual('selenium', parsed.username)
     self.assertTrue(parsed.password)
 
@@ -343,7 +343,7 @@ class TestSSHServer(SeleniumServerTestCase):
   def test_connect(self):
     parameter_dict = self.computer_partition.getConnectionParameterDict()
     ssh_url = parameter_dict['ssh-url']
-    parsed = urlparse.urlparse(ssh_url)
+    parsed = urllib.parse.urlparse(ssh_url)
     self.assertEqual('ssh', parsed.scheme)
 
     client = paramiko.SSHClient()
@@ -359,9 +359,9 @@ class TestSSHServer(SeleniumServerTestCase):
 
     with contextlib.closing(client):
       client.connect(
-          username=urlparse.urlparse(ssh_url).username,
-          hostname=urlparse.urlparse(ssh_url).hostname,
-          port=urlparse.urlparse(ssh_url).port,
+          username=urllib.parse.urlparse(ssh_url).username,
+          hostname=urllib.parse.urlparse(ssh_url).hostname,
+          port=urllib.parse.urlparse(ssh_url).port,
           pkey=self.ssh_key,
       )
 
@@ -378,7 +378,7 @@ class TestSSHServer(SeleniumServerTestCase):
       # Paramiko does not allow to get the fingerprint as SHA256 easily yet
       # https://github.com/paramiko/paramiko/pull/1103
       self.assertEqual(
-          fingerprint,
+          fingerprint.encode(),
           # XXX with sha256, we need to remove that trailing =
           base64.b64encode(
               hashlib.new(fingerprint_algorithm,
@@ -386,15 +386,15 @@ class TestSSHServer(SeleniumServerTestCase):
 
       channel = client.invoke_shell()
       channel.settimeout(30)
-      received = ''
+      received = b''
       while True:
         r = channel.recv(1024)
         if not r:
           break
         received += r
-        if 'Selenium Server.' in received:
+        if b'Selenium Server.' in received:
           break
-      self.assertIn("Welcome to SlapOS Selenium Server.", received)
+      self.assertIn(b"Welcome to SlapOS Selenium Server.", received)
 
 
 class TestFirefox52(
@@ -405,8 +405,9 @@ class TestFirefox52(
   desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='52.9.0esr')
   user_agent = 'Gecko/20100101 Firefox/52.0'
   # resizing window is not supported on firefox 52 geckodriver
-  test_resize_window = unittest.expectedFailure(
-      BrowserCompatibilityMixin.test_resize_window)
+  @unittest.expectedFailure
+  def test_resize_window(self):
+    super().test_resize_window()
 
 
 class TestFirefox60(

software/slapos-sr-testing/software-py3.cfg

@@ -16,18 +16,26 @@ extra =
   caddy-frontend ${slapos.test.caddy-frontend-setup:setup}
   erp5testnode ${slapos.test.erp5testnode-setup:setup}
   galene ${slapos.test.galene-setup:setup}
+  grafana ${slapos.test.grafana-setup:setup}
   headless-chromium ${slapos.test.headless-chromium-setup:setup}
   helloworld ${slapos.test.helloworld-setup:setup}
+  html5as ${slapos.test.html5as-setup:setup}
+  html5as-base ${slapos.test.html5as-base-setup:setup}
   hugo ${slapos.test.hugo-setup:setup}
+  jstestnode ${slapos.test.jstestnode-setup:setup}
   jupyter ${slapos.test.jupyter-setup:setup}
   kvm ${slapos.test.kvm-setup:setup}
   matomo ${slapos.test.matomo-setup:setup}
+  metabase ${slapos.test.metabase-setup:setup}
   monitor ${slapos.test.monitor-setup:setup}
   nextcloud ${slapos.test.nextcloud-setup:setup}
   nginx-push-stream ${slapos.test.nginx-push-stream-setup:setup}
+  ors-amarisoft ${slapos.test.ors-amarisoft-setup:setup}
   plantuml ${slapos.test.plantuml-setup:setup}
   powerdns ${slapos.test.powerdns-setup:setup}
   proftpd ${slapos.test.proftpd-setup:setup}
   repman ${slapos.test.repman-setup:setup}
   restic-rest-server ${slapos.test.restic_rest_server-setup:setup}
+  seleniumserver ${slapos.test.seleniumserver-setup:setup}
   theia ${slapos.test.theia-setup:setup}
+  turnserver ${slapos.test.turnserver-setup:setup}

software/slapos-sr-testing/software.cfg

@@ -359,8 +359,6 @@ tests =
 # here, to check there's no promise issue when slapos node runs with Python 2.
   erp5 ${slapos.test.erp5-setup:setup}
   fluentd ${slapos.test.fluentd-setup:setup}
-  metabase ${slapos.test.metabase-setup:setup}
-  ors-amarisoft ${slapos.test.ors-amarisoft-setup:setup}
 ###
   ${:extra}
 
@@ -373,17 +371,11 @@ extra =
   cloudooo ${slapos.test.cloudooo-setup:setup}
   dream ${slapos.test.dream-setup:setup}
   gitlab ${slapos.test.gitlab-setup:setup}
-  grafana ${slapos.test.grafana-setup:setup}
-  html5as ${slapos.test.html5as-setup:setup}
-  html5as-base ${slapos.test.html5as-base-setup:setup}
   htmlvalidatorserver ${slapos.test.htmlvalidatorserver-setup:setup}
   jscrawler ${slapos.test.jscrawler-setup:setup}
-  jstestnode ${slapos.test.jstestnode-setup:setup}
   re6stnet ${slapos.test.re6stnet-setup:setup}
-  seleniumserver ${slapos.test.seleniumserver-setup:setup}
   slapos-master ${slapos.test.slapos-master-setup:setup}
   slaprunner ${slapos.test.slaprunner-setup:setup}
-  turnserver ${slapos.test.turnserver-setup:setup}
   upgrade_erp5 ${slapos.test.upgrade_erp5-setup:setup}
 
 [versions]

software/turnserver/software.cfg

@@ -14,9 +14,6 @@ parts =
   coturn
   instance-cfg
 
-[python]
-part = python2.7
-
 [download-base]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:filename}

software/turnserver/test/test.py

@@ -29,7 +29,7 @@ import os
 import subprocess
 import json
 import glob
-import ConfigParser
+import configparser
 
 from slapos.recipe.librecipe import generateHashFromFiles
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
@@ -96,7 +96,7 @@ class TestServices(TurnServerTestCase):
     secret_file = os.path.join(self.partition_path, 'etc/.turnsecret')
     self.assertTrue(os.path.exists(self.partition_path))
     self.assertTrue(os.path.exists(secret_file))
-    config = ConfigParser.ConfigParser()
+    config = configparser.ConfigParser()
     with open(secret_file) as f:
       config.readfp(f)
     secret = config.get('turnserver', 'secret')
@@ -151,7 +151,7 @@ class TestParameters(TurnServerTestCase):
     secret_file = os.path.join(self.partition_path, 'etc/.turnsecret')
     self.assertTrue(os.path.exists(self.partition_path))
     self.assertTrue(os.path.exists(secret_file))
-    config = ConfigParser.ConfigParser()
+    config = configparser.ConfigParser()
     with open(secret_file) as f:
       config.readfp(f)
     secret = config.get('turnserver', 'secret')

stack/slapos.cfg

@@ -194,7 +194,7 @@ setuptools-dso = 1.7
 rubygemsrecipe  = 0.4.3
 six = 1.16.0
 slapos.cookbook = 1.0.253
-slapos.core = 1.7.13
+slapos.core = 1.8.0
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.25
 slapos.rebootstrap = 4.5