Update Release Candidate

component/ZODB/buildout.cfg

@@ -96,7 +96,7 @@ setup-eggs = ${python-cffi:egg}
 [versions]
 BTrees = 4.5.1
 persistent = 4.6.4
-zodbpickle = 1.0.4
+zodbpickle = 2.0.0
 
 # Provide ZODB3 for those eggs that still care about ZODB3 compatibility -
 # for example wendelin.core. ZODB3 3.11 is just a dependency egg on _latest_

component/caddy/buildout.cfg

@@ -10,7 +10,7 @@ parts =
 <= go-git-package
 go.importpath = github.com/caddyserver/caddy
 repository    = https://lab.nexedi.com/nexedi/caddy.git
-revision      = nxd-v1.0.3-1-g2c11cedc
+revision      = nxd-v1.0.3-1-03fba31bf
 
 [gowork]
 golang  = ${golang1.17:location}

component/dcron/buildout.cfg

@@ -2,7 +2,6 @@
 extends =
   ../coreutils/buildout.cfg
   ../patch/buildout.cfg
-  ../randomsleep/buildout.cfg
 
 parts = dcron-output
 
@@ -20,7 +19,6 @@ make-options =
   PREFIX=${buildout:parts-directory}/${:_buildout_section_name_}
 post-install =
   chmod u-s %(location)s/bin/crontab
-dummy = ${randomsleep:recipe}
 
 [dcron-output]
 # Shared binary location to ease migration

component/inkscape/buildout.cfg

@@ -23,6 +23,8 @@ extends =
   ../popt/buildout.cfg
   ../xorg/buildout.cfg
   ../zlib/buildout.cfg
+# Inkscape < 1.1 only supports python2
+  ../python-2.7/buildout.cfg
 
 [gsl]
 recipe = slapos.recipe.cmmi
@@ -52,7 +54,7 @@ configure-options =
   -DWITH_LIBVISIO=OFF
   -DWITH_LIBWPG=OFF
 environment =
-  PATH=${cmake:location}/bin:${freetype:location}/bin:${gdk-pixbuf:location}/bin:${gettext:location}/bin:${glib:location}/bin:${intltool:location}/bin:${libxml2:location}/bin:${pango:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:%(PATH)s
+  PATH=${cmake:location}/bin:${freetype:location}/bin:${gdk-pixbuf:location}/bin:${gettext:location}/bin:${glib:location}/bin:${intltool:location}/bin:${libxml2:location}/bin:${pango:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:${python2.7:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${:pkg_config_depends}
   CMAKE_INCLUDE_PATH=${boost-lib:location}/include:${freetype:location}/include:${garbage-collector:location}/include:${libjpeg:location}/include:${lcms2:location}/include:${libpng:location}/include:${zlib:location}/include
   CMAKE_LIBRARY_PATH=${boost-lib:location}/lib:${freetype:location}/lib:${garbage-collector:location}/lib:${lcms2:location}/lib:${libjpeg:location}/lib:${libpng:location}/lib:${zlib:location}/lib

component/lunzip/buildout.cfg

@@ -5,7 +5,7 @@ parts =
 [lunzip]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://download.savannah.gnu.org/releases-redirect/lzip/lunzip/lunzip-1.12.tar.gz
-md5sum = 09caf2475c58aa40e94b599a4f7a2d13
+url = http://download-mirror.savannah.gnu.org/releases/lzip/lunzip/lunzip-1.13.tar.gz
+md5sum = 4bc15e65fef99db64e27f4cd369ae02e
 configure-options =
   --disable-static

component/randomsleep/buildout.cfg

@@ -4,10 +4,10 @@ extends =
   ../bash/buildout.cfg
 
 [randomsleep]
-recipe = slapos.recipe.template:jinja2
-rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
-template =
-  inline:#!${bash:location}/bin/bash
+recipe = slapos.recipe.template
+output = ${buildout:bin-directory}/${:_buildout_section_name_}
+inline =
+  #!${bash:location}/bin/bash
   [ $# = 1 ] || {
     echo "usage: ${:_buildout_section_name_} maxseconds"
     exit 1

component/sqlite3/buildout.cfg

@@ -8,8 +8,8 @@ parts =
 [sqlite3]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://sqlite.org/2021/sqlite-autoconf-3360000.tar.gz
-md5sum = f5752052fc5b8e1b539af86a3671eac7
+url = https://sqlite.org/2022/sqlite-autoconf-3370200.tar.gz
+md5sum = 683cc5312ee74e71079c14d24b7a6d27
 configure-options =
   --disable-static
   --enable-readline

component/wendelin.core/buildout.cfg

@@ -50,7 +50,7 @@ CGO_LDFLAGS += -Wl,-rpath=${zlib:location}/lib
 recipe  = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/wendelin.core.git
 branch   = master
-revision = wendelin.core-2.0.alpha2-1-gad6305c0
+revision = wendelin.core-2.0.alpha2-2-g3d0f134c
 # dir is pretty name as top-level recipe
 location = ${buildout:parts-directory}/wendelin.core
 git-executable = ${git:location}/bin/git

slapos/recipe/check_parameter/template/check_parameter.py.in

 #!%(python_path)s
 # BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
+from __future__ import print_function
 import socket
 import sys
 
@@ -9,9 +10,9 @@ expected = "%(expected-value)s"
 not_expected = "%(expected-not-value)s"
 
 if expected != "" and value != expected:
-  print "FAIL: %%s != %%s" %% (value, expected)
+  print("FAIL: %%s != %%s" %% (value, expected))
   sys.exit(127)
 
 if not_expected != "" and value == not_expected: 
-  print "FAIL: %%s == %%s" %% (value, not_expected)
+  print("FAIL: %%s == %%s" %% (value, not_expected))
   sys.exit(127)

slapos/recipe/generic_mysql/mysql.py

@@ -8,7 +8,7 @@ import pytz
 
 def updateMysql(mysql_upgrade_binary, mysql_binary, mysql_script_file):
   sleep = 0
-  with open(mysql_script_file) as script_file:
+  with open(mysql_script_file, 'rb') as script_file:
     mysql_script = script_file.read()
   mysql_list = mysql_binary, '-B'
   mysql_tzinfo_to_sql_list = (

slapos/recipe/zeo/__init__.py

@@ -57,7 +57,7 @@ class Recipe(GenericBaseRecipe):
     # Prepare all filestorages
     filestorage_snippet = ""
     storage = self.options['storage']
-    if isinstance(storage, basestring):
+    if isinstance(storage, str):
       for storage_definition in storage.splitlines():
         storage_definition = storage_definition.strip()
         if not storage_definition:

software/beremiz-runtime/buildout.hash.cfg

+[instance-profile]
+filename = instance.cfg.in
+md5sum = ed202dc7d69f517e325f01dc3ac4fe32

software/beremiz-runtime/instance.cfg.in

+#############################
+#
+# Deploy beremiz' runtime  instance
+#
+#############################
+[buildout]
+parts =
+  directory
+  publish-connection-parameter
+  download-plc
+  beremiz-runtime
+#  beremiz-runtime-promise
+
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
+offline = true
+extends = {{ template_monitor }}
+
+
+[download-plc]
+recipe = plone.recipe.command
+update-command = ${:command}
+url = ${instance-parameter:configuration.runtime_plc_url}
+md5sum = ${instance-parameter:configuration.runtime_plc_md5sum}
+# XXX:  do check md5sum !
+command = 
+  wget ${:url} -O plc.tgz
+  tar zxvf plc.tgz 
+
+[instance-parameter]
+recipe = slapos.cookbook:slapconfiguration
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+configuration.runtime_plc_url =
+configuration.runtime_plc_md5sum =
+# XXX: we can get rid of plc_name in future
+configuration.runtime_plc_name = 
+configuration.autostart = 1
+configuration.interface = 0.0.0.0
+# XXX: randomly generated one on slap's interface?
+configuration.port = 61248
+configuration.plc_runtime_path = ${directory:home}/${:configuration.runtime_plc_name}
+
+# Create all needed directories, depending on your needs
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc
+var = ${:home}/var
+script = ${:etc}/run/
+service = ${:etc}/service
+log = ${:var}/log
+
+[beremiz-runtime]
+logfile = ${directory:log}/beremiz-runtime.log
+recipe = slapos.cookbook:wrapper
+command-line =
+    {{ buildout['bin-directory'] }}/pythonwitheggs {{ buildout['directory'] }}/src/beremiz/Beremiz_service.py -a ${instance-parameter:configuration.autostart} -p ${instance-parameter:configuration.port} -i ${instance-parameter:configuration.interface} -x 1 ${directory:home}/${instance-parameter:configuration.runtime_plc_name}
+wrapper-path = ${directory:service}/beremiz-runtime
+
+[beremiz-runtime-promise]
+<= monitor-promise-base
+module = check_port_listening
+name = beremiz-runtime.py
+config-hostname= ${instance-parameter:configuration.interface}
+config-port    = ${instance-parameter:configuration.port}
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+port = ${instance-parameter:configuration.port}
+interface = ${instance-parameter:configuration.interface}

software/beremiz-runtime/software.cfg

+# the default SlapOs profile which setup a modbus server instance usually over OSIE's coupler
+[buildout]
+allow-picked-versions = true
+extensions = mr.developer
+auto-checkout = beremiz
+extends =
+  buildout.hash.cfg
+  https://lab.nexedi.com/nexedi/slapos/raw/master/stack/monitor/buildout.cfg
+  https://lab.nexedi.com/nexedi/slapos/raw/master/stack/slapos.cfg
+  https://lab.nexedi.com/nexedi/slapos/raw/master/component/python3/buildout.cfg
+
+parts = 
+  python-interpreter
+  slapos-cookbook
+  instance-profile
+
+# fix for pypi: https://mail.python.org/pipermail/distutils-sig/2017-October/031712.html
+index = https://pypi.python.org/simple/
+
+[sources]
+beremiz = git https://github.com/beremiz/beremiz.git branch=default
+
+[beremiz]
+recipe  = zc.recipe.egg:develop
+egg     = beremiz
+setup   = ${buildout:directory}/src/beremiz
+
+[Twisted]
+recipe  = zc.recipe.egg:custom
+egg     =  Twisted
+setup-eggs =
+    six
+    pathlib
+
+[python-interpreter]
+recipe = zc.recipe.egg
+interpreter = pythonwitheggs
+eggs = click
+       prompt_toolkit
+       pygments
+       bitarray
+       future
+       six
+       Pyro
+       zeroconf-py2compat
+       pathlib
+       Nevow
+       msgpack
+       autobahn
+       ${beremiz:egg}
+       ${Twisted:egg}
+
+[instance-profile]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/${:filename}
+mode = 0644
+rendered = ${buildout:directory}/instance.cfg
+extensions = jinja2.ext.do
+context =
+  section buildout  buildout
+  raw template_monitor ${monitor2-template:rendered}
+# md5sum is fetched from buildout.hash.cfg and can be recalculated automatically by
+# calling update-hash
+
+[versions]
+Twisted = 20.3.0
+attrs = 19.2.0
+Automat = 0.3.0
+zope.interface = 4.4.2
+Nevow = 0.14.5
+PyHamcrest = 2.0.2
+Pygments = 2.9.0
+Pyro = 3.16
+bitarray = 2.1.3
+constantly = 15.1.0
+future = 0.18.2
+hyperlink = 21.0.0
+incremental = 21.3.0
+pathlib = 1.0.1
+prompt-toolkit = 3.0.19
+zeroconf-py2compat = 0.19.10
+
+# Required by:
+# Automat==0.3.0
+characteristic = 14.3.0
+
+# Required by:
+# zeroconf-py2compat==0.19.10
+ifcfg = 0.21
+
+# Required by:
+# hyperlink==21.0.0
+typing = 3.10.0.0
+
+autobahn = 19.11.2
+txaio = 18.8.1
+idna = 2.10
+mr.developer = 2.0.1

software/caddy-frontend/buildout.hash.cfg

@@ -62,7 +62,7 @@ md5sum = 975177dedf677d24e14cede5d13187ce
 
 [template-trafficserver-records-config]
 _update_hash_filename_ = templates/trafficserver/records.config.jinja2
-md5sum = 88a2db868720009f6092843784b06611
+md5sum = e87238c53d080ef9ef90040e57bc1395
 
 [template-trafficserver-storage-config]
 _update_hash_filename_ = templates/trafficserver/storage.config.jinja2

software/caddy-frontend/templates/trafficserver/records.config.jinja2

@@ -16,6 +16,8 @@ CONFIG proxy.config.output.logfile STRING traffic.out
 CONFIG proxy.config.admin.user_id STRING {{ '#%s' % os_module.geteuid() }}
 LOCAL proxy.local.incoming_ip_to_bind STRING {{ ats_configuration['local-ip'] }}
 CONFIG proxy.config.log.logfile_dir STRING {{ ats_directory['log'] }}
+# Never change Server header
+CONFIG proxy.config.http.response_server_enabled INT 0
 # Implement RFC 5861 with core
 CONFIG proxy.config.http.cache.open_write_fail_action INT 2
 CONFIG proxy.config.body_factory.template_sets_dir STRING  {{ ats_configuration['templates-dir'] }}

software/caddy-frontend/test/test.py

@@ -485,6 +485,9 @@ def fakeHTTPResult(domain, path, port=HTTP_PORT,
 class TestHandler(BaseHTTPRequestHandler):
   identification = None
   configuration = {}
+  # override Server header response
+  server_version = "TestBackend"
+  sys_version = ""
 
   def log_message(self, *args):
     if os.environ.get('SLAPOS_TEST_DEBUG'):
@@ -843,6 +846,19 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     except Exception as e:
       self.fail(e)
 
+  def assertResponseHeaders(self, result):
+    headers = result.headers.copy()
+    self.assertKeyWithPop('Date', headers)
+    # drop vary-keys
+    headers.pop('Connection', None)
+    headers.pop('Content-Length', None)
+    headers.pop('Keep-Alive', None)
+    headers.pop('Transfer-Encoding', None)
+
+    self.assertEqual('TestBackend', headers.pop('Server', ''))
+
+    return headers
+
   def assertLogAccessUrlWithPop(self, parameter_dict):
     log_access_url = parameter_dict.pop('log-access-url')
 
@@ -3612,18 +3628,9 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
     self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
 
-    headers = result.headers.copy()
-
-    self.assertKeyWithPop('Server', headers)
-    self.assertKeyWithPop('Date', headers)
+    headers = self.assertResponseHeaders(result)
     self.assertKeyWithPop('Age', headers)
 
-    # drop keys appearing randomly in headers
-    headers.pop('Transfer-Encoding', None)
-    headers.pop('Content-Length', None)
-    headers.pop('Connection', None)
-    headers.pop('Keep-Alive', None)
-
     self.assertEqual(
       {
         'Content-type': 'application/json',
@@ -3654,18 +3661,9 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
     self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
 
-    headers = result.headers.copy()
+    headers = self.assertResponseHeaders(result)
 
-    self.assertKeyWithPop('Server', headers)
-    self.assertKeyWithPop('Date', headers)
     self.assertKeyWithPop('Age', headers)
-
-    # drop keys appearing randomly in headers
-    headers.pop('Transfer-Encoding', None)
-    headers.pop('Content-Length', None)
-    headers.pop('Connection', None)
-    headers.pop('Keep-Alive', None)
-
     self.assertEqual(
       {
         'Content-type': 'application/json',
@@ -3712,18 +3710,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
     self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
 
-    headers = result.headers.copy()
-
-    self.assertKeyWithPop('Server', headers)
-    self.assertKeyWithPop('Date', headers)
+    headers = self.assertResponseHeaders(result)
     self.assertKeyWithPop('Age', headers)
-
-    # drop keys appearing randomly in headers
-    headers.pop('Transfer-Encoding', None)
-    headers.pop('Content-Length', None)
-    headers.pop('Connection', None)
-    headers.pop('Keep-Alive', None)
-
     self.assertEqual(
       {
         'Content-type': 'application/json',
@@ -3743,7 +3731,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertEqual(httplib.OK, result.status_code)
     self.assertEqualResultJson(result, 'Path', '/HTTPS/test')
 
-    headers = result.headers.copy()
+    self.assertResponseHeaders(result)
 
     result = fakeHTTPSResult(
       parameter_dict['domain'],
@@ -3753,8 +3741,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
     self.assertEqual(httplib.OK, result.status_code)
     self.assertEqualResultJson(result, 'Path', '/HTTP/test')
-
-    headers = result.headers.copy()
+    self.assertResponseHeaders(result)
 
   def test_enable_cache(self):
     parameter_dict = self.assertSlaveBase('enable_cache')
@@ -3771,18 +3758,10 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
     self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
 
-    headers = result.headers.copy()
+    headers = self.assertResponseHeaders(result)
 
-    self.assertKeyWithPop('Server', headers)
-    self.assertKeyWithPop('Date', headers)
     self.assertKeyWithPop('Age', headers)
 
-    # drop keys appearing randomly in headers
-    headers.pop('Transfer-Encoding', None)
-    headers.pop('Content-Length', None)
-    headers.pop('Connection', None)
-    headers.pop('Keep-Alive', None)
-
     self.assertEqual(
       {
         'Content-type': 'application/json',
@@ -3973,18 +3952,10 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
     self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
 
-    headers = result.headers.copy()
+    headers = self.assertResponseHeaders(result)
 
-    self.assertKeyWithPop('Server', headers)
-    self.assertKeyWithPop('Date', headers)
     self.assertKeyWithPop('Age', headers)
 
-    # drop keys appearing randomly in headers
-    headers.pop('Transfer-Encoding', None)
-    headers.pop('Content-Length', None)
-    headers.pop('Connection', None)
-    headers.pop('Keep-Alive', None)
-
     self.assertEqual(
       {
         'Content-type': 'application/json',
@@ -4118,18 +4089,10 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
-    headers = result.headers.copy()
+    headers = self.assertResponseHeaders(result)
 
-    self.assertKeyWithPop('Server', headers)
-    self.assertKeyWithPop('Date', headers)
     self.assertKeyWithPop('Age', headers)
 
-    # drop keys appearing randomly in headers
-    headers.pop('Transfer-Encoding', None)
-    headers.pop('Content-Length', None)
-    headers.pop('Connection', None)
-    headers.pop('Keep-Alive', None)
-
     self.assertEqual(
       {
         'Content-type': 'application/json',
@@ -4165,18 +4128,10 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
-    headers = result.headers.copy()
+    headers = self.assertResponseHeaders(result)
 
-    self.assertKeyWithPop('Server', headers)
-    self.assertKeyWithPop('Date', headers)
     self.assertKeyWithPop('Age', headers)
 
-    # drop keys appearing randomly in headers
-    headers.pop('Transfer-Encoding', None)
-    headers.pop('Content-Length', None)
-    headers.pop('Connection', None)
-    headers.pop('Keep-Alive', None)
-
     self.assertEqual(
       {
         'Content-type': 'application/json',
@@ -4206,16 +4161,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
-    headers = result.headers.copy()
-
-    self.assertKeyWithPop('Server', headers)
-    self.assertKeyWithPop('Date', headers)
-
-    # drop vary-keys
-    headers.pop('Content-Length', None)
-    headers.pop('Transfer-Encoding', None)
-    headers.pop('Connection', None)
-    headers.pop('Keep-Alive', None)
+    headers = self.assertResponseHeaders(result)
 
     self.assertEqual(
       {
@@ -4240,17 +4186,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
-    headers = result.headers.copy()
-
-    self.assertKeyWithPop('Server', headers)
-    self.assertKeyWithPop('Date', headers)
-
-    # drop vary-keys
-    headers.pop('Content-Length', None)
-    headers.pop('Transfer-Encoding', None)
-    headers.pop('Connection', None)
-    headers.pop('Keep-Alive', None)
-
+    headers = self.assertResponseHeaders(result)
     self.assertEqual(
       {
         'Content-type': 'application/json',
@@ -5017,17 +4953,7 @@ class TestSlaveGlobalDisableHttp2(TestSlave):
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
-    headers = result.headers.copy()
-
-    self.assertKeyWithPop('Server', headers)
-    self.assertKeyWithPop('Date', headers)
-
-    # drop vary-keys
-    headers.pop('Content-Length', None)
-    headers.pop('Transfer-Encoding', None)
-    headers.pop('Connection', None)
-    headers.pop('Keep-Alive', None)
-
+    headers = self.assertResponseHeaders(result)
     self.assertEqual(
       {
         'Content-type': 'application/json',

software/erp5/instance-mariadb-schema.json

@@ -111,6 +111,16 @@
       "default": false,
       "type": "boolean"
     },
+    "character-set-server": {
+      "description": "The server default character set",
+      "default": "utf8mb4",
+      "type": "string"
+    },
+    "collation-server": {
+      "description": "The server default collation",
+      "default": "utf8mb4_general_ci",
+      "type": "string"
+    },
     "ssl": {
       "description": "Enable and define SSL support for network connections",
       "default": {},

software/galene/software.cfg

@@ -25,7 +25,7 @@ eggs    =
 <= go-git-package
 go.importpath = lab.nexedi.com/nexedi/galene
 repository = https://lab.nexedi.com/nexedi/galene.git
-revision = galene-0.4
+revision = galene-0.4.4
 
 [gowork]
 install =

software/kvm/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = 087f226ba90928dcc5a722d7008c867a
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = d949e6100fd4e01819cc2867a65c6e65
+md5sum = baa3ee5b653731124bfc2ac2fa835787
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in

software/kvm/instance-kvm.cfg.jinja2

@@ -879,6 +879,7 @@ ipv6-network-info =
 
 
 {% if use_tap and slap_configuration.get('tap-ipv4-addr') -%}
+{%   do part_list.append('network-config-ipv4') %}
 [network-config-ipv4]
 recipe = plone.recipe.command
 filename = netconfig.sh

software/kvm/test/test.py

@@ -443,6 +443,12 @@ class TestAccessDefaultBootstrap(MonitorAccessMixin, InstanceTestCase):
       result.status_code
     )
     self.assertIn('<title>noVNC</title>', result.text)
+    # check that expected files to configure the VM are exposed by the instance
+    self.assertEqual(
+      ['delDefaultIface', 'netconfig.sh'],
+      sorted(os.listdir(os.path.join(
+        self.computer_partition_root_path, 'srv', 'public')))
+    )
 
 
 @skipUnlessKvm

software/neoppod/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 2bd1779425b7561682c0de5496d808ed
 
 [root-common]
 filename = root-common.cfg.in
-md5sum = c13b4f1a5aa526a8d3f8e02bf6baf785
+md5sum = ae00507d9e69209a0babd725cf6be536
 
 [instance-neo-admin]
 filename = instance-neo-admin.cfg.in
@@ -38,7 +38,7 @@ md5sum = a8939dcf440126a30db661b5a0a1bfb7
 
 [template-neo]
 filename = instance.cfg.in
-md5sum = 9e63e16eda75e73ad4ffb50afde0505d
+md5sum = 3c7770c03c51ee12d5a94f0924e98990
 
 [cluster]
 filename = cluster.cfg.in

software/neoppod/instance.cfg.in

@@ -9,7 +9,7 @@ rendered = ${buildout:parts-directory}/${:_buildout_section_name_}.cfg
 template = {{ cluster }}
 extra-context =
     import re re
-    import urlparse urlparse
+    import urllib_parse six.moves.urllib.parse
 import-list =
     rawfile root_common {{ root_common }}
 

software/neoppod/root-common.cfg.in

@@ -2,13 +2,13 @@
 {% set part_list = [] -%}
 
 {% set sla_dict = {} -%}
-{% for sla, ref_list in slapparameter_dict.get('sla-dict', {}).iteritems() -%}
+{% for sla, ref_list in six.iteritems(slapparameter_dict.get('sla-dict', {})) -%}
 {%   do sla_dict.update(dict.fromkeys(ref_list, sla)) -%}
 {% endfor -%}
 
 {% macro sla(name, required=False) -%}
 {% if required or name in sla_dict -%}
-{%  for k, (v,) in urlparse.parse_qs(sla_dict.pop(name), strict_parsing=1).iteritems() -%}
+{%  for k, (v,) in six.iteritems(urllib_parse.parse_qs(sla_dict.pop(name), strict_parsing=1)) -%}
 sla-{{ k }} = {{ v }}
 {%  endfor -%}
 {% else -%}
@@ -133,7 +133,7 @@ config-monitor-port = {{ dumps(port) }}
 {%-     endif %}
 config-monitor = {{ dumps(parameter_dict.get('monitor', {})) }}
 {%-   endif %}
-{%-   for k, v in node.iteritems() %}
+{%-   for k, v in six.iteritems(node) %}
 config-{{ k }} = {{ dumps(v) }}
 {%-   endfor %}
 {{ sla(section_id) }}

software/ors-amarisoft/buildout.hash.cfg

@@ -20,23 +20,23 @@ md5sum = 6ea4fa210a91c15278c847a809de5991
 
 [template-lte-enb-epc]
 _update_hash_filename_ = instance-enb-epc.jinja2.cfg
-md5sum = cf6c400d9fa5b0942f9be7145f77b8de
+md5sum = 1358f8e0a4cecefe4cdf389b69067f34
 
 [template-lte-enb]
 _update_hash_filename_ = instance-enb.jinja2.cfg
-md5sum = fe249168a3f50b0efe6aeae39afb03ae
+md5sum = e4c224da723ad56091f27ed5c0b0bbca
 
 [template-lte-gnb-epc]
 _update_hash_filename_ = instance-gnb-epc.jinja2.cfg
-md5sum = f94c3e2f714629d9e1fc9b2f7c8eb586
+md5sum = b9a58fa4037d32fc1dc4f5ef89e6211a
 
 [template-lte-gnb]
 _update_hash_filename_ = instance-gnb.jinja2.cfg
-md5sum = 6deb38b3de44f6e0a43b49fd13f0f072
+md5sum = 0b74993990a0dfa3c6429dc4ac716826
 
 [template-lte-epc]
 _update_hash_filename_ = instance-epc.jinja2.cfg
-md5sum = 089f62e736cdc620eafc2c47b050fe13
+md5sum = 35556ebd82be804584482a7fe5eff0be
 
 [ue_db.jinja2.cfg]
 filename = config/ue_db.jinja2.cfg

software/ors-amarisoft/instance-enb-epc.jinja2.cfg

@@ -52,6 +52,12 @@ return = monitor-base-url
 name = EPC
 software-type = epc
 config-name = epc
+{% if slapparameter_dict.get("mme_config_link", None) %}
+config-mme_config_link = {{ dumps(slapparameter_dict["mme_config_link"]) }}
+{% endif %}
+{% if slapparameter_dict.get("mme_config_version", None) %}
+config-mme_config_version = {{ dumps(slapparameter_dict["mme_config_version"]) }}
+{% endif %}
 
 [lte-enb-request]
 <= request-common-base
@@ -76,6 +82,12 @@ config-mme_addr = {{ dumps(slapparameter_dict["mme_addr"]) }}
 {% if slapparameter_dict.get("enb_id", None) %}
 config-enb_id = {{ dumps(slapparameter_dict["enb_id"]) }}
 {% endif %}
+{% if slapparameter_dict.get("enb_config_link", None) %}
+config-enb_config_link = {{ dumps(slapparameter_dict["enb_config_link"]) }}
+{% endif %}
+{% if slapparameter_dict.get("enb_config_version", None) %}
+config-enb_config_version = {{ dumps(slapparameter_dict["enb_config_version"]) }}
+{% endif %}
 
 [monitor-base-url-dict]
 lte-epc-request =  ${lte-epc-request:connection-monitor-base-url}

software/ors-amarisoft/instance-enb.jinja2.cfg

@@ -60,6 +60,14 @@ extensions = jinja2.ext.do
 context =
   section directory directory
 
+{% if slapparameter_dict.get("enb_config_link", None) %}
+[enb-config-dl]
+recipe = slapos.recipe.build:download
+url = {{ slapparameter_dict.get("enb_config_link") }}
+version = {{ slapparameter_dict.get("enb_config_version") }}
+offline = false
+{% endif %}
+
 [lte-enb-sh-wrapper]
 recipe = slapos.recipe.template:jinja2
 rendered = ${directory:bin}/${:_buildout_section_name_}
@@ -98,7 +106,11 @@ context =
 
 [lte-enb-config]
 <= config-base
+{% if slapparameter_dict.get("enb_config_link", None) %}
+template = ${enb-config-dl:target}
+{% else %}
 template = {{ enb_template }}
+{% endif %}
 rendered = ${directory:etc}/enb.cfg
 
 [publish-connection-information]

software/ors-amarisoft/instance-epc.jinja2.cfg

@@ -58,6 +58,14 @@ extensions = jinja2.ext.do
 context =
   section directory directory
 
+{% if slapparameter_dict.get("mme_config_link", None) %}
+[mme-config-dl]
+recipe = slapos.recipe.build:download
+url = {{ slapparameter_dict.get("mme_config_link") }}
+version = {{ slapparameter_dict.get("mme_config_version") }}
+offline = false
+{% endif %}
+
 ### IMS
 [lte-ims-service]
 recipe = slapos.cookbook:wrapper
@@ -122,7 +130,11 @@ rendered = ${directory:etc}/ims.cfg
 
 [lte-mme-config]
 <= config-base
+{% if slapparameter_dict.get("mme_config_link", None) %}
+template = ${mme-config-dl:target}
+{% else %}
 template = {{ mme_template }}
+{% endif %}
 rendered = ${directory:etc}/mme.cfg
 context =
   section directory directory

software/ors-amarisoft/instance-gnb-epc.jinja2.cfg

@@ -52,6 +52,12 @@ return = monitor-base-url
 name = EPC
 software-type = epc
 config-name = epc
+{% if slapparameter_dict.get("mme_config_link", None) %}
+config-mme_config_link = {{ dumps(slapparameter_dict["mme_config_link"]) }}
+{% endif %}
+{% if slapparameter_dict.get("mme_config_version", None) %}
+config-mme_config_version = {{ dumps(slapparameter_dict["mme_config_version"]) }}
+{% endif %}
 
 [lte-gnb-request]
 <= request-common-base
@@ -79,6 +85,12 @@ config-mme_addr = {{ dumps(slapparameter_dict["mme_addr"]) }}
 {% if slapparameter_dict.get("enb_id", None) %}
 config-enb_id = {{ dumps(slapparameter_dict["enb_id"]) }}
 {% endif %}
+{% if slapparameter_dict.get("gnb_config_link", None) %}
+config-gnb_config_link = {{ dumps(slapparameter_dict["gnb_config_link"]) }}
+{% endif %}
+{% if slapparameter_dict.get("gnb_config_version", None) %}
+config-gnb_config_version = {{ dumps(slapparameter_dict["gnb_config_version"]) }}
+{% endif %}
 
 [monitor-base-url-dict]
 lte-epc-request =  ${lte-epc-request:connection-monitor-base-url}

software/ors-amarisoft/instance-gnb.jinja2.cfg

@@ -60,6 +60,14 @@ extensions = jinja2.ext.do
 context =
   section directory directory
 
+{% if slapparameter_dict.get("gnb_config_link", None) %}
+[gnb-config-dl]
+recipe = slapos.recipe.build:download
+url = {{ slapparameter_dict.get("gnb_config_link") }}
+version = {{ slapparameter_dict.get("gnb_config_version") }}
+offline = false
+{% endif %}
+
 [lte-enb-sh-wrapper]
 recipe = slapos.recipe.template:jinja2
 rendered = ${directory:bin}/${:_buildout_section_name_}
@@ -98,7 +106,11 @@ context =
 
 [lte-gnb-config]
 <= config-base
+{% if slapparameter_dict.get("gnb_config_link", None) %}
+template = ${gnb-config-dl:target}
+{% else %}
 template = {{ gnb_template }}
+{% endif %}
 rendered = ${directory:etc}/gnb.cfg
 
 [publish-connection-information]

software/repman/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [instance.cfg]
 filename = instance.cfg.in
-md5sum = af2fc4a7a0f782fed2cb1112ef3cb397
+md5sum = 1c9c4c254f371337fe08b7922901f8a9
 
 [instance-repman.cfg]
 _update_hash_filename_ = instance-repman.cfg.jinja2.in
@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
 
 [template-mariadb.cfg]
 _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
-md5sum = 938f1e8087395757f5ae861d20500658
+md5sum = aa25693b324ce5acc00f29b96a2516c0
 
 [template-my-cnf]
 _update_hash_filename_ = templates/my.cnf.in

software/repman/instance-mariadb.cfg.jinja2.in

@@ -214,16 +214,10 @@ wait-for-files =
 {{ supervisord_lib.supervisord_program("mariadb", maradb_program_dict) }}
 {% do part_list.append("supervisord-mariadb") %}
 
-[odbc-ini-text]
-text = {{ dumps(base64.b64encode( slapparameter_dict.get('odbc-ini', '').encode() )) }}
-
 [{{ section('odbc-ini') }}]
-< = jinja2-template-base
-rendered = ${directory:etc}/odbc.ini
-template = inline:{% raw -%}
-  {{ parameter_dict['text'].decode('base64') }}
-  {%- endraw %}
-context = section parameter_dict odbc-ini-text
+recipe = slapos.recipe.template
+output = ${directory:etc}/odbc.ini
+inline = {{ dumps(slapparameter_dict.get('odbc-ini', '')) }}
 
 [{{ section('logrotate-entry-mariadb') }}]
 < = logrotate-entry-base

software/repman/instance.cfg.in

@@ -108,4 +108,3 @@ template = {{ template_mariadb }}
 filename = instance-mariadb.cfg
 extra-context =
     section parameter_dict template-mariadb-parameters
-    import base64 base64

software/theia/buildout.hash.cfg

@@ -23,7 +23,7 @@ md5sum = f2f01a47d98a980177dc1755e618bbb7
 
 [instance-import]
 _update_hash_filename_ = instance-import.cfg.jinja.in
-md5sum = b0a2c2b3d59fd6c8ba76c634b83a1ba2
+md5sum = 74b9b46769a25add50f9f63365de58ad
 
 [instance-export]
 _update_hash_filename_ = instance-export.cfg.jinja.in

software/theia/instance-import.cfg.jinja.in

@@ -64,15 +64,14 @@ seed = Import {{ root_title }}
 # in post-notification-run:output and in importer:wrapper.
 
 [post-notification-run]
-recipe = slapos.recipe.template:jinja2
-rendered = $${directory:bin}/post-notification-run-script
-output = $${:rendered}
-mode = 0700
-template =
-  inline:#!${software-info:bash}
-  # Do nothing because the backup signature will
-  # be verified by the import script itself
-  exit 0
+# Do nothing because the backup signature will
+# be verified by the import script itself
+recipe = slapos.recipe.build
+output = $${directory:bin}/post-notification-run-script
+location = $${:output}
+install =
+  import os
+  os.symlink("/bin/true", location)
 
 [importer]
 wrapper = $${theia-import-script:rendered}

stack/erp5/buildout.cfg

@@ -584,7 +584,7 @@ extra-paths =
 patch-binary = ${patch:location}/bin/patch
 PyPDF2-patches = ${:_profile_base_location_}/../../component/egg-patch/PyPDF2/0001-Custom-implementation-of-warnings.formatwarning-remo.patch#d25bb0f5dde7f3337a0a50c2f986f5c8
 PyPDF2-patch-options = -p1
-Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic.patch#1d9a56e9af4371f5b6951ebf217a15d7
+Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic-2.13.12.patch#1d9a56e9af4371f5b6951ebf217a15d7
 Acquisition-patch-options = -p1
 python-magic-patches = ${:_profile_base_location_}/../../component/egg-patch/python_magic/magic.patch#de0839bffac17801e39b60873a6c2068
 python-magic-patch-options = -p1

stack/erp5/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = d10b8e35b02b5391cf46bf0c7dbb1196
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = f553aa7d6596dcf98e7e61bfb6bd81c7
+md5sum = b664a2221077db5db498422b6c90f015
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
@@ -34,15 +34,15 @@ md5sum = cfe4696a67bf4886a5d8252a5274a941
 
 [template-zope-conf]
 filename = zope.conf.in
-md5sum = 70f30111e137d158aeca3d67c4abf643
+md5sum = ce8d03a1b4c1a9e5085ec54ea2744007
 
 [site-zcml]
 filename = site.zcml
-md5sum = d32417746fcf671d4e86a70379815039
+md5sum = 43556e5bca8336dd543ae8068512aa6d
 
 [template-my-cnf]
 filename = my.cnf.in
-md5sum = 1de449e8c0c4a85c5ce2b447785b7654
+md5sum = c0bde08ec6bd6d333315a15026266b65
 
 [template-mariadb-initial-setup]
 filename = mariadb_initial_setup.sql.in
@@ -66,35 +66,35 @@ md5sum = 0969fbb25b05c02ef3c2d437b2f4e1a0
 
 [template-run-zelenium]
 filename = run-zelenium-test.py.in
-md5sum = 7a14019abf48ca100eb94d9add20f5ae
+md5sum = 274365ebbade26558ca4836837e781aa
 
 [template]
 filename = instance.cfg.in
-md5sum = 0444d646607f8643e987bb62d1cbbc1d
+md5sum = 165b0a50672138ee4a3fbe9239af11ba
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = cb41e15a1585973896a7645dc2ae3a5c
+md5sum = f5a1661449c9681b3de7d4af645124ba
 
 [template-zeo]
 filename = instance-zeo.cfg.in
-md5sum = 1f33f3b93da32b34e2fd11471648835d
+md5sum = 3c59315a8f102a970dc54ded85df735a
 
 [template-zodb-base]
 filename = instance-zodb-base.cfg.in
-md5sum = bc821f9f9696953b10a03ad7b59a1936
+md5sum = 0ac4b74436f554cd677f19275d18d880
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = 5520e81f1fff3dc26568a614b595da37
+md5sum = 06265195ca29805696873b6f8a57a5fd
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = cbcbe2daf3a51dfab50446a18c0be76e
+md5sum = 8b4d6c29e9c5d8fb2e50e6ac96906d97
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in
-md5sum = 9d61e05c8578e0f17e349603ccaaf52c
+md5sum = d2d98ed3fafce764991b72371e3e09d5
 
 [template-rsyslogd-cfg]
 filename = rsyslogd.cfg.in

stack/erp5/haproxy.cfg.in

@@ -150,7 +150,7 @@ defaults
 {% set family_path_routing_dict = parameter_dict['family-path-routing-dict'] %}
 {% set path_routing_list = parameter_dict['path-routing-list'] %}
 
-{% for name, (port, _, certificate_authentication, backend_list) in sorted(parameter_dict['backend-dict'].iteritems()) -%}
+{% for name, (port, _, certificate_authentication, backend_list) in sorted(six.iteritems(parameter_dict['backend-dict'])) -%}
 listen family_{{ name }}
 {%-  if parameter_dict.get('ca-cert') -%}
 {%-    set ssl_auth = ' ca-file ' ~ parameter_dict['ca-cert'] ~ ' verify' ~ ( ' required' if certificate_authentication else ' optional' ) ~ ' crl-file ' ~ parameter_dict['crl'] %}
@@ -192,7 +192,7 @@ listen family_{{ name }}
 {% endfor %}
 
 
-{% for (ip, port), (_, backend_dict) in sorted(parameter_dict['zope-virtualhost-monster-backend-dict'].iteritems()) -%}
+{% for (ip, port), (_, backend_dict) in sorted(six.iteritems(parameter_dict['zope-virtualhost-monster-backend-dict'])) -%}
 {%   set group_name = 'testrunner_' ~ loop.index0 %}
 frontend frontend_{{ group_name }}
   bind {{ ip }}:{{ port }} {{ bind_ssl_crt }}
@@ -211,6 +211,6 @@ frontend frontend_{{ group_name }}
 backend backend_{{ group_name }}_{{ name }}
   http-request replace-path ^/{{ name }}(.*) /VirtualHostBase/https/{{ ip }}:{{ port }}/VirtualHostRoot/_vh_{{ name }}\1
   timeout server 8h
-  server {{ name }} {{ urlparse.urlparse(url).netloc }}
+  server {{ name }} {{ urllib_parse.urlparse(url).netloc }}
 {%-  endfor %}
 {% endfor %}

stack/erp5/instance-balancer.cfg.in

@@ -38,7 +38,7 @@ mode = 644
 
 {% set frontend_caucase_url_hash_list = [] -%}
 {% for frontend_caucase_url in frontend_caucase_url_list -%}
-{%   set hash = hashlib.md5(frontend_caucase_url).hexdigest() -%}
+{%   set hash = hashlib.md5(six.ensure_binary(frontend_caucase_url)).hexdigest() -%}
 {%   do frontend_caucase_url_hash_list.append(hash) -%}
 {%   set data_dir = '${directory:client-cert-ca}/%s' % hash -%}
 {{   caucase.updater(
@@ -120,9 +120,9 @@ update-command = ${:command}
 {% set haproxy_dict = {} -%}
 {% set zope_virtualhost_monster_backend_dict = {} %}
 {% set test_runner_url_dict = {} %} {# family_name => list of URLs #}
-{% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%}
+{% set next_port = functools.partial(next, itertools.count(slapparameter_dict['tcpv4-port'])) -%}
 {% for family_name, parameter_id_list in sorted(
-  slapparameter_dict['zope-family-dict'].iteritems()) -%}
+  six.iteritems(slapparameter_dict['zope-family-dict'])) -%}
 {%   set zope_family_address_list = [] -%}
 {%   set ssl_authentication = slapparameter_dict['ssl-authentication-dict'].get(family_name, False) -%}
 {%   set has_webdav = [] -%}
@@ -205,7 +205,7 @@ template = {{ parameter_dict['template-haproxy-cfg'] }}
 rendered = ${directory:etc}/haproxy.cfg
 context =
   section parameter_dict haproxy-cfg-parameter-dict
-  import urlparse urlparse
+  import urllib_parse six.moves.urllib.parse
 extensions = jinja2.ext.do
 
 [haproxy-reload]
@@ -306,7 +306,7 @@ certificate-and-key = ${directory:etc}/certificate-and-key-generated.pem
 promise = check_socket_listening
 name = haproxy.py
 config-host = {{ ipv4 }}
-config-port = {{ haproxy_dict.values()[0][0] }}
+config-port = {{ next(six.itervalues(haproxy_dict))[0] }}
 
 [{{ section('publish') }}]
 recipe = slapos.cookbook:publish.serialised

stack/erp5/instance-erp5.cfg.in

@@ -28,7 +28,7 @@
 {% set partition_thread_count_list = [] -%}
 {% set zope_partition_dict = slapparameter_dict.get('zope-partition-dict', {'1': {}}) -%}
 {% set zope_family_override_dict = slapparameter_dict.get('family-override', {}) -%}
-{% for zope_parameter_dict in zope_partition_dict.values() -%}
+{% for zope_parameter_dict in six.itervalues(zope_partition_dict) -%}
 {#   Apply some zope_parameter_dict default values, to avoid duplication. -#}
 {%   do zope_parameter_dict.setdefault('thread-amount', 4) -%}
 {%   do zope_parameter_dict.setdefault('instance-count', 1) -%}
@@ -49,7 +49,7 @@ config-computer-memory-percent-threshold = {{ dumps(monitor_dict.get('computer-m
 name = {{ name }}
 software-type = {{ software_type }}
 return = {{ ' '.join(ret) }}
-{% for ret, publish in ret.iteritems() -%}
+{% for ret, publish in six.iteritems(ret) -%}
 {%   if publish -%}
 {%     do publish_dict.__setitem__(name ~ '-' ~ ret, '${' ~ section ~ ':connection-' ~ ret ~ '}') %}
 {%   endif -%}
@@ -58,10 +58,10 @@ return = {{ ' '.join(ret) }}
 {%   endif -%}
 {% endfor -%}
 {{ root_common.sla(name) }}
-{% for k, v in config.iteritems() -%}
+{% for k, v in six.iteritems(config) -%}
 config-{{ k }} = {{ dumps(v) }}
 {% endfor -%}
-{% for k, v in key_config.iteritems() -%}
+{% for k, v in six.iteritems(key_config) -%}
 config-{{ k }} = {{ '${' ~ v ~ '}' }}
 {% endfor -%}
 config-name = {{ name }}
@@ -177,7 +177,7 @@ connection-url = smtp://127.0.0.2:0/
 {% endfor -%}
 {% do assert(len(mountpoints) == len(zodb_dict)) -%}
 {% set neo = [] -%}
-{% for server_type, server_dict in storage_dict.iteritems() -%}
+{% for server_type, server_dict in six.iteritems(storage_dict) -%}
 {%   if server_type == 'neo' -%}
 {%     set ((name, server_dict),) = server_dict.items() -%}
 {%     do neo.append(server_dict.get('cluster')) -%}
@@ -196,7 +196,7 @@ connection-url = smtp://127.0.0.2:0/
 
 [request-zodb-base]
 config-zodb-dict = {{ dumps(zodb_dict) }}
-{% for server_type, server_dict in storage_dict.iteritems() -%}
+{% for server_type, server_dict in six.iteritems(storage_dict) -%}
 {%   if server_type == 'neo' -%}
 config-neo-cluster = ${publish-early:neo-cluster}
 config-neo-name = {{ server_dict.keys()[0] }}
@@ -272,7 +272,7 @@ software-type = zope
 {% set zope_backend_path_dict = {} -%}
 {% set ssl_authentication_dict = {} -%}
 {% set jupyter_zope_family_default = [] -%}
-{% for custom_name, zope_parameter_dict in zope_partition_dict.items() -%}
+{% for custom_name, zope_parameter_dict in six.iteritems(zope_partition_dict) -%}
 {%   set partition_name = 'zope-' ~ custom_name -%}
 {%   set section_name = 'request-' ~ partition_name -%}
 {%   set promise_software_url_section_name = 'promise-software-url' ~ partition_name -%}
@@ -461,7 +461,7 @@ recipe = slapos.cookbook:publish.serialised
 Pick any published hosts-dict, they are expected to be identical - and there is
 no way to check here.
 -#}
-hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-dict}' }}
+hosts-dict = {{ '${' ~ next(iter(zope_address_list_id_dict)) ~ ':connection-hosts-dict}' }}
 {% endif -%}
 {% for name, value in publish_dict.items() -%}
 {{   name }} = {{ value }}

stack/erp5/instance-mariadb.cfg.in

@@ -142,6 +142,8 @@ innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0
 innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }}
 innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }}
 relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
+character-set-server = {{ dumps(slapparameter_dict.get('character-set-server', 'utf8mb4')) }}
+collation-server = {{ dumps(slapparameter_dict.get('collation-server', 'utf8mb4_general_ci')) }}
 {% if incremental_backup_retention_days > -1 -%}
 binlog-path = ${directory:mariadb-backup-incremental}/binlog
 # XXX: binlog rotation happens along with other log's rotation
@@ -195,16 +197,11 @@ environ =
   {{ variable }}
   {%- endfor %}
 
-[odbc-ini-text]
-text = {{ dumps(slapparameter_dict.get('odbc-ini', '').encode('base64')) }}
 
 [{{ section('odbc-ini') }}]
-< = jinja2-template-base
-rendered = ${directory:etc}/odbc.ini
-template = inline:{% raw -%}
-  {{ parameter_dict['text'].decode('base64') }}
-  {%- endraw %}
-context = section parameter_dict odbc-ini-text
+recipe = slapos.recipe.template
+output = ${directory:etc}/odbc.ini
+inline = {{ dumps(slapparameter_dict.get('odbc-ini', '')) }}
 
 [{{ section('logrotate-entry-mariadb') }}]
 < = logrotate-entry-base

stack/erp5/instance-zeo.cfg.in

-{% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%}
+{% set ports = itertools.count(slapparameter_dict['tcpv4-port']) -%}
 {% set ipv4 = (ipv4_set | list)[0] -%}
 {% set backup_periodicity = slapparameter_dict.get('backup-periodicity', 'daily') -%}
 {% set part_list = [] -%}
@@ -19,13 +19,13 @@ ip = {{ ipv4 }}
 
 {% set known_tid_storage_identifier_dict = {} -%}
 {% set zodb_dict = {} -%}
-{% for name, zodb in slapparameter_dict['zodb-dict'].iteritems() -%}
+{% for name, zodb in six.iteritems(slapparameter_dict['zodb-dict']) -%}
 {%   do zodb_dict.setdefault(zodb.get('family', 'default').lower(), []).append((name, zodb)) -%}
 {% endfor -%}
 {% set tidstorage_port = slapparameter_dict['tcpv4-port'] + len(zodb_dict) -%}
-{% for family, zodb in zodb_dict.iteritems() -%}
+{% for family, zodb in six.iteritems(zodb_dict) -%}
 {%   set storage_list = [] -%}
-{%   set current_port = next_port() -%}
+{%   set current_port = next(ports) -%}
 {%   set known_tid_storage_identifier_host = (ipv4, current_port), -%}
 {%   for name, zodb in zodb -%}
 {%     do storage_dict.__setitem__(name, {'server': ipv4 ~ ':' ~ current_port, 'storage': name}) %}
@@ -140,7 +140,7 @@ input = inline: #!/bin/sh
   zodb_backup_directory="{{ default_backup_path }}"
   repozo="${tidstorage:repozo-binary}"
   EXIT_CODE=0
-  {% for family, zodb in zodb_dict.iteritems() -%}
+  {% for family, zodb in six.iteritems(zodb_dict) -%}
   {%   for name, zodb in zodb -%}
   {%   set zeo_section_name = 'zeo-' ~ family %}
   storage_name="{{ name }}"
@@ -209,7 +209,7 @@ tidstorage = {{ tidstorage_repozo_path }}
 
 [monitor-instance-parameter]
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
-monitor-httpd-port = {{ next_port() }}
+monitor-httpd-port = {{ next(ports) }}
 monitor-title = {{ slapparameter_dict['name'] }}
 password = {{ slapparameter_dict['monitor-passwd'] }}
 

stack/erp5/instance-zodb-base.cfg.in

@@ -4,7 +4,7 @@
 {% set zodb_dict = slapparameter_dict['zodb-dict'] -%}
 
 {% set zeo_dict = slapparameter_dict.get('zodb-zeo', {}) -%}
-{% for name, zodb in zodb_dict.iteritems() -%}
+{% for name, zodb in six.iteritems(zodb_dict) -%}
 {%   set storage_dict = zodb.setdefault('storage-dict', {}) -%}
 {%   if zodb['type'] == 'zeo' -%}
 {%     do storage_dict.update(zeo_dict.get(name, ())) -%}

stack/erp5/instance-zope.cfg.in

@@ -2,7 +2,7 @@
 {% set wsgi = slapparameter_dict['wsgi'] -%}
 {% set webdav = slapparameter_dict['webdav'] -%}
 {% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
-{% set next_port = itertools.count(slapparameter_dict['port-base']).next -%}
+{% set ports = itertools.count(slapparameter_dict['port-base']) -%}
 {% set site_id = slapparameter_dict['site-id'] -%}
 {% set instance_index_list = range(slapparameter_dict['instance-count']) -%}
 {% set node_id_base = slapparameter_dict['name'] -%}
@@ -36,14 +36,14 @@ partition. No more (undefined result), no less (IndexError).
     ('erp5-cloudooo', slapparameter_dict['cloudooo-url']),
     ('erp5-smtp', slapparameter_dict['smtp-url']),
   ) -%}
-{%   set parsed_url = urlparse.urlparse(url) -%}
+{%   set parsed_url = urllib_parse.urlparse(url) -%}
 {%   do port_dict.__setitem__(alias, parsed_url.port)  -%}
 {%   do publishable_hosts_dict.__setitem__(alias, parsed_url.hostname)  -%}
 {%- endfor %}
 {% for i, url in enumerate(slapparameter_dict['mysql-url-list']) -%}
 {%   do publishable_hosts_dict.__setitem__(
        'erp5-catalog-' ~ i,
-       urlparse.urlparse(url).hostname,
+       urllib_parse.urlparse(url).hostname,
      ) -%}
 {%- endfor %}
 {% do publishable_hosts_dict.update(slapparameter_dict['hosts-dict']) -%}
@@ -202,7 +202,7 @@ environment =
 command-line = '${userhosts-bin:rendered}' ${:wrapped-command-line}
 
 {# Hack to deploy SSL certs via instance parameters -#}
-{% for zodb in zodb_dict.itervalues() -%}
+{% for zodb in six.itervalues(zodb_dict) -%}
 {%   set storage_dict = zodb.setdefault('storage-dict', {}) -%}
 {%   if zodb['type'] == 'neo' and storage_dict.get('ssl', 1) -%}
 {%     for k, v in (('_ca', 'ca.crt'),
@@ -243,7 +243,7 @@ file-list = {{ parameter_dict['site-zcml'] }}
 [{{ section('zope-inituser') }}]
 < = jinja2-template-base
 rendered = ${directory:instance}/inituser
-template = inline:{{ slapparameter_dict['inituser-login'] }}:{SHA}{{ hashlib.sha1(slapparameter_dict['inituser-password']).digest().encode('base64').rstrip() }}
+template = inline:{{ slapparameter_dict['inituser-login'] }}:{SHA}{{ base64.b64encode(hashlib.sha1(slapparameter_dict['inituser-password'].encode('utf-8')).digest()) }}
 mode = 600
 once = ${:rendered}_done
 
@@ -253,7 +253,7 @@ site-id = {{ site_id }}
 {% if site_id -%}
 mysql-url = {{ slapparameter_dict['mysql-url-list'][0] }}
 inituser = {{ slapparameter_dict['inituser-login'] }}
-{%  set mysql = urlparse.urlsplit(slapparameter_dict['mysql-url-list'][0]) -%}
+{%  set mysql = urllib_parse.urlsplit(slapparameter_dict['mysql-url-list'][0]) -%}
 {%  set mysql_db = mysql.path.split('/')[1] -%}
 sql-connection-string = {{ '%s@erp5-catalog-0:%s %s %s' % (
     mysql_db, mysql.port, mysql.username, mysql.password) }}
@@ -264,7 +264,7 @@ home = ${buildout:directory}
 # We only want to change the hostname to 'erp5-cloudooo' if we use the internal
 # cloudooo. We plan to remove the ability to have an internal one, so this
 # heuristic is enough.
-{%  set cloudooo = urlparse.urlsplit(slapparameter_dict['cloudooo-url']) -%}
+{%  set cloudooo = urllib_parse.urlsplit(slapparameter_dict['cloudooo-url']) -%}
 cloudooo-url = {{ (cloudooo if cloudooo.port == None else
   cloudooo._replace(netloc='erp5-cloudooo:%s' % cloudooo.port)).geturl() }}
 
@@ -317,7 +317,7 @@ z2-log = ${directory:log}/{{ name }}-Z2.log
 node-id = {{ dumps(node_id_base ~ (node_id_index_format % index)) }}
 {% set log_list = [] -%}
 {% set import_set = set() -%}
-{% for db_name, zodb in zodb_dict.iteritems() -%}
+{% for db_name, zodb in six.iteritems(zodb_dict) -%}
 {%   do zodb.setdefault('pool-size', thread_amount) -%}
 {%   if zodb['type'] == 'neo' -%}
 {%     do import_set.add('neo.client') -%}
@@ -358,11 +358,17 @@ hash-existing-files =
   ${buildout:directory}/software_release/buildout.cfg
 
 [{{ section("promise-" ~ name) }}]
+{% if six.PY3 -%}
+# Disable the promise in Python 3. ERP5 is not compatible with Python 3 yet, so
+# the promise would always fail.
+recipe =
+{% else -%}
 <= monitor-promise-base
 promise = check_socket_listening
 name = {{ name }}.py
 config-host = {{ ipv4 }}
 config-port = {{ port }}
+{% endif -%}
 
 {% if use_ipv6 -%}
 [{{ zope_tunnel_section_name }}]
@@ -402,7 +408,7 @@ post = test ! -s {{ '${' ~ conf_parameter_name ~ ':pid-file}' }} || {{ bin_direc
 {% for i in instance_index_list -%}
 {{   zope(
        i,
-       next_port(),
+       next(ports),
        slapparameter_dict['longrequest-logger-timeout'],
        slapparameter_dict['longrequest-logger-interval'],
      ) }}
@@ -420,7 +426,7 @@ wrapper-path = ${buildout:bin-directory}/${:_buildout_section_name_}
 
 {% if test_runner_enabled and test_runner_node_count -%}
 {% for _ in range(test_runner_node_count) %}
-{%   do test_runner_address_list.append((ipv4, next_port())) %}
+{%   do test_runner_address_list.append((ipv4, next(ports))) %}
 {% endfor %}
 
 {% if selenium_server_configuration_dict -%}
@@ -454,7 +460,7 @@ wrapper-path = ${buildout:bin-directory}/runTestSuite
 
 {%   set connection_string_list = [] -%}
 {%   for url in slapparameter_dict['mysql-test-url-list'] -%}
-{%     set parsed_url = urlparse.urlparse(url) -%}
+{%     set parsed_url = urllib_parse.urlparse(url) -%}
 {%     do connection_string_list.append(
          '%s@%s:%s %s %s' % (
            parsed_url.path.lstrip('/'),
@@ -563,7 +569,7 @@ software-release-url = ${slap-connection:software-release-url}
 
 [monitor-instance-parameter]
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
-monitor-httpd-port = {{ next_port() }}
+monitor-httpd-port = {{ next(ports) }}
 monitor-title = {{ slapparameter_dict['name'] }}
 password = {{ slapparameter_dict['monitor-passwd'] }}
 

stack/erp5/instance.cfg.in

@@ -50,7 +50,7 @@ extra-context =
     key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
     key openssl_location :openssl-location
     import re re
-    import urlparse urlparse
+    import urllib_parse six.moves.urllib.parse
 import-list =
     file root_common context:root-common
     file caucase context:caucase-jinja2-library
@@ -80,6 +80,7 @@ extra-context =
     section parameter_dict dynamic-template-balancer-parameters
     import itertools itertools
     import hashlib hashlib
+    import functools functools
 import-list =
     file caucase context:caucase-jinja2-library
 
@@ -125,7 +126,8 @@ extra-context =
     key buildout_directory buildout:directory
     key root_common context:root-common
     section parameter_dict dynamic-template-zope-parameters
-    import urlparse urlparse
+    import base64 base64
+    import urllib_parse six.moves.urllib.parse
     import hashlib hashlib
     import itertools itertools
     import json json

stack/erp5/my.cnf.in

@@ -76,9 +76,8 @@ relay-log = mariadb-relay-bin
 {{x}}innodb_doublewrite = 0
 {{x}}sync_frm = 0
 
-# Force utf8 usage
-character_set_server = utf8mb4
-collation_server = utf8mb4_general_ci
+character_set_server = {{ parameter_dict['character-set-server'] }}
+collation_server = {{ parameter_dict['collation-server'] }}
 skip_character_set_client_handshake
 
 {% if 'ssl-key' in parameter_dict -%}

stack/erp5/run-zelenium-test.py.in

@@ -64,7 +64,7 @@ def main():
         if response.code == 200:
           static_dict = json.loads(response.read())
           activity_list = []
-          for _, value in static_dict.iteritems():
+          for _, value in six.iteritems(static_dict):
             activity_list += value['line_list']
           if len(activity_list) == 0:
             break
@@ -181,7 +181,7 @@ def main():
 
   try:
     test_result = tool.createTestResult(revision = revision,
-                                        test_name_list = test_line_dict.keys(),
+                                        test_name_list = list(test_line_dict),
                                         node_title = args.test_node_title,
                                         test_title = test_suite_title,
                                         project_title = args.project_title)

stack/erp5/site.zcml

@@ -21,6 +21,6 @@
 
 
   <securityPolicy
-      component="Products.Five.security.FiveSecurityPolicy" />
+      component="AccessControl.security.SecurityPolicy" />
 
 </configure>

stack/erp5/zope.conf.in

@@ -153,17 +153,17 @@ large-file-threshold {{ parameter_dict['large-file-threshold'] }}
 %import {{ m }}
 {% endfor -%}
 {% set type_dict = {'neo': 'NEOStorage', 'zeo': 'zeoclient'} %}
-{% for name, zodb_dict in parameter_dict['zodb-dict'].iteritems() %}
+{% for name, zodb_dict in six.iteritems(parameter_dict['zodb-dict']) %}
 <zodb_db {{ name }}>
 {%- set storage_type = type_dict[zodb_dict.pop('type')] %}
 {%- set storage_dict = zodb_dict.pop('storage-dict') %}
 {%- do root_common.apply_overrides(zodb_dict, node_id) %}
-{%- for key, value in zodb_dict.iteritems() %}
+{%- for key, value in six.iteritems(zodb_dict) %}
   {{ key }} {{ value }}
 {%- endfor %}
   <{{ storage_type }}>
 {%- do root_common.apply_overrides(storage_dict, node_id) %}
-{%- for key, value in storage_dict.iteritems() %}
+{%- for key, value in six.iteritems(storage_dict) %}
     {{ key }} {{ value }}
 {%- endfor %}
   </{{ storage_type }}>

stack/monitor/buildout.cfg

@@ -8,6 +8,7 @@ extends =
   ../../component/openssl/buildout.cfg
   ../../component/lxml-python/buildout.cfg
   ../../component/python-cryptography/buildout.cfg
+  ../../component/randomsleep/buildout.cfg
   ../../stack/logrotate/buildout.cfg
   ../../stack/slapos.cfg
 
@@ -65,6 +66,7 @@ rendered = ${buildout:directory}/template-monitor.cfg
 context =
     key apache_location apache:location
     key template_logrotate_base template-logrotate-base:rendered
+    key randomsleep randomsleep:output
     raw monitor_bin ${buildout:bin-directory}/monitor.bootstrap
     raw monitor_collect ${buildout:bin-directory}/monitor.collect
     raw monitor_statistic ${buildout:bin-directory}/monitor.statistic

stack/monitor/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [monitor2-template]
 filename = instance-monitor.cfg.jinja2.in
-md5sum = 3cba541a8b0b22c2648848ed1d259174
+md5sum = fb10eabe010d136764365c7df2993814
 
 [monitor-httpd-conf]
 _update_hash_filename_ = templates/monitor-httpd.conf.in

stack/monitor/instance-monitor.cfg.jinja2.in

@@ -250,7 +250,7 @@ recipe = slapos.cookbook:cron.d
 cron-entries = ${cron:cron-entries}
 name = monitor-globalstate
 frequency = */2 * * * *
-command = {{ bin_directory }}/randomsleep 20 && ${monitor-globalstate-wrapper:wrapper-path}
+command = {{ randomsleep }} 20 && ${monitor-globalstate-wrapper:wrapper-path}
 
 [monitor-globalstate-first-run]
 recipe = plone.recipe.command
@@ -262,14 +262,14 @@ recipe = slapos.cookbook:cron.d
 cron-entries = ${cron:cron-entries}
 name = monitor-configurator
 frequency = * * * * *
-command = {{ bin_directory }}/randomsleep 10 && ${monitor-configurator-wrapper:wrapper-path}
+command = {{ randomsleep }} 10 && ${monitor-configurator-wrapper:wrapper-path}
 
 [monitor-collect-cron-entry]
 recipe = slapos.cookbook:cron.d
 cron-entries = ${cron:cron-entries}
 name = monitor_collect
 frequency = * * * * *
-command = {{ bin_directory }}/randomsleep 40 && ${monitor-collect-wrapper:wrapper-path}
+command = {{ randomsleep }} 40 && ${monitor-collect-wrapper:wrapper-path}
 
 [logrotate-entry-monitor-data]
 recipe = collective.recipe.template

stack/slapos.cfg

@@ -132,7 +132,7 @@ eggs =
 [versions]
 setuptools = 44.1.1
 # Use SlapOS patched zc.buildout
-zc.buildout = 2.7.1+slapos015
+zc.buildout = 2.7.1+slapos016
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
 zc.recipe.egg = 2.0.3+slapos003
 
@@ -190,14 +190,14 @@ setuptools-dso = 1.7
 rubygemsrecipe  = 0.4.3
 six = 1.12.0
 slapos.cookbook = 1.0.226
-slapos.core = 1.7.2
+slapos.core = 1.7.3
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.22
 slapos.rebootstrap = 4.5
-slapos.recipe.build = 0.53
+slapos.recipe.build = 0.54
 slapos.recipe.cmmi = 0.19
-slapos.recipe.template = 4.6
+slapos.recipe.template = 5.0
 slapos.toolbox = 0.126
 stevedore = 1.21.0:whl
 subprocess32 = 3.5.4