Merge branch 'master+ZODB4-wc2' into y/wc2-next

* master+ZODB4-wc2: (32 commits)
  wendelin.core: v↑ 0.13 -> 2.0.alpha1
  caddy-frontend: Support not resolving backend hostnames
  component/caddy: Use golang 1.17
  software/metabase: configure fonts and tempdir
  software/metabase: generate proper passwords
  software/metabase: version up metabase v0.41.2
  software/metabase: sleep a bit more after "this will erase data" warning
  software/metabase: use temurin JRE
  component/java: add temurin JRE
  Fix "kvm: Increase default VM parameters to match nowadays requirement"
  kvm: Merge tests
  kvm: Allow to specify network adapter
  software/slapos-sr-testing: install new psycopg2 dependency
  recipe/postgres: support non standard port
  recipe/test_postgres: test with a real postgres server
  software/slapos-testing: install psycopg2 for slapos.cookbook
  software/slapos-testing: make postgresql available for tests
  component/psycopg2: new component
  golang: Switch default to Go1.17
  erp5testnode: update deploy script
  ...

component/caddy/buildout.cfg

@@ -13,7 +13,7 @@ repository    = https://lab.nexedi.com/nexedi/caddy.git
 revision      = nxd-v1.0.3-1-g2c11cedc
 
 [gowork]
-golang  = ${golang1.16:location}
+golang  = ${golang1.17:location}
 install =
   ${caddy-get:location}:./...
 

component/golang/buildout.cfg

@@ -46,8 +46,8 @@ environment-extra =
 
 [golang1.16]
 <= golang-common
-url = https://golang.org/dl/go1.16.8.src.tar.gz
-md5sum = 92e69a5e1bb6ea5e7498d12d03160032
+url = https://golang.org/dl/go1.16.10.src.tar.gz
+md5sum = 49f0a54f0bdcba297bac194d8dafe431
 
 # go1.16 needs go1.4 to bootstrap
 environment-extra =
@@ -55,8 +55,8 @@ environment-extra =
 
 [golang1.17]
 <= golang-common
-url = https://golang.org/dl/go1.17.1.src.tar.gz
-md5sum = a78205838c2a7054522cb91c12982f26
+url = https://golang.org/dl/go1.17.3.src.tar.gz
+md5sum = 3ea82e5966275f405f0db4f52511bb6e
 
 # go1.17 needs go1.4 to bootstrap
 environment-extra =
@@ -122,7 +122,7 @@ bin	= ${gowork.dir:bin}
 depends = ${gowork.goinstall:recipe}
 
 # go version used for the workspace (possible to override in applications)
-golang  = ${golang1.16:location}
+golang  = ${golang1.17:location}
 
 # no special build flags by default
 buildflags =

component/java/buildout.cfg

 [buildout]
-
+extends =
+  ../coreutils/buildout.cfg
+  ../patchelf/buildout.cfg
+  ../alsa/buildout.cfg
+  ../libpng/buildout.cfg
+  ../freetype/buildout.cfg
+  ../fontconfig/buildout.cfg
+  ../xorg/buildout.cfg
+  ../zlib/buildout.cfg
 parts =
   java
 
@@ -9,12 +17,12 @@ parts =
 [java-re]
 <= java-re-7
 
-[java-common]
+[java-sun-common]
 recipe = slapos.recipe.build:download-unpacked
-url = http://javadl.sun.com/webapps/download/AutoDL?BundleId=${:bundle-id}
+url = https://javadl.sun.com/webapps/download/AutoDL?BundleId=${:bundle-id}
 
 [java-re-7]
-<= java-common
+<= java-sun-common
 # http://java.com/en/download/manual_java7.jsp
 
 [java-re-7:linux and platform.machine() == 'i686']
@@ -26,7 +34,7 @@ bundle-id = 97800
 md5sum = 7605134662f6c87131eca5745895fe84
 
 [java-re-8]
-<= java-common
+<= java-sun-common
 # https://www.java.com/en/download/manual.jsp
 # Update 161
 
@@ -45,3 +53,25 @@ stop-on-error = true
 update-command = ${:command}
 command = ${coreutils-output:test} -x ${:keytool}
 keytool = ${java-re-8:location}/bin/keytool
+
+
+[java-re-temurin-11]
+recipe = slapos.recipe.build
+update =
+  from zc.buildout import UserError
+  raise UserError("unsupported platform")
+
+[java-re-temurin-11:linux and platform.machine() == 'x86_64']
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.13%2B8/OpenJDK11U-jre_x64_linux_hotspot_11.0.13_8.tar.gz
+md5sum = 1b06100bcd0923d3f3279c2f09773af0
+configure-command = :
+make-binary = :
+post-install =
+  mv * %(location)s
+  for file in %(location)s/bin/* %(location)s/lib/*.so %(location)s/lib/*/*.so ; do
+    echo appending rpath to $file
+    ${patchelf:location}/bin/patchelf --set-rpath %(rpath)s $file
+  done
+rpath = ${alsa:location}/lib:${freetype:location}/lib:${fontconfig:location}/lib:${libpng:location}/lib:${libXrender:location}/lib:${libXtst:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXext:location}/lib:${libXdmcp:location}/lib:${libXi:location}/lib:${libxcb:location}/lib:${zlib:location}/lib:@@LOCATION@@/lib:@@LOCATION@@/lib/server:@@LOCATION@@/lib/jli

component/nxdtest/buildout.cfg

@@ -20,6 +20,6 @@ egg     = nxdtest
 [nxdtest-repository]
 recipe  = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/nxdtest.git
-revision = 0ad45a9c
+revision = 9f413221
 location = ${buildout:parts-directory}/nxdtest
 git-executable = ${git:location}/bin/git

component/psycopg2/buildout.cfg

+[buildout]
+extends =
+  ../postgresql/buildout.cfg
+
+[psycopg2-env]
+PATH = ${postgresql:location}/bin:%(PATH)s
+
+[psycopg2]
+recipe = zc.recipe.egg:custom
+egg = psycopg2
+define = PSYCOPG_EXTENSIONS,PSYCOPG_DISPLAY_SIZE,PSYCOPG_NEW_BOOLEAN,HAVE_PQFREEMEM,HAVE_PQPROTOCOL3
+environment = psycopg2-env
+include-dirs =
+  ${postgresql:location}/include
+library-dirs =
+  ${postgresql:location}/lib
+rpath = 
+  ${postgresql:location}/lib

component/wendelin.core/activate-WC2-preview.cfg

-# SlapOS extension that switches Wendelin.core to Wendelin.core 2 preview.
-#
-# Should go away once wendelin.core 2 lands to nexedi/wendelin.core@master.
-# Must be extended from last, for example:
-#
-# [buildout]
-# extends =
-#   .../stack/erp5/buildout.cfg
-#   .../component/wendelin.core/activate-WC2-preview.cfg
-
-[wendelin.core-repository]
-repository = https://lab.nexedi.com/kirr/wendelin.core.git
-branch   = t
-revision =
-
-# Wendelin.core 2 targets Go1.17
-[gowork]
-golang = ${golang1.17:location}

component/wendelin.core/buildout.cfg

@@ -50,7 +50,7 @@ CGO_LDFLAGS += -Wl,-rpath=${zlib:location}/lib
 recipe  = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/wendelin.core.git
 branch   = master
-revision = v0.13-0-gb26ba55
+revision = wendelin.core-2.0.alpha1-0-g49f826b1
 # dir is pretty name as top-level recipe
 location = ${buildout:parts-directory}/wendelin.core
 git-executable = ${git:location}/bin/git

component/wendelin.core/test-common.cfg

@@ -14,6 +14,7 @@ extends =
 
 parts =
 # keep neoppod first and in parts so that ZODB is built correctly
+    neoppod-develop
     neoppod
 
 #   for instance

component/zbar/buildout.cfg

@@ -25,6 +25,7 @@ configure-options =
   --without-python 
   --without-x
   --without-jpg
+  --without-dbus
 environment =
   PATH=${autoconf:location}/bin:${automake:location}/bin:${gettext:location}/bin:${libtool:location}/bin:${m4:location}/bin:${bzip2:location}/bin:%(PATH)s
   CFLAGS=

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.213'
+version = '1.0.214'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read()
 
@@ -39,6 +39,7 @@ extras_require = {
     'test': (
         'jsonschema',
         'mock',
+        'psycopg2',
         'testfixtures',
         'requests',
     ),

slapos/recipe/postgres.py

@@ -104,6 +104,7 @@ class Recipe(GenericBaseRecipe):
         else:
             paths.extend(self.createConfig())
             paths.extend(self.createRunScript())
+            self.updateSuperuser()
 
         return paths
 
@@ -146,6 +147,7 @@ class Recipe(GenericBaseRecipe):
         with open(postgres_conf, 'w') as cfg:
             cfg.write(textwrap.dedent("""\
                     listen_addresses = '%s'
+                    port = %s
                     logging_collector = on
                     log_rotation_size = 50MB
                     max_connections = 100
@@ -161,6 +163,7 @@ class Recipe(GenericBaseRecipe):
                     unix_socket_permissions = 0700
                     """ % (
                         ','.join(set(ipv4).union(ipv6)),
+                        self.options['port'],
                         pgdata,
                         )))
 
@@ -205,9 +208,25 @@ class Recipe(GenericBaseRecipe):
 
         # encrypt the password to avoid storing in the logs
         enc_password = 'md5' + hashlib.md5((password + user).encode()).hexdigest()
+        change_password_query = """ALTER USER "%s" ENCRYPTED PASSWORD '%s'""" % (user, enc_password)
 
-        self.runPostgresCommand(cmd="""ALTER USER "%s" ENCRYPTED PASSWORD '%s'""" % (user, enc_password))
-
+        pgdata = self.options['pgdata-directory']
+        if os.path.exists(os.path.join(pgdata, 'postmaster.pid')):
+            psql_binary = os.path.join(self.options['bin'], 'psql')
+            # connect to a running postgres deamon
+            p = subprocess.Popen([
+                    psql_binary,
+                    '-h', pgdata,
+                    '-p', self.options['port'],
+                    '-U', user,
+                    '-d', self.options['dbname'],
+                ],
+                stdin=subprocess.PIPE)
+            p.communicate((change_password_query + '\n').encode())
+            if p.returncode:
+                raise UserError("Error updating password")
+        else:
+            self.runPostgresCommand(cmd=change_password_query)
 
     def runPostgresCommand(self, cmd):
         """\

slapos/test/recipe/test_postgres.py

-import unittest
-import tempfile
+import os
 import shutil
-import os.path
+import tempfile
+import textwrap
+import time
+import unittest
+
+try:
+  import subprocess32 as subprocess
+except ImportError:
+  import subprocess
+
+import psycopg2
 import zc.buildout.testing
 
 
 class PostgresTest(unittest.TestCase):
+  ipv4 = os.environ['SLAPOS_TEST_IPV4']
+  ipv6 = os.environ['SLAPOS_TEST_IPV6']
+  port = 5432
+
   def setUp(self):
     self.buildout = buildout = zc.buildout.testing.Buildout()
     self.pgdata_directory = tempfile.mkdtemp()
     self.addCleanup(shutil.rmtree, self.pgdata_directory)
     self.services_directory = tempfile.mkdtemp()
     self.addCleanup(shutil.rmtree, self.services_directory)
+
+    self.postgres_bin_directory = os.environ['SLAPOS_TEST_POSTGRESQL_PREFIX'] + '/bin'
     buildout['postgres'] = {
-      'bin': 'software/parts/postgres/bin/',
+      'bin': self.postgres_bin_directory,
       'dbname': 'dbname',
-      'ipv4': '127.0.0.1',
-      'ipv6': '::1',
-      'port': '5443',
-      'pgdata-directory': self.pgdata_directory,
+      'ipv4': self.ipv4,
+      'ipv6': self.ipv6,
+      'port': self.port,
+      'pgdata-directory': os.path.join(self.pgdata_directory, 'pgdata'),
       'services': self.services_directory,
       'superuser': 'superuser',
       'password': 'secret',
@@ -29,21 +44,73 @@ class PostgresTest(unittest.TestCase):
         'postgres',
         buildout['postgres'])
 
+  def start_postgres_server(self):
+    server_process = subprocess.Popen(
+        [ os.path.join(self.services_directory, 'postgres-start') ],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
+    self.addCleanup(server_process.wait)
+    self.addCleanup(server_process.terminate)
+
+    # wait for server to accept connections
+    for i in range(60):
+      time.sleep(i)
+      try:
+        psycopg2.connect(self.buildout['postgres']['url']).close()
+      except psycopg2.OperationalError as e:
+        pass
+      else:
+        break
+
   def test_options(self):
     self.assertEqual(
-        'postgresql://superuser:secret@[::1]:5443/dbname',
-        self.buildout['postgres']['url'])
+        self.buildout['postgres']['url'],
+        'postgresql://superuser:secret@[{self.ipv6}]:{self.port}/dbname'.format(self=self),
+    )
 
   def test_install(self):
     installed = self.recipe.install()
 
-    self.assertIn('postgresql.conf', os.listdir(self.pgdata_directory))
-    self.assertIn('pg_hba.conf', os.listdir(self.pgdata_directory))
+    pgdata_directory = os.path.join(self.pgdata_directory, 'pgdata')
+    self.assertIn('postgresql.conf', os.listdir(pgdata_directory))
+    self.assertIn('pg_hba.conf', os.listdir(pgdata_directory))
     self.assertIn('postgres-start', os.listdir(self.services_directory))
 
     self.assertEqual(
         sorted(installed),
         sorted([
-            os.path.join(self.pgdata_directory, 'postgresql.conf'),
-            os.path.join(self.pgdata_directory, 'pg_hba.conf'),
+            os.path.join(pgdata_directory, 'postgresql.conf'),
+            os.path.join(pgdata_directory, 'pg_hba.conf'),
             os.path.join(self.services_directory, 'postgres-start')]))
+
+    self.start_postgres_server()
+    with psycopg2.connect(self.buildout['postgres']['url']) as cnx:
+      with cnx.cursor() as cursor:
+        cursor.execute("SELECT 1+1")
+        self.assertEqual(cursor.fetchone(), (2,))
+    cnx.close()
+
+  def test_update_password(self):
+    self.recipe.install()
+    self.start_postgres_server()
+    self.recipe.options['password'] = 'new'
+    self.recipe.install()
+
+    dsn = self.buildout['postgres']['url']
+    with psycopg2.connect(psycopg2.extensions.make_dsn(dsn, password='new')) as cnx:
+      with cnx.cursor() as cursor:
+        cursor.execute("SELECT 1+1")
+        self.assertEqual(cursor.fetchone(), (2,))
+    cnx.close()
+
+    # old password can no longer connect
+    with self.assertRaisesRegexp(
+        psycopg2.OperationalError,
+        'password authentication failed'
+    ):
+      psycopg2.connect(dsn)
+
+
+class PostgresTestNonStandardPort(PostgresTest):
+  port = 5433

software/caddy-frontend/buildout.hash.cfg

@@ -50,7 +50,7 @@ md5sum = 37475d79f28c5f126bc1947fdb938fdb
 
 [template-backend-haproxy-configuration]
 _update_hash_filename_ = templates/backend-haproxy.cfg.in
-md5sum = 5e126be0f74d8ae390a5594e1e912a59
+md5sum = d2851c7ebd2c9baa2edecb3ca3485511
 
 [template-empty]
 _update_hash_filename_ = templates/empty.in

software/caddy-frontend/templates/backend-haproxy.cfg.in

@@ -13,6 +13,8 @@ defaults
   timeout client {{ configuration['request-timeout'] }}s
   timeout connect {{ configuration['backend-connect-timeout'] }}s
   retries {{ configuration['backend-connect-retries'] }}
+  {#- Allow to start with not resolved yet servers #}
+  default-server init-addr last,libc,none
 
 {%- set SCHEME_PREFIX_MAPPING = { 'http': 'http_backend', 'https': 'https_backend'} %}
 {%- macro frontend_entry(slave_instance, scheme, wildcard) %}

software/caddy-frontend/test/test.py

@@ -1310,6 +1310,9 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     return {
       'empty': {
       },
+      'bad-backend': {
+        'url': 'http://bad.backend/',
+      },
       'Url': {
         # make URL "incorrect", with whitespace, nevertheless it shall be
         # correctly handled

software/erp5/test/test/test_balancer.py

@@ -110,6 +110,7 @@ class CaucaseService(ManagedResource):
             '--netloc', backend_caucased_netloc,
             '--service-auto-approve-count', '1',
         ],
+        # capture subprocess output not to pollute test's own stdout
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,
     )
@@ -127,6 +128,7 @@ class CaucaseService(ManagedResource):
     # type: () -> None
     self._caucased_process.terminate()
     self._caucased_process.wait()
+    self._caucased_process.stdout.close()
     shutil.rmtree(self.directory)
 
 
@@ -525,26 +527,27 @@ class TestHTTP(BalancerTestCase):
   def test_keep_alive(self):
     # type: () -> None
     # when doing two requests, connection is established only once
-    session = requests.Session()
-    session.verify = False
+    with requests.Session() as session:
+      session.verify = False
 
-    # do a first request, which establish a first connection
-    session.get(self.default_balancer_url).raise_for_status()
-
-    # "break" new connection method and check we can make another request
-    with mock.patch(
-        "requests.packages.urllib3.connectionpool.HTTPSConnectionPool._new_conn",
-    ) as new_conn:
+      # do a first request, which establish a first connection
       session.get(self.default_balancer_url).raise_for_status()
-    new_conn.assert_not_called()
-
-    parsed_url = six.moves.urllib.parse.urlparse(self.default_balancer_url)
-    # check that we have an open file for the ip connection
-    self.assertTrue([
-        c for c in psutil.Process(os.getpid()).connections()
-        if c.status == 'ESTABLISHED' and c.raddr.ip == parsed_url.hostname
-        and c.raddr.port == parsed_url.port
-    ])
+
+      # "break" new connection method and check we can make another request
+      with mock.patch(
+          "requests.packages.urllib3.connectionpool.HTTPSConnectionPool._new_conn",
+      ) as new_conn:
+        session.get(self.default_balancer_url).raise_for_status()
+      new_conn.assert_not_called()
+
+      parsed_url = six.moves.urllib.parse.urlparse(self.default_balancer_url)
+
+      # check that we have an open file for the ip connection
+      self.assertTrue([
+          c for c in psutil.Process(os.getpid()).connections()
+          if c.status == 'ESTABLISHED' and c.raddr.ip == parsed_url.hostname
+          and c.raddr.port == parsed_url.port
+      ])
 
 
 class ContentTypeHTTPServer(ManagedHTTPServer):

software/erp5/test/test/test_erp5.py

@@ -64,28 +64,28 @@ class TestPublishedURLIsReachableMixin(object):
     # erp5 site is not created, with 500 when mysql is not yet reachable, so we
     # configure this requests session to retry.
     # XXX we should probably add a promise instead
-    session = requests.Session()
-    session.mount(
-        base_url,
-        requests.adapters.HTTPAdapter(
-            max_retries=requests.packages.urllib3.util.retry.Retry(
-                total=60,
-                backoff_factor=.5,
-                status_forcelist=(404, 500, 503))))
-
-    r = session.get(virtual_host_url, verify=verify, allow_redirects=False)
-    self.assertEqual(r.status_code, requests.codes.found)
-    # access on / are redirected to login form, with virtual host preserved
-    self.assertEqual(r.headers.get('location'), 'https://virtual-host-name:1234/virtual_host_root/login_form')
-
-    # login page can be rendered and contain the text "ERP5"
-    r = session.get(
-        six.moves.urllib.parse.urljoin(base_url, '{}/login_form'.format(site_id)),
-        verify=verify,
-        allow_redirects=False,
-    )
-    self.assertEqual(r.status_code, requests.codes.ok)
-    self.assertIn("ERP5", r.text)
+    with requests.Session() as session:
+      session.mount(
+          base_url,
+          requests.adapters.HTTPAdapter(
+              max_retries=requests.packages.urllib3.util.retry.Retry(
+                  total=60,
+                  backoff_factor=.5,
+                  status_forcelist=(404, 500, 503))))
+
+      r = session.get(virtual_host_url, verify=verify, allow_redirects=False)
+      self.assertEqual(r.status_code, requests.codes.found)
+      # access on / are redirected to login form, with virtual host preserved
+      self.assertEqual(r.headers.get('location'), 'https://virtual-host-name:1234/virtual_host_root/login_form')
+
+      # login page can be rendered and contain the text "ERP5"
+      r = session.get(
+          six.moves.urllib.parse.urljoin(base_url, '{}/login_form'.format(site_id)),
+          verify=verify,
+          allow_redirects=False,
+      )
+      self.assertEqual(r.status_code, requests.codes.ok)
+      self.assertIn("ERP5", r.text)
 
   def test_published_family_default_v6_is_reachable(self):
     """Tests the IPv6 URL published by the root partition is reachable.

software/erp5testnode/testsuite/deploy-test/standalone-local-playbook

@@ -237,7 +237,7 @@ function GetOSVersion {
         os_RELEASE=$(lsb_release -r -s)
         os_UPDATE=""
         os_PACKAGE="rpm"
-        if [[ "Debian,Ubuntu,LinuxMint" =~ $os_VENDOR ]]; then
+        if [[ "Debian,Ubuntu,LinuxMint,Raspbian" =~ $os_VENDOR ]]; then
             os_PACKAGE="deb"
         elif [[ "SUSE LINUX" =~ $os_VENDOR ]]; then
             lsb_release -d -s | grep -q openSUSE
@@ -1036,7 +1036,8 @@ function update_package_repo {
         if [[ "$REPOS_UPDATED" != "True" || "$RETRY_UPDATE" = "True" ]]; then
             # if there are transient errors pulling the updates, that's fine.
             # It may be secondary repositories that we don't really care about.
-            apt_get update  || /bin/true
+            apt_get update --allow-releaseinfo-change || /bin/true
+            apt_get update || /bin/true
             REPOS_UPDATED=True
         fi
         $xtrace
@@ -1927,6 +1928,7 @@ $XTRACE
 # mode: shell-script
 # End:
 
+
 if [[ $EUID -gt 0 ]]; then
     echo "####################################################"
     echo "#                                                  #"
@@ -1969,7 +1971,7 @@ fi
 
 # Warn users who aren't on an explicitly supported distro, but allow them to
 # override check and attempt installation with ``export FORCE=yes``
-if [[ ! ${DISTRO} =~ (wheezy|jessie|trusty|vivid|xenial|rhel7|rhel6|LinuxMint-17.2) ]]; then
+if [[ ! ${DISTRO} =~ (jessie|stretch|buster|bullseye|xenial|bionic|focal|rhel7) ]]; then
     echo "WARNING: this script has not been tested on $DISTRO"
     if [[ "$FORCE" != "yes" ]]; then
         die $LINENO "If you wish to run this script anyway run with FORCE=yes"
@@ -1978,10 +1980,10 @@ fi
 
 # Make sure wheezy backports are available.
 if [[ $DISTRO == "wheezy" ]]; then
-    echo "deb http://ftp.debian.org/debian wheezy-backports main contrib " > /etc/apt/sources.list.d/wheezy-backports.list
+    echo "deb http://archive.debian.org/debian wheezy-backports main contrib " > /etc/apt/sources.list.d/wheezy-backports.list
 fi
 
-if is_fedora && [[ $DISTRO == "rhel7" ]]; then
+if is_fedora && [[ $DISTRO =~ rhel7|f20|f19 ]]; then
     # RHEL requires EPEL for many Ansible dependencies
 
     # NOTE: We always remove and install latest -- some environments
@@ -2007,6 +2009,8 @@ failovermethod=priority
 enabled=0
 gpgcheck=0
 EOF
+    yum clean all
+    yum makecache
     yum update -y
     yum_install yum-utils
     # Enable a bootstrap repo.  It is removed after finishing
@@ -2051,18 +2055,26 @@ EOF
     is_package_installed libselinux-python || install_package libselinux-python
 fi
 
+is_package_installed unzip || install_package unzip
 is_package_installed openssl || install_package openssl
-# stick to a specific ansible package version
-if is_ubuntu; then
-  # is_ubuntu also handle debian distro
-  is_package_installed ansible || install_package ansible=1.*
-elif is_fedora; then
-  # is_fedora also handle CentOS distro
-  is_package_installed ansible || install_package ansible1.9.noarch
+is_package_installed ansible || install_package ansible
+
+if ! is_ubuntu || [[ $DISTRO = wheezy || $DISTRO == jessie || $DISTRO == stretch || $DISTRO == xenial || $DISTRO == bionic ]]; then
+  is_package_installed python-pip || install_package python-pip
 else
-  is_package_installed ansible || install_package ansible
+  # Ansible is using python3 now
+  is_package_installed python3-pip || install_package python3-pip
+fi
+
+if is_ubuntu && [[ $DISTRO == focal || $DISTRO == buster ]]; then
+  # install python2 in distro where python2 is not installed by default and ansible python interpreter is still python2
+  is_package_installed python || install_package python
+fi
+
+if is_fedora && [[ $DISTRO == "f26" ]]; then
+  is_package_installed python2-rpm || install_package python2-rpm
+  is_package_installed yum || install_package yum
 fi
-is_package_installed python-setuptools || install_package python-setuptools
 
 if is_ubuntu && [[ $DISTRO == "trusty" ]]; then
   is_package_installed python-apt || install_package python-apt
@@ -2082,6 +2094,7 @@ fi
 ANSIBLE_PLUGIN_LOCATION="/usr/share/ansible_plugins/callback_plugins/"
 mkdir -p $ANSIBLE_PLUGIN_LOCATION
 /bin/cat << EOF > $ANSIBLE_PLUGIN_LOCATION/log_parse.py
+from __future__ import absolute_import
 import os
 import time
 import json
@@ -2111,11 +2124,11 @@ class CallbackModule(baseModule):
       else:
         for filename in os.listdir(self.log_path):
           filepath = os.path.join(self.log_path, filename)
-          if os.path.exists(filepath) and os.path.isfile(filepath):
+          if os.path.isfile(filepath):
             os.unlink(filepath)
 
     def writeLog(self, host, category, content):
-      if not self.fd_list.has_key(category):
+      if category not in self.fd_list:
         self.fd_list[category] = open(
           os.path.join(self.log_path, '%s_%s' % (host, category)), "a"
         )
@@ -2126,12 +2139,11 @@ class CallbackModule(baseModule):
       if host == "localhost":
         host = "127.0.0.1" # keep compatibility
 
-      if type(data) == dict:
+      if type(data) is dict:
         if '_ansible_verbose_override' in data:
           # avoid logging extraneous data
           return
 
-        data = data.copy()
         content = json.dumps(data)
 
         if ignore_errors:
@@ -2168,8 +2180,17 @@ class CallbackModule(baseModule):
 
 EOF
 
-ansible localhost -m easy_install -a name=slapcache --connection=local
-ansible localhost -m easy_install -a name=requests --connection=local
+ansible localhost -m pip -a name=setuptools --connection=local
+ansible localhost -m pip -a name=wheel --connection=local
+ansible localhost -m pip -a name=slapcache --connection=local
+ansible localhost -m pip -a name=requests --connection=local
+
+if [ ! -f /usr/share/ansible_plugins/mitogen.zip ]; then
+    wget -O /usr/share/ansible_plugins/mitogen.zip https://shacache.nxdcdn.com/3a935ff257ddc0ad4e0f23d71681e026f14f309f4bed0a8e2a217da9b294be2c676196703f0dde856ece49d711d0221deae70812f035b24aa5cdd0ca02790e85
+    unzip /usr/share/ansible_plugins/mitogen.zip -d /usr/share/ansible_plugins/mitogen/
+    mv /usr/share/ansible_plugins/mitogen/mitogen-*/* /usr/share/ansible_plugins/mitogen/
+fi
+
 
 # Include Additional Functions
 if [ ! -f /etc/opt/slapcache.cfg ]; then

software/kvm/buildout.hash.cfg

@@ -19,11 +19,11 @@ md5sum = f2b0f1ed27148504f220e06eaceff935
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = 17c58f74d1ed4cb7dce11bf9af71dd33
+md5sum = f902dd10cb052ac262a4a96b9362b3a3
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in
-md5sum = fcb35c32ef985e3d69a7914711675dcc
+md5sum = 67901b7a97b324aa36851d8e8b43cad6
 
 [template-kvm-resilient]
 filename = instance-kvm-resilient.cfg.jinja2
@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
 
 [template-kvm-run]
 filename = template/template-kvm-run.in
-md5sum = f840e8b7af83982525f66d7ec12b7085
+md5sum = 875261817970d0f83335824373288b9d
 
 [template-kvm-controller]
 filename = template/kvm-controller-run.in

software/kvm/instance-kvm-cluster-input-schema.json

@@ -339,6 +339,22 @@
               "type": "string",
               "default": "host"
             },
+            "network-adapter": {
+              "title": "Network adapter",
+              "description": "Network adapter provided to the guest. Allows to support legacy guests, which do not have drivers for default virtio-net-pci.",
+              "type": "string",
+              "default": "virtio-net-pci",
+              "enum": [
+                "virtio-net-pci",
+                "e1000",
+                "ne2k_isa",
+                "ne2k_pci",
+                "pcnet",
+                "rtl8139",
+                "usb-net",
+                "vmxnet3"
+              ]
+            },
             "nbd-host": {
               "title": "NBD hostname or IP",
               "description": "hostname (or IP) of the NBD server containing the boot image.",

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -85,7 +85,8 @@ config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 40)) }}
 config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }}
 config-disk-format = {{ dumps(kvm_parameter_dict.get('disk-format', 'qcow2')) }}
 config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 2)) }}
-config-cpu-max-count = {{ dumps(kvm_parameter_dict.get('cpu-max-count', int(kvm_parameter_dict.get('cpu-count', 1)) + 1)) }}
+config-cpu-max-count = {{ dumps(kvm_parameter_dict.get('cpu-max-count', int(kvm_parameter_dict.get('cpu-count', 2)) + 1)) }}
+config-network-adapter = {{ dumps(kvm_parameter_dict.get('network-adapter', 'virtio-net-pci')) }}
 {{ setconfig('numa', kvm_parameter_dict.get('numa', '')) }}
 {{ setconfig('machine-options', kvm_parameter_dict.get('machine-options', '')) }}
 {{ setconfig('nbd-host', kvm_parameter_dict.get('nbd-host', '')) }}

software/kvm/instance-kvm-input-schema.json

@@ -127,6 +127,22 @@
       "type": "string",
       "default": "host"
     },
+    "network-adapter": {
+      "title": "Network adapter",
+      "description": "Network adapter provided to the guest. Allows to support legacy guests, which do not have drivers for default virtio-net-pci.",
+      "type": "string",
+      "default": "virtio-net-pci",
+      "enum": [
+        "virtio-net-pci",
+        "e1000",
+        "ne2k_isa",
+        "ne2k_pci",
+        "pcnet",
+        "rtl8139",
+        "usb-net",
+        "vmxnet3"
+      ]
+    },
     "nbd-host": {
       "title": "NBD hostname",
       "description": "hostname (or IP) of the NBD server containing the boot image.",

software/kvm/instance-kvm.cfg.jinja2

@@ -434,6 +434,8 @@ disk-format = ${slap-parameter:disk-format}
 disk-device-path = ${slap-parameter:disk-device-path}
 disk-path = ${directory:srv}/virtual.${slap-parameter:disk-format}
 
+network-adapter = ${slap-parameter:network-adapter}
+
 pid-file-path = ${kvm-controller-parameter-dict:pid-file}
 socket-path = ${kvm-controller-parameter-dict:socket-path}
 
@@ -1075,6 +1077,8 @@ disk-type = virtio
 disk-format = qcow2
 disk-device-path =
 
+network-adapter = virtio-net-pci
+
 cpu-count = 2
 disk-cache =
 disk-aio =

software/kvm/template/template-kvm-run.in

@@ -19,6 +19,8 @@ qemu_path = {{ repr(parameter_dict["qemu-path"]) }}
 disk_size = {{ repr(parameter_dict["disk-size"]) }}
 disk_type = {{ repr(parameter_dict["disk-type"]) }}
 
+network_adapter = {{ repr(parameter_dict["network-adapter"]) }}
+
 socket_path = '{{ parameter_dict.get("socket-path") }}'
 nbd_list = (('{{ parameter_dict.get("nbd-host") }}',
               {{ parameter_dict.get("nbd-port") }}),
@@ -255,7 +257,7 @@ if use_nat == 'true':
   if use_tap == 'true' and tap_ipv6_addr != '':
     rules += ',ipv6=off'
   nat_network_parameter = ['-netdev', rules,
-          '-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, mac_address)]
+          '-device', '%s,netdev=lan%s,mac=%s' % (network_adapter, number, mac_address)]
 if use_tap == 'true':
   number += 1
   vhost = ''
@@ -264,7 +266,7 @@ if use_tap == 'true':
   tap_network_parameter = ['-netdev',
           'tap,id=lan%s,ifname=%s,script=no,downscript=no%s' % (number,
             tap_interface, vhost),
-          '-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, tap_mac_address)]
+          '-device', '%s,netdev=lan%s,mac=%s' % (network_adapter, number, tap_mac_address)]
 
 smp = '%s,maxcpus=%s' % (init_smp_count, smp_max_count)
 ram = '%sM,slots=128,maxmem=%sM' % (init_ram_size, ram_max_size)

software/kvm/test/test.py

@@ -1179,27 +1179,6 @@ class TestBootImageUrlSelectKvmCluster(TestBootImageUrlListKvmCluster):
   config_file_name = 'boot-image-url-select.json'
 
 
-@skipUnlessKvm
-class TestCpuMemMaxDynamic(InstanceTestCase):
-  __partition_reference__ = 'cmm'
-
-  @classmethod
-  def getInstanceParameterDict(cls):
-    return {
-      'cpu-count': 2,
-      'ram-size': 2048
-    }
-
-  def test(self):
-    with open(os.path.join(
-     self.computer_partition_root_path, 'bin', 'kvm_raw'), 'r') as fh:
-      kvm_raw = fh.read()
-    self.assertIn('smp_count = 2', kvm_raw)
-    self.assertIn('smp_max_count = 3', kvm_raw)
-    self.assertIn('ram_size = 2048', kvm_raw)
-    self.assertIn("ram_max_size = '2560'", kvm_raw)
-
-
 @skipUnlessKvm
 class TestNatRules(InstanceTestCase):
   __partition_reference__ = 'nr'
@@ -1611,3 +1590,101 @@ INF: Storing errors in %(error_state_file)s
     self.assertFalse(
       os.path.exists(
         os.path.join(self.destination_directory, 'destination')))
+
+
+@skipUnlessKvm
+class TestParameterDefault(InstanceTestCase, KvmMixin):
+  __partition_reference__ = 'pd'
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'default'
+
+  def mangleParameterDict(self, parameter_dict):
+    return parameter_dict
+
+  def _test(self, parameter_dict, expected):
+    self.rerequestInstance(self.mangleParameterDict(parameter_dict))
+    self.slap.waitForInstance(max_retry=10)
+    
+    kvm_raw = glob.glob(os.path.join(
+      self.slap.instance_directory, '*', 'bin', 'kvm_raw'))
+    self.assertEqual(len(kvm_raw), 1)
+    kvm_raw = kvm_raw[0]
+    with open(kvm_raw, 'r') as fh:
+      kvm_raw = fh.read()
+    self.assertIn(expected, kvm_raw)
+
+  def test_disk_type_default(self):
+    self._test({}, "disk_type = 'virtio'")
+
+  def test_disk_type_set(self):
+    self._test({'disk-type': 'ide'}, "disk_type = 'ide'")
+
+  def test_network_adapter_default(self):
+    self._test({}, "network_adapter = 'virtio-net-pci")
+
+  def test_network_adapter_set(self):
+    self._test({'network-adapter': 'e1000'}, "network_adapter = 'e1000'")
+
+  def test_cpu_count_default(self):
+    self._test({}, "init_smp_count = 2")
+
+  def test_cpu_count_default_max(self):
+    self._test({}, "smp_max_count = 3")
+
+  def test_cpu_count_set(self):
+    self._test({'cpu-count': 4}, "init_smp_count = 4")
+
+  def test_cpu_count_set_max(self):
+    self._test({'cpu-count': 4}, "smp_max_count = 5")
+
+  def test_ram_size_default(self):
+    self._test({}, "init_ram_size = 4096")
+
+  def test_ram_size_default_max(self):
+    self._test({}, "ram_max_size = '4608'")
+
+  def test_ram_size_set(self):
+    self._test({'ram-size': 2048}, "init_ram_size = 2048")
+
+  def test_ram_size_set_max(self):
+    self._test({'ram-size': 2048}, "ram_max_size = '2560'")
+
+
+@skipUnlessKvm
+class TestParameterResilient(TestParameterDefault):
+  __partition_reference__ = 'pr'
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'kvm-resilient'
+
+
+@skipUnlessKvm
+class TestParameterCluster(TestParameterDefault):
+  __partition_reference__ = 'pc'
+
+  parameter_dict = {
+    "disable-ansible-promise": True
+  }
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'_': json.dumps({
+      "kvm-partition-dict": {
+        "KVM0": cls.parameter_dict
+      }
+    })}
+
+  def mangleParameterDict(self, parameter_dict):
+    local_parameter_dict = self.parameter_dict.copy()
+    local_parameter_dict.update(parameter_dict)
+    return {'_': json.dumps({
+      "kvm-partition-dict": {
+        "KVM0": local_parameter_dict
+      }
+    })}
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'kvm-cluster'

software/metabase/README.md

-# Metabae
+# Metabase
 
 https://www.metabase.com/
 
 ## TODO:
  * export backups for resilience
- * security (proper passwords, verifiable certificate, study metabase encryption option)
+ * security (verifiable certificate, study metabase encryption option)

software/metabase/buildout.hash.cfg

 [instance-profile]
 filename = instance.cfg.in
-md5sum = 143f46b125389f39905226ec9482ce2a
+md5sum = 5f2f7c4c2f793d609ad3c4fa0aa2f8a5

software/metabase/instance.cfg.in

@@ -7,10 +7,28 @@ eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
+[fontconfig-conf]
+recipe = slapos.recipe.template:jinja2
+template = ${template-fonts-conf:output}
+rendered = $${directory:etc}/fonts.conf
+context =
+    key cachedir directory:fontconfig-cache
+    key fonts :fonts
+    key includes :includes
+fonts =
+    ${android-fonts:location}
+    ${dejavu-fonts:location}
+    ${ipa-fonts:location}
+    ${ipaex-fonts:location}
+    ${liberation-fonts:location}
+    ${ocrb-fonts:location}
+includes =
+    ${fontconfig:location}/etc/fonts/conf.d
+
 [metabase-instance]
 recipe = slapos.cookbook:wrapper
 wrapper-path = $${directory:service}/$${:_buildout_section_name_}
-command-line = sh -c "cd $${directory:srv-metabase}; ${java-re-8:location}/bin/java $JAVA_ARGS -jar ${metabase.jar:location}/metabase.jar"
+command-line = sh -c "cd $${directory:srv-metabase}; ${java:location}/bin/java $JAVA_ARGS -jar ${metabase.jar:location}/metabase.jar"
 # https://www.metabase.com/docs/latest/operations-guide/customizing-jetty-webserver.html
 # note that we set org.quartz.scheduler.instanceId through $JAVA_ARGS as a workaround for machines
 # which cannot resolve their hostnames. See also https://github.com/metabase/metabase/issues/8373
@@ -28,7 +46,8 @@ environment =
   MB_DB_USER=$${postgresql:superuser}
   MB_DB_PASS=$${postgresql:password}
   MB_DB_HOST=$${postgresql:ipv4}
-  JAVA_ARGS=-Dorg.quartz.scheduler.instanceId=$${slap-connection:computer-id}.$${slap-connection:partition-id}
+  FONTCONFIG_FILE=$${fontconfig-conf:rendered}
+  JAVA_ARGS=-Dorg.quartz.scheduler.instanceId=$${slap-connection:computer-id}.$${slap-connection:partition-id} -Djava.io.tmpdir="$${directory:tmp}"
 hash-existing-files =
   $${buildout:directory}/software_release/buildout.cfg
 
@@ -49,19 +68,56 @@ promise = check_url_available
 name = $${:_buildout_section_name_}.py
 config-url= $${metabase-instance:url}/api/session/properties
 
+[metabase-keystore-password]
+recipe = slapos.cookbook:generate.password
+bytes = 24
+
 [metabase-keystore]
 recipe = plone.recipe.command
+stop-on-error = true
 command =
-  ${java-re-8-output:keytool} \
-    -genkeypair \
-    -alias "metabase" \
-    -keyalg RSA \
-    -keypass "$${:password}" \
-    -dname "CN=$${metabase-instance:ip},OU=Unit,O=Organization,L=City,S=State,C=Country" \
-    -keystore "$${:file}" \
-    -storepass "$${:password}"
+  if [ -f $${:file} ]
+  then
+    # XXX password used to be "insecure", but we changed to proper password.
+    # We try to list the store with the new password and if it fail we change
+    # the keystore password.
+    if ! ${java:location}/bin/keytool \
+      -list \
+      -keystore "$${:file}" \
+      -storepass "$${:password}"
+    then
+      echo "Migrating keystore password" && \
+      ${java:location}/bin/keytool \
+        -storepasswd \
+        -keystore "$${:file}" \
+        -storepass insecure \
+        -new "$${:password}" && \
+      echo "Migrating certificate key password" && \
+      ${java:location}/bin/keytool \
+        -keypasswd \
+        -alias "$${:alias}" \
+        -keypass insecure \
+        -new "$${:password}" \
+        -keystore "$${:file}" \
+        -storepass "$${:password}"
+    fi
+  else
+    ${java:location}/bin/keytool \
+      -genkeypair \
+      -alias "$${:alias}" \
+      -keyalg RSA \
+      -keypass "$${:password}" \
+      -dname "CN=$${metabase-instance:ip},OU=Unit,O=Organization,L=City,S=State,C=Country" \
+      -keystore "$${:file}" \
+      -storepass "$${:password}"
+  fi
 file = $${directory:etc}/.metabase_keystore
-password = insecure
+password = $${metabase-keystore-password:passwd}
+alias = metabase
+
+[postgresql-password]
+recipe = slapos.cookbook:generate.password
+bytes = 24
 
 [postgresql]
 recipe = slapos.cookbook:postgres
@@ -69,7 +125,7 @@ bin = ${postgresql10:location}/bin/
 services = $${directory:service}
 dbname  = metabase_db
 superuser = metabase-psql
-password = insecure
+password = $${postgresql-password:passwd}
 pgdata-directory = $${directory:srv}/postgresql
 
 ipv4 = $${instance-parameter:ipv4-random}
@@ -125,7 +181,7 @@ wrapper-path = $${directory:bin}/$${:_buildout_section_name_}
 command-line =
   sh -e -c "\
     echo 'This will replace current database with latest backup. Hit Ctrl+C to cancel';
-    sleep 5;
+    sleep 30;
     $${postgresql:bin}/pg_restore \
         --exit-on-error \
         -h $${postgresql:pgdata-directory} \
@@ -176,7 +232,7 @@ tmp = $${buildout:directory}/tmp
 service = $${:etc}/service
 srv-metabase = $${:srv}/metabase
 srv-backup = $${:srv}/backup
-
+fontconfig-cache = $${buildout:directory}/.fontconfig
 
 [publish-connection-parameter]
 recipe = slapos.cookbook:publish

software/metabase/software.cfg

 [buildout]
 extends =
     ../../component/defaults.cfg
+    ../../component/fontconfig/buildout.cfg
+    ../../component/fonts/buildout.cfg
     ../../component/java/buildout.cfg
     ../../component/postgresql/buildout.cfg
     ../../component/dcron/buildout.cfg
@@ -15,10 +17,13 @@ parts =
 [python]
 part = python3
 
+[java]
+<= java-re-temurin-11
+
 [metabase.jar]
 recipe = slapos.recipe.build:download
-url = https://downloads.metabase.com/v0.38.3/metabase.jar
-md5sum = b91e4b9ae13c892894fec8bab2fd195f
+url = https://downloads.metabase.com/v0.41.2/metabase.jar
+md5sum = 630068d1ccbdc95556931fe9cfc12e61
 
 [instance-profile]
 recipe = slapos.recipe.template

software/metabase/test/test.py

@@ -85,7 +85,7 @@ class TestMetabaseSetup(MetabaseTestCase):
             "username": email,
             "password": "wrong"
         })
-    self.assertEqual(requests.codes.bad_request, resp.status_code)
+    self.assertEqual(resp.status_code, requests.codes.unauthorized)
 
     session = requests.post(
         parse.urljoin(url, '/api/session'),

software/slapos-master/software.cfg

 [buildout]
 extends =
   ../../software/erp5/software.cfg
-  ../../component/wendelin.core/activate-WC2-preview.cfg
   buildout.hash.cfg
 
 parts +=

software/slapos-sr-testing/software.cfg

@@ -8,6 +8,7 @@ extends =
   ../../component/pycurl/buildout.cfg
   ../../component/faketime/buildout.cfg
   ../../component/pillow/buildout.cfg
+  ../../component/psycopg2/buildout.cfg
   ../../component/python-cryptography/buildout.cfg
   ../../component/python-mysqlclient/buildout.cfg
   ../../component/python-pynacl/buildout.cfg
@@ -242,6 +243,7 @@ extra-eggs =
   ${python-mysqlclient:egg}
   ${backports.lzma:egg}
   ${bcrypt:egg}
+  ${psycopg2:egg}
   slapos.libnetworkcache
   supervisor
   ${slapos.cookbook-setup:egg}
@@ -393,3 +395,4 @@ mysqlclient = 1.3.12
 pexpect = 4.8.0
 ptyprocess = 0.6.0
 typing = 3.7.4.3
+psycopg2 = 2.8.6

software/slapos-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = 2a82b7d5163d042e85b14a645707a093
+md5sum = 597c29546519aabe7259e416d0b92095

software/slapos-testing/instance.cfg

@@ -32,6 +32,16 @@ repository = ${kedifa-repository:location}
 <= download-source
 repository = ${caucase-repository:location}
 
+[caucase-test-runner]
+recipe = slapos.recipe.template:jinja2
+template = inline:#!/bin/sh
+  export HOSTS="$(mktemp)"
+  trap 'rm "$HOSTS"' EXIT
+  printf '%s testhost\n%s testhost\n' "$SLAPOS_TEST_IPV4" "$SLAPOS_TEST_IPV6" > "$HOSTS"
+  export CAUCASE_NETLOC=testhost:8000 LD_PRELOAD=${userhosts:location}/lib/userhosts.so:$LD_PRELOAD
+  exec python -m unittest discover -v
+rendered = $${caucase:location}/host_setting.sh
+
 [slapos.libnetworkcache]
 <= download-source
 repository = ${slapos.libnetworkcache-repository:location}
@@ -77,11 +87,12 @@ repository = ${rubygemsrecipe-repository:location}
 recipe = slapos.recipe.template:jinja2
 rendered = $${create-directory:etc}/$${:_buildout_section_name_}
 template = inline:
-  export PATH=${coreutils:location}/bin:${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:${socat:location}/bin:${lmsensors:location}/bin:${rsync:location}/bin/:${buildout:bin-directory}:$PATH
+  export PATH=${coreutils:location}/bin:${curl:location}/bin:${openssl:location}/bin:${jq:location}/bin:${sed:location}/bin:${grep:location}/bin:${git:location}/bin:${libxslt:location}/bin:${socat:location}/bin:${lmsensors:location}/bin:${rsync:location}/bin/:${buildout:bin-directory}:$PATH
   export SLAPOS_TEST_IPV4=$${slap-configuration:ipv4-random}
   export SLAPOS_TEST_IPV6=$${slap-configuration:ipv6-random}
   export SLAPOS_TEST_EGGS_DIRECTORY=$${buildout:eggs-directory}
   export SLAPOS_TEST_DEVELOP_EGGS_DIRECTORY=$${buildout:develop-eggs-directory}
+  export SLAPOS_TEST_POSTGRESQL_PREFIX=${postgresql:location}
 
 
 [slapos-test-runner-dot-nxdtest]
@@ -98,9 +109,7 @@ template = inline:
   )
   TestCase(
       "caucase",
-      # XXX caucase uses 2to3 dynamically in setup.py, so it only supports
-      # runnning tests with python setup.py test
-      ['python', 'setup.py', 'test'],
+      ['$${caucase-test-runner:rendered}'],
       cwd="""$${caucase:location}""",
       summaryf=UnitTest.summary,
   )

software/slapos-testing/software.cfg

@@ -13,6 +13,12 @@ extends =
   ../../component/socat/buildout.cfg
   ../../component/lmsensors/buildout.cfg
   ../../component/rsync/buildout.cfg
+  ../../component/jq/buildout.cfg
+  ../../component/sed/buildout.cfg
+  ../../component/grep/buildout.cfg
+  ../../component/userhosts/buildout.cfg
+  ../../component/postgresql/buildout.cfg
+  ../../component/psycopg2/buildout.cfg
   ../../stack/slapos.cfg
   ../../stack/caucase/buildout.cfg
   ../../stack/nxdtest.cfg
@@ -62,6 +68,7 @@ egg = slapos.cookbook[test]
 setup = ${slapos.cookbook-repository:location}
 depends =
   ${slapos.core-setup:egg}
+  ${psycopg2:egg}
   ${slapos-cookbook-dependencies:eggs}
 
 [slapos.core-setup]
@@ -241,3 +248,4 @@ pyflakes = 2.0.0
 zope.testing = 4.6.2
 urllib3 = 1.24.1
 pathlib = 1.0.1
+psycopg2 = 2.8.6

stack/slapos.cfg

@@ -190,7 +190,7 @@ setproctitle = 1.1.10
 setuptools-dso = 1.7
 rubygemsrecipe  = 0.4.3
 six = 1.12.0
-slapos.cookbook = 1.0.213
+slapos.cookbook = 1.0.214
 slapos.core = 1.6.19
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0

stack/unstable/lapp/buildout.cfg

@@ -36,6 +36,7 @@ extends =
   ../../component/stunnel/buildout.cfg
   ../../component/dropbear/buildout.cfg
   ../../component/pycurl/buildout.cfg
+  ../../component/psycopg2/buildout.cfg
   ../slapos.cfg
   ../resilient/buildout.cfg
 
@@ -127,26 +128,6 @@ md5sum = d95205a5fc2825e9709ed6db295111e2
 mode = 0644
 
 
-#----------------
-#-- Postgres driver for Python recipes.
-
-[psycopg2-env]
-PATH = ${postgresql:location}/bin:%(PATH)s
-
-
-[psycopg2]
-recipe = zc.recipe.egg:custom
-egg = psycopg2
-define = PSYCOPG_EXTENSIONS,PSYCOPG_DISPLAY_SIZE,PSYCOPG_NEW_BOOLEAN,HAVE_PQFREEMEM,HAVE_PQPROTOCOL3
-environment = psycopg2-env
-include-dirs =
-    ${postgresql:location}/include
-library-dirs =
-    ${postgresql:location}/lib
-rpath = 
-    ${postgresql:location}/lib
-
-
 #----------------
 #--
 #-- Optional part allowing applications using this stack to run a custom