Update release candidate

component/apache/apache-backend.conf.in

@@ -131,13 +131,11 @@ SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:EC
 SSLSessionCache shmcb:{{ parameter_dict['ssl-session-cache'] }}(512000)
 SSLProxyEngine On
 
-# As backend is trusting REMOTE_USER header unset it always
-RequestHeader unset REMOTE_USER
-RequestHeader unset SSL_CLIENT_SERIAL
+# As backend is trusting Remote-User header unset it always
+RequestHeader unset Remote-User
 {% if parameter_dict['ca-cert'] -%}
 SSLVerifyClient optional
-RequestHeader set REMOTE_USER %{SSL_CLIENT_S_DN_CN}s
-RequestHeader set SSL_CLIENT_SERIAL "%{SSL_CLIENT_M_SERIAL}s"
+RequestHeader set Remote-User %{SSL_CLIENT_S_DN_CN}s
 SSLCACertificateFile {{ parameter_dict['ca-cert'] }}
 {%   if parameter_dict['crl'] -%}
 SSLCARevocationCheck chain

component/apache/buildout.hash.cfg

@@ -14,5 +14,5 @@
 # not need these here).
 [template-apache-backend-conf]
 filename = apache-backend.conf.in
-md5sum = 84d43d3535ffc67f677710b1d97e19aa
+md5sum = bb8c175a93336f0e1838fd47225426f9
 

component/caddy/buildout.cfg

@@ -17,7 +17,7 @@ depends =
 
 [caddy]
 # revision and repository can be used to control which caddy version is used
-revision = v1.0.1
+revision = v1.0.3
 repository = github.com/caddyserver/caddy/caddy
 
 recipe  = plone.recipe.command

component/git/buildout.cfg

@@ -18,8 +18,8 @@ parts =
 [git]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.17.1.tar.xz
-md5sum = 5179245515c637357b4a134e8d4e9a6f
+url = https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.23.0.tar.xz
+md5sum = 93ee0f867f81a39e0ef29eabfb1d2c5b
 configure-options =
   --with-curl=${curl:location}
   --with-openssl=${openssl:location}

component/golang/buildout.cfg

@@ -64,8 +64,8 @@ environment-extra =
 
 [golang1.12]
 <= golang-common
-url = https://dl.google.com/go/go1.12.7.src.tar.gz
-md5sum = 49d7a658cbd825f1cfe903d050bad29f
+url = https://dl.google.com/go/go1.12.9.src.tar.gz
+md5sum = 6132109d4050da349eadc9f7b0304ef4
 
 # go1.11 needs go1.4 to bootstrap
 environment-extra =

component/lz4/buildout.cfg

@@ -5,6 +5,6 @@ parts =
 
 [lz4]
 recipe = slapos.recipe.cmmi
-url = https://github.com/lz4/lz4/archive/v1.8.3.tar.gz
-md5sum = d5ce78f7b1b76002bbfffa6f78a5fc4e
+url = https://github.com/lz4/lz4/archive/v1.9.2.tar.gz
+md5sum = 3898c56c82fb3d9455aefd48db48eaad
 configure-command = true

component/zstd/buildout.cfg

@@ -5,8 +5,8 @@ parts =
 [zstd]
 recipe = slapos.recipe.cmmi
 url = https://github.com/facebook/zstd/releases/download/v${:version}/zstd-${:version}.tar.gz
-version = 1.4.2
-md5sum = 1d6aea1cd67a8eab7aa6905f4bf148f8
+version = 1.4.3
+md5sum = 8581c03b2f56c14ff097a737e60847b3
 shared = true
 location = @@LOCATION@@
 configure-command = :

setup.py

@@ -110,9 +110,7 @@ setup(name=name,
           'generic.cloudooo = slapos.recipe.generic_cloudooo:Recipe',
           'generic.kumofs = slapos.recipe.generic_kumofs:Recipe',
           'generic.memcached = slapos.recipe.generic_memcached:Recipe',
-          'generic.mysql = slapos.recipe.generic_mysql:Recipe',
           'generic.mysql.wrap_update_mysql = slapos.recipe.generic_mysql:WrapUpdateMySQL',
-          'generic.mysql.wrap_mysqld = slapos.recipe.generic_mysql:WrapMySQLd',
           'generic.varnish = slapos.recipe.generic_varnish:Recipe',
           'gitinit = slapos.recipe.gitinit:Recipe',
           'haproxy = slapos.recipe.haproxy:Recipe',

slapos/recipe/generic_mysql/__init__.py

@@ -25,206 +25,6 @@
 #
 ##############################################################################
 from slapos.recipe.librecipe import GenericBaseRecipe
-import os
-
-class Recipe(GenericBaseRecipe):
-
-  def _options(self, options):
-    options['password'] = self.generatePassword()
-    if 'test-database' in options:
-      options['test-password'] = self.generatePassword()
-    options.setdefault('parallel-test-database-amount', '0')
-    for x in xrange(int(options['parallel-test-database-amount'])):
-      options['test-password-%s' % x] = self.generatePassword()
-
-  def install(self):
-    path_list = []
-
-    template_filename = self.getTemplateFilename('my.cnf.in')
-
-    mysql_binary = self.options['mysql-binary']
-    socket = self.options['socket']
-
-    if 'ip' in self.options:
-      networking = 'port = %s\nbind-address = %s' % (
-        self.options['port'],
-        self.options['ip'],
-      )
-    else:
-      networking = 'skip-networking'
-
-    log_bin = self.options.get('binlog-path', '')
-    if log_bin:
-      log_bin = 'log_bin = %s' % log_bin
-    expire_logs_days = self.options.get('binlog-expire-days')
-    if expire_logs_days > 0:
-      expire_logs_days = 'expire_logs_days = %s' % expire_logs_days
-    else:
-      expire_logs_days = ''
-
-    mysql_conf_file = self.createFile(
-      self.options['conf-file'],
-      self.substituteTemplate(template_filename, {
-        'networking': networking,
-        'data_directory': self.options['data-directory'],
-        'pid_file': self.options['pid-file'],
-        'socket': self.options['socket'],
-        'error_log': self.options['error-log'],
-        'slow_query_log': self.options['slow-query-log'],
-        'log_bin': log_bin,
-        'expire_logs_days': expire_logs_days,
-      })
-    )
-    path_list.append(mysql_conf_file)
-
-    mysql_script_list = []
-
-    # user defined functions
-    udf_registration = "DROP FUNCTION IF EXISTS last_insert_grn_id;\nDROP FUNCTION IF EXISTS mroonga_snippet;\nDROP FUNCTION IF EXISTS mroonga_command;\n"
-    mroonga = self.options.get('mroonga', 'ha_mroonga.so')
-    if mroonga:
-      udf_registration += "CREATE FUNCTION last_insert_grn_id RETURNS " \
-        "INTEGER SONAME '" + mroonga + "';\n"
-      udf_registration += "CREATE FUNCTION mroonga_snippet RETURNS " \
-        "STRING SONAME '" + mroonga + "';\n"
-      udf_registration += "CREATE FUNCTION mroonga_command RETURNS " \
-        "STRING SONAME '" + mroonga + "';\n"
-    mysql_script_list.append(self.substituteTemplate(
-      self.getTemplateFilename('mysql-init-function.sql.in'),
-      {
-        'udf_registration': udf_registration,
-      }
-    ))
-    # real database
-    mysql_script_list.append(self.substituteTemplate(
-      self.getTemplateFilename('initmysql.sql.in'),
-      {
-        'mysql_database': self.options['database'],
-        'mysql_user': self.options['user'],
-        'mysql_password': self.options['password']
-      }
-    ))
-    # default test database
-    if 'test-database' in self.options:
-      mysql_script_list.append(self.substituteTemplate(
-        self.getTemplateFilename('initmysql.sql.in'),
-        {
-          'mysql_database': self.options['test-database'],
-          'mysql_user': self.options['test-user'],
-          'mysql_password': self.options['test-password']
-        }
-      ))
-    # parallel test databases
-    for x in xrange(int(self.options['parallel-test-database-amount'])):
-      mysql_script_list.append(self.substituteTemplate(
-        self.getTemplateFilename('initmysql.sql.in'),
-        {
-          'mysql_database': self.options['mysql-test-database-base'] + '_%s' % x,
-          'mysql_user': self.options['mysql-test-user-base'] + '_%s' % x,
-          'mysql_password': self.options['test-password-%s' % x]
-        }
-      ))
-    mysql_script_list.append('EXIT')
-    mysql_script = '\n'.join(mysql_script_list)
-
-    mysql_upgrade_binary = self.options['mysql-upgrade-binary']
-    mysql_update = self.createPythonScript(
-      self.options['update-wrapper'],
-      '%s.mysql.updateMysql' % __name__,
-      [dict(
-        mysql_script=mysql_script,
-        mysql_binary=mysql_binary,
-        mysql_upgrade_binary=mysql_upgrade_binary,
-        socket=socket,
-      )]
-    )
-    path_list.append(mysql_update)
-
-    mysqld = self.createPythonScript(
-      self.options['wrapper'],
-      '%s.mysql.runMysql' % __name__,
-      [dict(
-        mysql_base_directory=self.options['mysql-base-directory'],
-        mysql_install_binary=self.options['mysql-install-binary'],
-        mysqld_binary=self.options['mysqld-binary'],
-        data_directory=self.options['data-directory'],
-        mysql_binary=mysql_binary,
-        socket=socket,
-        configuration_file=mysql_conf_file,
-       )]
-    )
-    path_list.append(mysqld)
-    environment = {'PATH': self.options['bin-directory']}
-    # TODO: move to a separate recipe (ack'ed by Cedric)
-    if 'backup-script' in self.options:
-      # backup configuration
-      full_backup = self.options['full-backup-directory']
-      incremental_backup = self.options['incremental-backup-directory']
-      innobackupex_argument_list = [self.options['perl-binary'],
-          self.options['innobackupex-binary'],
-          '--defaults-file=%s' % mysql_conf_file,
-          '--socket=%s' % socket.strip(), '--user=root',
-          '--ibbackup=%s'% self.options['xtrabackup-binary']]
-      innobackupex_incremental = self.createWrapper(
-        self.options['innobackupex-incremental'],
-        innobackupex_argument_list + ['--incremental'], environment)
-      path_list.append(innobackupex_incremental)
-      innobackupex_full = self.createWrapper(
-        self.options['innobackupex-full'],
-        innobackupex_argument_list, environment)
-      path_list.append(innobackupex_full)
-      backup_controller = self.createPythonScript(self.options['backup-script'], __name__ + '.innobackupex.controller', [innobackupex_incremental, innobackupex_full, full_backup, incremental_backup])
-      path_list.append(backup_controller)
-    # TODO: move to a separate recipe (ack'ed by Cedric)
-    # percona toolkit (formerly known as maatkit) installation
-    for pt_script_name in (
-        'pt-align',
-        'pt-archiver',
-        'pt-config-diff',
-        'pt-deadlock-logger',
-        'pt-diskstats',
-        'pt-duplicate-key-checker',
-        'pt-fifo-split',
-        'pt-find',
-        'pt-fingerprint',
-        'pt-fk-error-logger',
-        'pt-heartbeat',
-        'pt-index-usage',
-        'pt-ioprofile',
-        'pt-kill',
-        'pt-mext',
-        'pt-mysql-summary',
-        'pt-online-schema-change',
-        'pt-pmp',
-        'pt-query-digest',
-        'pt-show-grants',
-        'pt-sift',
-        'pt-slave-delay',
-        'pt-slave-find',
-        'pt-slave-restart',
-        'pt-stalk',
-        'pt-summary',
-        'pt-table-checksum',
-        'pt-table-sync',
-        'pt-table-usage',
-        'pt-upgrade',
-        'pt-variable-advisor',
-        'pt-visual-explain',
-        ):
-      option_name = pt_script_name + '-binary'
-      if option_name not in self.options:
-        continue
-      pt_argument_list = [self.options['perl-binary'],
-          self.options[option_name],
-          '--defaults-file=%s' % mysql_conf_file,
-          '--socket=%s' % socket.strip(), '--user=root',
-          ]
-      pt_exe = self.createWrapper(
-        os.path.join(self.options['bin-directory'], pt_script_name),
-        pt_argument_list, environment)
-      path_list.append(pt_exe)
-
-    return path_list
 
 class WrapUpdateMySQL(GenericBaseRecipe):
   def install(self):
@@ -239,19 +39,3 @@ class WrapUpdateMySQL(GenericBaseRecipe):
         }]
       ),
     ]
-
-class WrapMySQLd(GenericBaseRecipe):
-  def install(self):
-    return [
-      self.createPythonScript(
-        self.options['output'],
-        __name__ + '.mysql.runMysql',
-        [{
-          'mysqld_binary': self.options['binary'],
-          'configuration_file': self.options['configuration-file'],
-          'data_directory': self.options['data-directory'],
-          'mysql_install_binary': self.options['mysql-install-binary'],
-          'mysql_base_directory': self.options['mysql-base-directory'],
-        }]
-      ),
-    ]

slapos/recipe/generic_mysql/innobackupex.py

-import os
-import glob
-def controller(innobackupex_incremental, innobackupex_full,
-               full_backup, incremental_backup):
-  """Creates full or incremental backup
-
-  If no full backup is done, it is created
-
-  If full backup exists incremental backup is done starting with base
-
-  base is the newest (according to date) full or incremental backup
-  """
-  if len(os.listdir(full_backup)) == 0:
-    print 'Doing full backup in %r' % full_backup
-    os.execv(innobackupex_full, [innobackupex_full, full_backup])
-  else:
-    backup_list = filter(os.path.isdir, glob.glob(full_backup + "/*") +
-      glob.glob(incremental_backup + "/*"))
-    backup_list.sort(key=lambda x: os.path.getmtime(x), reverse=True)
-    base = backup_list[0]
-    print 'Doing incremental backup in %r using %r as a base' % (
-        incremental_backup, base)
-    os.execv(innobackupex_incremental, [innobackupex_incremental,
-      '--incremental-basedir=%s'%base, incremental_backup])

slapos/recipe/generic_mysql/mysql.py

@@ -4,55 +4,6 @@ import time
 import sys
 import pytz
 
-
-def runMysql(conf):
-  sleep = 60
-  mysqld_wrapper_list = [conf['mysqld_binary'], '--defaults-file=%s' %
-      conf['configuration_file']]
-  # we trust mysql_install that if mysql directory is available mysql was
-  # correctly initalised
-  if not os.path.isdir(os.path.join(conf['data_directory'], 'mysql')):
-    while True:
-      # XXX: Protect with proper root password
-      # XXX: Follow http://dev.mysql.com/doc/refman/5.0/en/default-privileges.html
-      popen = subprocess.Popen([conf['mysql_install_binary'],
-        '--defaults-file=%s' % conf['configuration_file'],
-        '--skip-name-resolve',
-        '--datadir=%s' % conf['data_directory'],
-        '--basedir=%s' % conf['mysql_base_directory']],
-        stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-      result = popen.communicate()[0]
-      if popen.returncode is None or popen.returncode != 0:
-        print "Failed to initialise server.\nThe error was: %s" % result
-        print "Waiting for %ss and retrying" % sleep
-        time.sleep(sleep)
-      else:
-        print "Mysql properly initialised"
-        break
-  else:
-    print "MySQL already initialised"
-  print "Starting %r" % mysqld_wrapper_list[0]
-  sys.stdout.flush()
-  sys.stderr.flush()
-  # try to increase the maximum number of open file descriptors.
-  # it seems that mysqld requires (max_connections + 810) file descriptors.
-  # to make it possible, you need to set the hard limit of nofile in
-  # /etc/security/limits.conf like the following :
-  #   @slapsoft hard nofile 2048
-  try:
-    import resource
-    required_nofile = 2048 # XXX hardcoded value more than 1000 + 810
-    nofile_limit_list = [max(x, required_nofile) for x in resource.getrlimit(resource.RLIMIT_NOFILE)]
-    resource.setrlimit(resource.RLIMIT_NOFILE, nofile_limit_list)
-  except ImportError:
-    # resource library is only available on Unix platform.
-    pass
-  except ValueError:
-    # 'ValueError: not allowed to raise maximum limit'
-    pass
-  os.execl(mysqld_wrapper_list[0], *mysqld_wrapper_list)
-
-
 def updateMysql(conf):
   sleep = 30
   is_succeed = False

slapos/recipe/generic_mysql/template/initmysql.sql.in

-CREATE DATABASE IF NOT EXISTS %(mysql_database)s;
-GRANT ALL PRIVILEGES ON %(mysql_database)s.* TO %(mysql_user)s@'%%' IDENTIFIED BY '%(mysql_password)s';
-GRANT ALL PRIVILEGES ON %(mysql_database)s.* TO %(mysql_user)s@'localhost' IDENTIFIED BY '%(mysql_password)s';

slapos/recipe/generic_mysql/template/my.cnf.in

-# ERP5 buildout my.cnf template based on my-huge.cnf shipped with mysql
-# The MySQL server
-[mysqld]
-# ERP5 by default requires InnoDB storage. MySQL by default fallbacks to using
-# different engine, like MyISAM. Such behaviour generates problems only, when
-# tables requested as InnoDB are silently created with MyISAM engine.
-#
-# Loud fail is really required in such case.
-sql-mode="NO_ENGINE_SUBSTITUTION"
-
-skip-show-database
-%(networking)s
-socket = %(socket)s
-datadir = %(data_directory)s
-pid-file = %(pid_file)s
-log-error = %(error_log)s
-slow_query_log
-slow_query_log_file = %(slow_query_log)s
-long_query_time = 1
-max_allowed_packet = 128M
-query_cache_size = 0
-query_cache_type = 0
-innodb_file_per_table = 0
-
-plugin-load = ha_mroonga.so;handlersocket.so
-
-# By default only 100 connections are allowed, when using zeo
-# we may have much more connections
-max_connections = 1000
-
-# The following are important to configure and depend a lot on to the size of
-# your database and the available resources.
-#innodb_buffer_pool_size = 4G
-#innodb_log_file_size = 256M
-#innodb_log_buffer_size = 8M
-
-# very important to allow parallel indexing
-innodb_locks_unsafe_for_binlog = 1
-
-# Some dangerous settings you may want to uncomment if you only want
-# performance or less disk access. Useful for unit tests.
-#innodb_flush_log_at_trx_commit = 0
-#innodb_flush_method = nosync
-#innodb_doublewrite = 0
-#sync_frm = 0
-
-%(log_bin)s
-%(expire_logs_days)s
-
-# Force utf8 usage
-collation_server = utf8_unicode_ci
-character_set_server = utf8
-skip-character-set-client-handshake
-
-[mysql]
-no-auto-rehash
-socket = %(socket)s
-
-[mysqlhotcopy]
-interactive-timeout

slapos/recipe/generic_mysql/template/mysql-init-function.sql.in

-USE mysql;
-DROP FUNCTION IF EXISTS last_insert_grn_id;
-DROP FUNCTION IF EXISTS mroonga_snippet;
-DROP FUNCTION IF EXISTS mroonga_command;
-DROP FUNCTION IF EXISTS sphinx_snippets;
-%(udf_registration)s

slapos/test/utils.py

@@ -54,16 +54,17 @@ def makeRecipe(recipe_class, options, name='test', slap_connection=None):
   if os.path.exists(buildout_cfg):
     parser = ConfigParser()
     parser.readfp(open(buildout_cfg))
-    eggs_directory = parser.get(
-        'buildout',
-        'eggs-directory',
-        # default, for the case when buildout_cfg is a software buildout
-        # like with SLAPOS-SR-TEST.
-        vars={'eggs-directory': os.path.join(base_directory, 'eggs')})
-    develop_eggs_directory = parser.get(
-        'buildout',
-        'develop-eggs-directory',
-        vars={'develop-eggs-directory': os.path.join(base_directory, 'develop-eggs')})
+    if parser.has_option('buildout', 'eggs-directory'):
+      # when buildout_cfg is an instance buildout (like in SLAPOS-EGG-TEST),
+      # there's a ${buildout:eggs-directory} we can use.
+      eggs_directory = parser.get('buildout', 'eggs-directory')
+      develop_eggs_directory = parser.get('buildout', 'develop-eggs-directory')
+    else:
+      # when when buildout_cfg is a software buildout, we can only guess the
+      # standard eggs directories.
+      eggs_directory = os.path.join(base_directory, 'eggs')
+      develop_eggs_directory = os.path.join(base_directory, 'develop-eggs')
+
     logging.getLogger(__name__).info(
         'Using eggs-directory (%s) and develop-eggs-directory (%s) from buildout at %s',
         eggs_directory,

software/apache-frontend/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = f686f765e55d1dce2e55a400f0714b3e
 
 [template-apache-frontend]
 filename = instance-apache-frontend.cfg
-md5sum = a6b566a29f1b5021d0f1f3c4fa20d749
+md5sum = d6398d727eb1e1bc3df1768a9b9a7e0c
 
 [template-apache-replicate]
 filename = instance-apache-replicate.cfg.in
@@ -38,7 +38,7 @@ md5sum = 665e83d660c9b779249b2179d7ce4b4e
 
 [template-apache-frontend-configuration]
 filename = templates/apache.conf.in
-md5sum = 05239181f4d5d0e3fe6bccda587fa9a5
+md5sum = b666d7c4a5c1fd8020713aa53b44a386
 
 [template-custom-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in

software/apache-frontend/templates/apache.conf.in

@@ -20,9 +20,6 @@ TypesConfig {{ httpd_home }}/conf/mime.types
 AddType application/x-compress .Z
 AddType application/x-gzip .gz .tgz
 
-# As backend is trusting REMOTE_USER header unset it always
-RequestHeader unset REMOTE_USER
-
 ServerTokens Prod
 
 # Disable TRACE Method

software/caddy-frontend/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg.in
-md5sum = 4832bb055d31be6e99e2ef890b2206b0
+md5sum = d8ce8da7ea7d82c33958bdbabbaad956
 
 [template-common]
 filename = instance-common.cfg.in
@@ -22,7 +22,7 @@ md5sum = c801b7f9f11f0965677c22e6bbe9281b
 
 [template-apache-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = 74f730a4fb079416118bd412a458cea8
+md5sum = 2903758a104186b7dae9573c3470be78
 
 [template-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
@@ -30,7 +30,7 @@ md5sum = 491a19d1747bbf795c27b094cf67114d
 
 [template-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = 13338a7844f5a4b749f6647ba8163a8d
+md5sum = c33df53e7752f43b89c5fda7e92a5a78
 
 [template-slave-configuration]
 filename = templates/custom-virtualhost.conf.in
@@ -42,7 +42,7 @@ md5sum = eb9ca67763d60843483d95dab2c301b1
 
 [template-caddy-frontend-configuration]
 filename = templates/Caddyfile.in
-md5sum = dfec964a9f194293567b09d0f10e4b3d
+md5sum = 908b859ff76469381024947f5c98c891
 
 [caddy-backend-url-validator]
 filename = templates/caddy-backend-url-validator.in
@@ -54,11 +54,11 @@ md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
 
 [template-default-slave-virtualhost]
 filename = templates/default-virtualhost.conf.in
-md5sum = 7e21418a03529db22181962ea804da53
+md5sum = 9a984febd7fa14a4ea94599f3e83139c
 
 [template-cached-slave-virtualhost]
 filename = templates/cached-virtualhost.conf.in
-md5sum = 6ca9a3251830d602cf25e0a0389fc74b
+md5sum = a73839d777fbd548286bbeccf47be335
 
 [template-log-access]
 filename = templates/template-log-access.conf.in
@@ -74,7 +74,7 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
 
 [template-trafficserver-records-config]
 filename = templates/trafficserver/records.config.jinja2
-md5sum = 5ef0ebc37437ada7cc176e663da5f36c
+md5sum = 3c342b0388f94f819b04b05b46744427
 
 [template-trafficserver-storage-config]
 filename = templates/trafficserver/storage.config.jinja2

software/caddy-frontend/instance-apache-frontend.cfg.in

@@ -264,6 +264,7 @@ extra-context =
     key enable_http2_by_default configuration:enable-http2-by-default
     key global_disable_http2 configuration:global-disable-http2
     key ciphers configuration:ciphers
+    key request_timeout configuration:request-timeout
     key proxy_try_duration configuration:proxy-try-duration
     key proxy_try_interval configuration:proxy-try-interval
     key access_log caddy-configuration:access-log
@@ -442,6 +443,7 @@ synthetic-port = ${configuration:trafficserver-synthetic-port}
 mgmt-port = ${configuration:trafficserver-mgmt-port}
 ram-cache-size = ${configuration:ram-cache-size}
 templates-dir = {{ parameter_dict['trafficserver'] }}/etc/trafficserver/body_factory
+request-timeout = ${configuration:request-timeout}
 
 [trafficserver-configuration-directory]
 recipe = plone.recipe.command

software/caddy-frontend/instance-caddy-input-schema.json

@@ -107,6 +107,12 @@
       "description": "List of ciphers. Empty defaults to Caddy list of ciphers. See https://caddyserver.com/docs/tls for more information.",
       "title": "Ordered space separated list of ciphers",
       "type": "string"
+    },
+    "request-timeout": {
+      "default": 600,
+      "description": "Timeout for HTTP requests.",
+      "title": "HTTP Request timeout in seconds",
+      "type": "integer"
     }
   },
   "title": "Input Parameters",

software/caddy-frontend/instance.cfg.in

@@ -116,6 +116,7 @@ configuration.re6st-verification-url = http://[2001:67c:1254:4::1]/index.html
 configuration.enable-http2-by-default = true
 configuration.global-disable-http2 = false
 configuration.ciphers =
+configuration.request-timeout = 600
 configuration.enable-quic = false
 configuration.mpm-graceful-shutdown-timeout = 5
 configuration.monitor-httpd-port = 8072

software/caddy-frontend/templates/Caddyfile.in

@@ -7,8 +7,6 @@ import {{ slave_with_cache_configuration_directory }}/*.conf
 :{{ https_port }} {
   tls {{ master_certificate }} {{ master_certificate }}
   bind {{ local_ipv4 }}
-  # Compress the output
-  gzip
   status 404 /
   log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
     rotate_size 0
@@ -21,8 +19,6 @@ import {{ slave_with_cache_configuration_directory }}/*.conf
 
 :{{ http_port }} {
   bind {{ local_ipv4 }}
-  # Compress the output
-  gzip
   status 404 /
   log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
     rotate_size 0
@@ -36,8 +32,6 @@ import {{ slave_with_cache_configuration_directory }}/*.conf
 # Access to server-status Caddy-style
 https://[{{ global_ipv6 }}]:{{ https_port }}/server-status, https://{{ local_ipv4 }}:{{ https_port }}/server-status {
   tls {{ frontend_configuration['ip-access-certificate'] }} {{ frontend_configuration['ip-access-certificate'] }}
-  # Compress the output
-  gzip
   bind {{ local_ipv4 }}
   basicauth "{{ username }}" {{ password | trim }} {
     "Server Status"

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -241,6 +241,7 @@ http_port = {{ dumps('' ~ http_port) }}
 local_ipv4 = {{ dumps('' ~ local_ipv4) }}
 cached_port = {{ dumps('' ~ cached_port) }}
 ssl_cached_port = {{ ('' ~ ssl_cached_port) }}
+request_timeout = {{ ('' ~ request_timeout) }}
 {# BBB: apache_custom_https and apache_custom_http #}
 {%     set caddy_custom_http_template = slave_instance.pop('caddy_custom_http', slave_instance.pop('apache_custom_http', '')) %}
 {%     set caddy_custom_https_template = slave_instance.pop('caddy_custom_https', slave_instance.pop('apache_custom_https', '')) %}

software/caddy-frontend/templates/cached-virtualhost.conf.in

@@ -17,17 +17,13 @@
 # SSL-disabled backends
 {{ http_backend_host_list|join(', ') }} {
   bind {{ slave_parameter['local_ipv4'] }}
-  # Compress the output
-  gzip
 # Rewrite part
   proxy / {{ slave_parameter.get('backend_url', '') }} {
     try_duration {{ slave_parameter['proxy_try_duration'] }}s
     try_interval {{ slave_parameter['proxy_try_interval'] }}ms
-    # As backend is trusting REMOTE_USER header unset it always
-    header_upstream -REMOTE_USER
 
     transparent
-    timeout 600s
+    timeout {{ slave_parameter['request_timeout'] }}s
 {%- if ssl_proxy_verify %}
 {%-   if 'path_to_ssl_proxy_ca_crt' in slave_parameter %}
     ca_certificates {{ slave_parameter['path_to_ssl_proxy_ca_crt'] }}
@@ -48,15 +44,11 @@
 # SSL-enabled backends
 {{ https_backend_host_list|join(', ') }} {
   bind {{ slave_parameter['local_ipv4'] }}
-  # Compress the output
-  gzip
   proxy / {{ slave_parameter.get('https_backend_url', '') }} {
     try_duration {{ slave_parameter['proxy_try_duration'] }}s
     try_interval {{ slave_parameter['proxy_try_interval'] }}ms
-    # As backend is trusting REMOTE_USER header unset it always
-    header_upstream -REMOTE_USER
     transparent
-    timeout 600s
+    timeout {{ slave_parameter['request_timeout'] }}s
 {%- if ssl_proxy_verify %}
 {%-   if 'path_to_ssl_proxy_ca_crt' in slave_parameter %}
     ca_certificates {{ slave_parameter['path_to_ssl_proxy_ca_crt'] }}

software/caddy-frontend/templates/default-virtualhost.conf.in

@@ -50,8 +50,6 @@
 {{ http_host_list|join(', ') }} {
 {%- endif %}
   bind {{ slave_parameter['local_ipv4'] }}
-  # Compress the output
-  gzip
 {%- if tls %}
   tls {{ slave_parameter['certificate'] }} {{ slave_parameter['certificate'] }} {
 {%- if cipher_list %}
@@ -110,8 +108,6 @@
     without /prefer-gzip
     header_upstream Accept-Encoding gzip
 {%-     endif %} {#-     if proxy_name == 'prefer-gzip' #}
-    # As backend is trusting REMOTE_USER header unset it always
-    header_upstream -REMOTE_USER
 {%- for disabled_cookie in disabled_cookie_list %}
     # Remove cookie {{ disabled_cookie }} from client Cookies
     header_upstream Cookie "(.*)(^{{ disabled_cookie }}=[^;]*; |; {{ disabled_cookie }}=[^;]*|^{{ disabled_cookie }}=[^;]*$)(.*)" "$1 $3"
@@ -126,7 +122,7 @@
     header_upstream -Pragma
 {%-   endif %} {#-   if disable_no_cache_header #}
     transparent
-    timeout 600s
+    timeout {{ slave_parameter['request_timeout'] }}s
 {%-   if ssl_proxy_verify %}
 {%-     if 'path_to_ssl_proxy_ca_crt' in slave_parameter %}
     ca_certificates {{ slave_parameter['path_to_ssl_proxy_ca_crt'] }}
@@ -247,8 +243,6 @@
     without /prefer-gzip
     header_upstream Accept-Encoding gzip
 {%-     endif %} {#-     if proxy_name == 'prefer-gzip' #}
-    # As backend is trusting REMOTE_USER header unset it always
-    header_upstream -REMOTE_USER
 {%- for disabled_cookie in disabled_cookie_list %}
     # Remove cookie {{ disabled_cookie }} from client Cookies
     header_upstream Cookie "(.*)(^{{ disabled_cookie }}=[^;]*; |; {{ disabled_cookie }}=[^;]*|^{{ disabled_cookie }}=[^;]*$)(.*)" "$1 $3"
@@ -263,7 +257,7 @@
     header_upstream -Pragma
 {%-     endif %} {#-     if disable_no_cache_header #}
     transparent
-    timeout 600s
+    timeout {{ slave_parameter['request_timeout'] }}s
 {%-     if ssl_proxy_verify %}
 {%-       if 'path_to_ssl_proxy_ca_crt' in slave_parameter %}
     ca_certificates {{ slave_parameter['path_to_ssl_proxy_ca_crt'] }}

software/caddy-frontend/templates/trafficserver/records.config.jinja2

@@ -68,8 +68,8 @@ CONFIG proxy.config.http.uncacheable_requests_bypass_parent INT 1
 ##############################################################################
 CONFIG proxy.config.http.keep_alive_no_activity_timeout_in INT 120
 CONFIG proxy.config.http.keep_alive_no_activity_timeout_out INT 120
-CONFIG proxy.config.http.transaction_no_activity_timeout_in INT 30
-CONFIG proxy.config.http.transaction_no_activity_timeout_out INT 30
+CONFIG proxy.config.http.transaction_no_activity_timeout_in INT {{ ats_configuration['request-timeout'] }}
+CONFIG proxy.config.http.transaction_no_activity_timeout_out INT {{ ats_configuration['request-timeout'] }}
 CONFIG proxy.config.http.transaction_active_timeout_in INT 900
 CONFIG proxy.config.http.transaction_active_timeout_out INT 0
 CONFIG proxy.config.http.accept_no_activity_timeout INT 120
@@ -79,11 +79,13 @@ CONFIG proxy.config.net.default_inactivity_timeout INT 86400
 # Origin server connect attempts. Docs:
 #    https://docs.trafficserver.apache.org/records.config#origin-server-connect-attempts
 ##############################################################################
-CONFIG proxy.config.http.connect_attempts_max_retries INT 3
-CONFIG proxy.config.http.connect_attempts_max_retries_dead_server INT 1
+# Try only once to connect (do not retry)
+CONFIG proxy.config.http.connect_attempts_max_retries INT 0
+# Try only once with server marked dead (do not retry)
+CONFIG proxy.config.http.connect_attempts_max_retries_dead_server INT 0
 CONFIG proxy.config.http.connect_attempts_rr_retries INT 3
-CONFIG proxy.config.http.connect_attempts_timeout INT 30
-CONFIG proxy.config.http.post_connect_attempts_timeout INT 1800
+CONFIG proxy.config.http.connect_attempts_timeout INT {{ ats_configuration['request-timeout'] }}
+CONFIG proxy.config.http.post_connect_attempts_timeout INT {{ ats_configuration['request-timeout'] }}
 CONFIG proxy.config.http.down_server.cache_time INT 60
 CONFIG proxy.config.http.down_server.abort_threshold INT 10
 

software/caddy-frontend/test/test.py

@@ -42,6 +42,12 @@ from forcediphttpsadapter.adapters import ForcedIPHTTPSAdapter
 import time
 import tempfile
 import ipaddress
+import StringIO
+import gzip
+import base64
+import re
+
+
 try:
     import lzma
 except ImportError:
@@ -652,9 +658,13 @@ class TestMasterRequestDomain(HttpFrontendTestCase, TestDataMixin):
 class TestHandler(BaseHTTPRequestHandler):
   def do_GET(self):
     timeout = int(self.headers.dict.get('timeout', '0'))
+    compress = int(self.headers.dict.get('compress', '0'))
     time.sleep(timeout)
     self.send_response(200)
 
+    drop_header_list = []
+    for header in self.headers.dict.get('x-drop-header', '').split():
+      drop_header_list.append(header)
     prefix = 'x-reply-header-'
     length = len(prefix)
     for key, value in self.headers.dict.items():
@@ -664,15 +674,33 @@ class TestHandler(BaseHTTPRequestHandler):
           value.strip()
         )
 
-    self.send_header("Content-type", "application/json")
-    self.send_header('Set-Cookie', 'secured=value;secure')
-    self.send_header('Set-Cookie', 'nonsecured=value')
+    if 'Content-Type' not in drop_header_list:
+      self.send_header("Content-Type", "application/json")
+    if 'Set-Cookie' not in drop_header_list:
+      self.send_header('Set-Cookie', 'secured=value;secure')
+      self.send_header('Set-Cookie', 'nonsecured=value')
+
+    if 'x-reply-body' not in self.headers.dict:
+      response = {
+        'Path': self.path,
+        'Incoming Headers': self.headers.dict
+      }
+      response = json.dumps(response, indent=2)
+    else:
+      response = base64.b64decode(self.headers.dict['x-reply-body'])
+    if compress:
+      self.send_header('Content-Encoding', 'gzip')
+      out = StringIO.StringIO()
+      # compress with level 0, to find out if in the middle someting would
+      # like to alter the compression
+      with gzip.GzipFile(fileobj=out, mode="w", compresslevel=0) as f:
+        f.write(response)
+      response = out.getvalue()
+      self.send_header('Backend-Content-Length', len(response))
+    if 'Content-Length' not in drop_header_list:
+      self.send_header('Content-Length', len(response))
     self.end_headers()
-    response = {
-      'Path': self.path,
-      'Incoming Headers': self.headers.dict
-    }
-    self.wfile.write(json.dumps(response, indent=2))
+    self.wfile.write(response)
 
 
 class SlaveHttpFrontendTestCase(HttpFrontendTestCase):
@@ -856,7 +884,6 @@ class SlaveHttpFrontendTestCase(HttpFrontendTestCase):
                       headers=None, cookies=None, source_ip=None):
     if headers is None:
       headers = {}
-    headers.setdefault('REMOTE_USER', 'SOME_REMOTE_USER')
     # workaround request problem of setting Accept-Encoding
     # https://github.com/requests/requests/issues/2234
     headers.setdefault('Accept-Encoding', 'dummy')
@@ -881,7 +908,6 @@ class SlaveHttpFrontendTestCase(HttpFrontendTestCase):
                      headers=None):
     if headers is None:
       headers = {}
-    headers.setdefault('REMOTE_USER', 'SOME_REMOTE_USER')
     # workaround request problem of setting Accept-Encoding
     # https://github.com/requests/requests/issues/2234
     headers.setdefault('Accept-Encoding', 'dummy')
@@ -1036,6 +1062,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'kedifa_port': KEDIFA_PORT,
       'caucase_port': CAUCASE_PORT,
       'mpm-graceful-shutdown-timeout': 2,
+      'request-timeout': '12',
     }
 
   @classmethod
@@ -1448,7 +1475,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         self.instance_path, '*', 'var', 'log', 'httpd', '_empty_access_log'
       ))[0]
 
-    log_regexp = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3} - SOME_REMOTE_USER ' \
+    log_regexp = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3} - - ' \
                  r'\[\d{2}\/.{3}\/\d{4}\:\d{2}\:\d{2}\:\d{2} \+\d{4}\] ' \
                  r'"GET \/test-path HTTP\/1.1" 404 \d+ "-" '\
                  r'"python-requests.*" \d+'
@@ -1485,7 +1512,10 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     result = self.fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'],
       'test-path/deep/.././deeper',
-      headers={'Timeout': '10'}  # more than default proxy-try-duration == 5
+      headers={
+        'Timeout': '10',  # more than default proxy-try-duration == 5
+        'Accept-Encoding': 'gzip',
+      }
     )
 
     self.assertEqual(
@@ -1535,6 +1565,63 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       self.assertTrue('try_duration 5s' in content)
       self.assertTrue('try_interval 250ms' in content)
 
+  def test_compressed_result(self):
+    parameter_dict = self.assertSlaveBase('Url')
+    result_compressed = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'],
+      'test-path/deep/.././deeper',
+      headers={
+        'Accept-Encoding': 'gzip',
+        'Compress': '1',
+      }
+    )
+    self.assertEqual(
+      'gzip',
+      result_compressed.headers['Content-Encoding']
+    )
+
+    # Assert that no tampering was done with the request
+    # (compression/decompression)
+    # Backend compresses with 0 level, so decompression/compression
+    # would change somthing
+    self.assertEqual(
+      result_compressed.headers['Content-Length'],
+      result_compressed.headers['Backend-Content-Length']
+    )
+
+    result_not_compressed = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'],
+      'test-path/deep/.././deeper',
+      headers={
+        'Accept-Encoding': 'gzip',
+      }
+    )
+    self.assertFalse('Content-Encoding' in result_not_compressed.headers)
+
+  def test_no_content_type_alter(self):
+    parameter_dict = self.assertSlaveBase('Url')
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'],
+      'test-path/deep/.././deeper',
+      headers={
+        'Accept-Encoding': 'gzip',
+        'X-Reply-Body': base64.b64encode(
+          b"""<?xml version="1.0" encoding="UTF-8"?>
+<note>
+  <to>Tove</to>
+  <from>Jani</from>
+  <heading>Reminder</heading>
+  <body>Don't forget me this weekend!</body>
+</note>"""),
+        'X-Drop-Header': 'Content-Type'
+      }
+    )
+
+    self.assertEqual(
+      'text/xml; charset=utf-8',
+      result.headers['Content-Type']
+    )
+
   @skip('Feature postponed')
   def test_url_ipv6_access(self):
     parameter_dict = self.parseSlaveParameterDict('url')
@@ -3265,6 +3352,89 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       result_direct_https_backend.headers['Set-Cookie']
     )
 
+  def test_enable_cache_ats_timeout(self):
+    parameter_dict = self.assertSlaveBase('enable_cache')
+    # check that timeout seen by ATS does not result in many queries done
+    # to the backend and that next request works like a charm
+    result = self.fakeHTTPResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'],
+      'test_enable_cache_ats_timeout', headers={
+        'Timeout': '15',
+        'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
+        'revalidate=3600, stale-if-error=3600'})
+
+    # ATS timed out
+    self.assertEqual(
+      httplib.GATEWAY_TIMEOUT,
+      result.status_code
+    )
+
+    caddy_log_file = glob.glob(
+      os.path.join(
+        self.instance_path, '*', 'var', 'log', 'httpd-cache-direct',
+        '_enable_cache_access_log'
+      ))[0]
+
+    matching_line_amount = 0
+    pattern = re.compile(
+      r'.*GET .test_enable_cache_ats_timeout.*" 499.*')
+    with open(caddy_log_file) as fh:
+      for line in fh.readlines():
+        if pattern.match(line):
+          matching_line_amount += 1
+
+    # Caddy used between ATS and the backend received only one connection
+    self.assertEqual(
+      1,
+      matching_line_amount)
+
+    timeout = 5
+    b = time.time()
+    # ATS created squid.log with a delay
+    while True:
+      if (time.time() - b) > timeout:
+        self.fail('Squid log file did not appear in %ss' % (timeout,))
+      ats_log_file_list = glob.glob(
+        os.path.join(
+          self.instance_path, '*', 'var', 'log', 'trafficserver', 'squid.log'
+        ))
+      if len(ats_log_file_list) == 1:
+        ats_log_file = ats_log_file_list[0]
+        break
+      time.sleep(0.1)
+
+    pattern = re.compile(
+      r'.*ERR_READ_TIMEOUT/504 .*test_enable_cache_ats_timeout'
+      '.*TIMEOUT_DIRECT*')
+    timeout = 5
+    b = time.time()
+    # ATS needs some time to flush logs
+    while True:
+      matching_line_amount = 0
+      if (time.time() - b) > timeout:
+        break
+      with open(ats_log_file) as fh:
+        for line in fh.readlines():
+          if pattern.match(line):
+            matching_line_amount += 1
+      if matching_line_amount > 0:
+        break
+      time.sleep(0.1)
+
+    # ATS has only one entry for this query
+    self.assertEqual(
+      1,
+      matching_line_amount)
+
+    # the result is available immediately after
+    result = self.fakeHTTPResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'],
+      'test-path/deep/.././deeper', headers={
+        'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
+        'revalidate=3600, stale-if-error=3600'})
+
+    self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
+
   def test_enable_cache_disable_no_cache_request(self):
     parameter_dict = self.assertSlaveBase(
       'enable_cache-disable-no-cache-request')

software/neoppod/cluster.cfg.in

 {% import "root_common" as common_macro with context %}
 
-[request-common]
-<= request-common-base
-
 {{ common_macro.request_neo(slapparameter_dict, 'neo', 'node-') }}
+
+[publish-early]
+recipe = slapos.cookbook:publish-early
+-init =
+  neo-admins node-cluster:admins
+  neo-masters node-cluster:masters
+
 [publish]
 recipe = slapos.cookbook:publish.serialised
-neo-masters = ${node-0-final:connection-masters}
-neo-admins = ${node-0-final:connection-admins}
+-extends = publish-early
+
 {{ common_macro.common_section() }}

software/neoppod/instance-common.cfg.in

@@ -42,5 +42,6 @@ extra-context =
     key admin_cfg neo-admin:rendered
 {%- if mariadb_location is defined %}
     raw mariadb_location {{ mariadb_location }}
+    raw template_mysqld_wrapper {{ template_mysqld_wrapper }}
     raw template_neo_my_cnf {{ template_neo_my_cnf }}
 {%- endif %}

software/neoppod/instance-neo-admin.cfg.in

@@ -13,7 +13,7 @@ ip = ${publish:ip}
 port = ${publish:port-admin}
 ssl = {{ dumps(bool(slapparameter_dict['ssl'])) }}
 cluster = {{ dumps(slapparameter_dict['cluster']) }}
-masters = ${publish:masters}
+masters = {{ dumps(slapparameter_dict['masters']) }}
 
 [neo-admin-promise]
 recipe = slapos.cookbook:check_port_listening

software/neoppod/instance-neo-master.cfg.in

@@ -21,7 +21,7 @@ autostart = {{ slapparameter_dict['autostart'] }}
 # No actual installation takes place at that time
 # (slapos.cookbook:neoppod.master raises), but cfg expansion must succeed. So
 # this default value is required.
-masters = ${publish:masters}
+masters = {{ dumps(slapparameter_dict['masters']) }}
 
 [neo-master-promise]
 recipe = slapos.cookbook:check_port_listening

software/neoppod/instance-neo.cfg.in

@@ -7,18 +7,17 @@
 {% set mysql = storage_type == 'MySQL' -%}
 {% if mysql -%}
 
-[mysqld]
-recipe = slapos.cookbook:generic.mysql.wrap_mysqld
-output = ${directory:etc_run}/mariadb
-binary = ${:mysql-base-directory}/bin/mysqld
-configuration-file = ${my-cnf:rendered}
-data-directory = ${directory:srv_mariadb}
-mysql-install-binary = ${:mysql-base-directory}/scripts/mysql_install_db
-mysql-base-directory = {{ mariadb_location }}
+[{{ section('mysqld') }}]
+recipe = slapos.recipe.template:jinja2
+template = {{ template_mysqld_wrapper }}
+rendered = ${directory:etc_run}/mariadb
+context =
+  key defaults_file my-cnf:rendered
+  key datadir my-cnf-parameters:data-directory
 
 [my-cnf-parameters]
 socket = ${directory:var_run}/mariadb.sock
-data-directory = ${mysqld:data-directory}
+data-directory = ${directory:srv}/mariadb
 tmp-directory = ${directory:tmp}
 pid-file = ${directory:var_run}/mariadb.pid
 error-log = ${directory:log}/mariadb_error.log
@@ -34,9 +33,9 @@ rendered = ${directory:etc}/mariadb.cnf
 template = {{ template_neo_my_cnf }}
 context = section parameter_dict my-cnf-parameters
 
-[{{ section('binary-wrap-mysql') }}]
+[binary-wrap-mysql]
 recipe = slapos.cookbook:wrapper
-command-line = ${mysqld:mysql-base-directory}/bin/${:command} --defaults-file=${my-cnf:rendered}
+command-line = '{{ mariadb_location }}/bin/${:command}' --defaults-file="${my-cnf:rendered}"
 wrapper-path = ${directory:bin}/${:command}
 command = mysql
 
@@ -47,17 +46,6 @@ recipe = slapos.cookbook:symbolic.link
 target-directory = ${directory:bin}
 link-binary = {{ bin_directory }}/neolog
 
-{% set master_list = [] -%}
-{% set admin_list = [] -%}
-{% for k, v in slapparameter_dict.iteritems() -%}
-{%   if k.startswith('master-') and v -%}
-{%     do master_list.append(v) -%}
-{%   endif -%}
-{%   if k.startswith('admin-') and v -%}
-{%     do admin_list.append(v) -%}
-{%   endif -%}
-{% endfor -%}
-
 [publish]
 recipe = slapos.cookbook:publish.serialised
 # TODO: make port a partition parameter
@@ -76,10 +64,6 @@ admin = ${:ip}:${:port-admin}
 {% else -%}
 admin =
 {% endif -%}
-masters = {{ ' '.join(sorted(master_list)) }}
-{%- if admin_list %}
-admins = {{ ' '.join(sorted(admin_list)) }}
-{%- endif %}
 
 {#- Hack to deploy SSL certs via instance parameters #}
 {%- for name, pem in zip(('ca.crt', 'neo.crt', 'neo.key'),
@@ -102,7 +86,7 @@ binary = {{ bin_directory }}/neostorage
 ip = ${publish:ip}
 ssl = {{ dumps(bool(slapparameter_dict['ssl'])) }}
 cluster = {{ dumps(slapparameter_dict['cluster']) }}
-masters = ${publish:masters}
+masters = {{ dumps(slapparameter_dict['masters']) }}
 database-adapter = {{ storage_type }}
 wait-database = -1
 engine = ${my-cnf-parameters:engine}
@@ -122,7 +106,7 @@ database-parameters = root@neo{{ i }}${my-cnf-parameters:socket}
 database-parameters = ${directory:db-{{i}}}/db.sqlite
 
 [directory]
-db-{{i}} = ${buildout:directory}/srv/{{ storage_id }}
+db-{{i}} = ${:srv}/{{ storage_id }}
 {%- endif %}
 
 [{{ section('logrotate-storage-' ~ i) }}]
@@ -143,9 +127,9 @@ etc_run =  ${:etc}/run
 var_run =  ${:var}/run
 log = ${buildout:directory}/var/log
 tmp = ${buildout:directory}/tmp
-{% if mysql -%}
-srv_mariadb = ${buildout:directory}/srv/mariadb
+srv = ${buildout:directory}/srv
 
+{% if mysql -%}
 [init-script]
 recipe = slapos.recipe.template:jinja2
 # XXX: is there a better location ?
@@ -157,7 +141,7 @@ template = inline:
 < = logrotate-entry-base
 name = mariadb
 log = ${my-cnf-parameters:error-log} ${my-cnf-parameters:slow-query-log}
-post = ${mysqld:mysql-base-directory}/bin/mysql --defaults-file="${my-cnf:rendered}" -e "FLUSH LOGS"
+post = ${binary-wrap-mysql:command-line} -e "FLUSH LOGS"
 
 {% if runTestSuite_in is defined -%}
 # bin/runTestSuite to run NEO tests
@@ -170,7 +154,7 @@ context =
     section directory         directory
     section my_cnf_parameters my-cnf-parameters
     raw     bin_directory     {{ bin_directory }}
-    raw     prepend_path      ${mysqld:mysql-base-directory}/bin
+    raw     prepend_path      {{ mariadb_location }}/bin
 {%- endif %}
 
 {%- endif %}

software/neoppod/root-common.cfg.in

@@ -39,6 +39,10 @@ parts =
 {% set section_id_list = [] -%}
 [{{ prefix }}request-common]
 <= request-common-base
+return =
+  master
+  admin
+config-masters = {{ '${' ~ prefix ~ 'cluster:masters}' }}
 config-cluster = {{ parameter_dict['cluster'] }}
 {% set replicas = parameter_dict.get('replicas', 0) -%}
 config-partitions = {{ dumps(parameter_dict.get('partitions', 12)) }}
@@ -62,35 +66,16 @@ config-autostart = {{ dumps(sum(storage_count)) }}
 {%   set section_id =  prefix ~ i -%}
 {%   do section_id_list.append(section_id) -%}
 
-[{{ section_id }}-base]
+[{{section_id}}]
+<= {{ prefix }}request-common
 name = {{ section_id }}
 {%   for k, v in node.iteritems() -%}
 config-{{ k }} = {{ dumps(v) }}
 {%   endfor -%}
 {{ sla(section_id) }}
-
-[{{ section_id }}]
-<= {{ prefix }}request-common
-  {{ section_id }}-base
-return =
-  master
-  admin
-{% endfor -%}
-[final-base]
-{% for i, section_id in enumerate(section_id_list) -%}
-config-master-{{i}} = {{ '${' + section_id + ':connection-master}' }}
-config-admin-{{i}} = {{ '${' + section_id + ':connection-admin}' }}
 {% endfor -%}
 
-{% for section_id in section_id_list -%}
-[{{ section(section_id + '-final') }}]
-<= {{ prefix }}request-common
-  final-base
-  {{ section_id }}-base
-{%   if loop.first -%}
-return =
-  masters
-  admins
-{%   endif -%}
-{% endfor -%}
+[{{section(prefix ~ 'cluster')}}]
+recipe = slapos.cookbook:neoppod.cluster
+nodes = {{ ' '.join(section_id_list) }}
 {% endmacro -%}

software/neoppod/software-common.cfg

@@ -94,7 +94,7 @@ mode = 644
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/${:_buildout_section_name_}.cfg.in
 rendered = ${buildout:directory}/${:_buildout_section_name_}.cfg
-md5sum = c0e22816537b56bceef0b4c2b40f6219
+md5sum = 0a3a54fcc7be0bbd63cbd64f006ceebc
 context =
     key bin_directory buildout:bin-directory
     key develop_eggs_directory buildout:develop-eggs-directory
@@ -107,33 +107,56 @@ context =
     ${:adapter-context}
 adapter-context =
     key mariadb_location mariadb:location
+    key template_mysqld_wrapper template-mysqld-wrapper:rendered
     key template_neo_my_cnf template-neo-my-cnf:target
 
 [root-common]
 <= download-base-neo
-md5sum = 66055aa82f9097c5301864c07e6e5d80
+md5sum = 15fa47a59cc3019f59612aaf33bd9ec5
 
 [instance-neo-admin]
 <= download-base-neo
-md5sum = 4d1ae570b4458e7725454857aabb37f6
+md5sum = ce0d9ff9e899bb706351a99df29238a9
 
 [instance-neo-master]
 <= download-base-neo
-md5sum = 1fee10f02c2fa2a581e21878ca0fd704
+md5sum = 4faee020eaf7cd495cd6210dfa4eb0c1
 
 [instance-neo]
 <= download-base-neo
-md5sum = d4e30d74316e6931da4a1e305f9bbc68
+md5sum = 5fc9fcaec3a5387625af34fe686097ae
 
 [template-neo-my-cnf]
 <= download-base-neo
 url = ${:_profile_base_location_}/my.cnf.in
 md5sum = 9f6f8f2b5f4cb0d97d50ffc1d3837e2f
 
+[template-mysqld-wrapper]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/mysqld.in
+mode = 644
+template =
+  inline:{% raw %}#!/bin/sh -e
+  datadir='{{datadir}}'
+  [ -e "$datadir" ] || {
+    rm -vrf "$datadir.new"
+    '${mariadb:location}/scripts/mysql_install_db' \
+      --defaults-file='{{defaults_file}}' \
+      --skip-name-resolve \
+      --basedir='${mariadb:location}' \
+      --datadir="$datadir.new"
+    mv -v "$datadir.new" "$datadir"
+  }
+  exec '${mariadb:location}/bin/mysqld' \
+    --defaults-file='{{defaults_file}}' \
+    "$@"
+  {% endraw %}
+
 [versions]
 BTrees = 4.5.1
 ZODB = 4.4.5
 coverage = 4.5.1
+mock = 3.0.5
 ecdsa = 0.13
 gitdb2 = 2.0.0
 msgpack = 0.5.6
@@ -146,7 +169,7 @@ setproctitle = 1.1.10
 slapos.recipe.template = 4.3
 smmap2 = 2.0.1
 transaction = 1.7.0
-zodbpickle = 0.6.0
+zodbpickle = 1.0.4
 zodbtools = 0.0.0.dev4
 cython-zstd = 0.2
 python-dateutil = 2.7.3
@@ -164,10 +187,12 @@ ZEO = 4.3.1+SlapOSPatched001
 # ZEO==4.3.1
 zdaemon = 4.2.0
 
+# Required by:
+# mock = 3.0.5
+funcsigs = 1.0.2
 
 # Test Suite: NEO-MASTER ran at 2019/08/28 16:24:58.949371 UTC
 #             22 failures, 1 errors, 839 total, status: FAIL
 
 [neoppod-repository]
 revision = c681f666c191581551c9d63e1f302270fd6a343d
-

software/neoppod/software-zodb5.cfg

@@ -2,12 +2,13 @@
 extends = software.cfg
 
 [neoppod]
+eggs += mock
 ZEO-patches =
 
 [versions]
-ZODB = 5.4.0
+ZODB = 5.5.1
 ZEO = 5.2.0
-transaction = 2.2.1
+transaction = 2.4.0
 
 # Required by:
 # ZEO==5.2.0

software/neoppod/software.cfg

@@ -22,7 +22,7 @@ context =
 
 [cluster]
 <= download-base-neo
-md5sum = ee8401a4e7d82bf488a57e3399f9ce48
+md5sum = 5afd326de385563b5aeac81039f23341
 
 [runTestSuite.in]
 recipe = slapos.recipe.build:download

software/slapos-master/apache-backend.conf.in

@@ -131,13 +131,11 @@ SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:EC
 SSLSessionCache shmcb:{{ parameter_dict['ssl-session-cache'] }}(512000)
 SSLProxyEngine On
 
-# As backend is trusting REMOTE_USER header unset it always
-RequestHeader unset REMOTE_USER
-RequestHeader unset SSL_CLIENT_SERIAL
+# As backend is trusting Remote-User header unset it always
+RequestHeader unset Remote-User
 {% if parameter_dict['ca-cert'] -%}
 SSLVerifyClient optional
-RequestHeader set REMOTE_USER %{SSL_CLIENT_S_DN_CN}s
-RequestHeader set SSL_CLIENT_SERIAL "%{SSL_CLIENT_M_SERIAL}s"
+RequestHeader set Remote-User %{SSL_CLIENT_S_DN_CN}s
 SSLCACertificateFile {{ parameter_dict['ca-cert'] }}
 {%   if not parameter_dict['shared-ca-cert'] %}
 {%     if parameter_dict['crl'] -%}
@@ -168,7 +166,7 @@ Listen {{ ip }}:{{ port }}
 {% if enable_authentication and parameter_dict['shared-ca-cert'] and parameter_dict['shared-crl'] -%}
   SSLVerifyClient require
   # Custom block we use for now different parameters.
-  RequestHeader set REMOTE_USER %{SSL_CLIENT_S_DN_CN}s
+  RequestHeader set Remote-User %{SSL_CLIENT_S_DN_CN}s
   SSLCACertificateFile {{ parameter_dict['shared-ca-cert'] }}
   SSLCARevocationPath {{ parameter_dict['shared-crl'] }}
 

software/slapos-master/buildout.hash.cfg

@@ -22,4 +22,4 @@ md5sum = e8033d4fd7b6348b525a6148762ccdb4
 
 [template-apache-backend-conf]
 filename = apache-backend.conf.in
-md5sum = aff99c44ccf16eaa2ca25430d76d3bd6
+md5sum = 48f086ce1acffca7bab942b43d856fb7

stack/erp5/buildout.cfg

@@ -820,12 +820,6 @@ unidiff = 0.5.5
 jsonpickle = 0.9.6
 decorator = 4.3.0
 
-mock = 3.0.4
-
-# Required by:
-# mock = 3.0.4
-funcsigs = 1.0.2
-
 responses = 0.10.6
 
 # Required by:

stack/erp5/buildout.hash.cfg

@@ -78,7 +78,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = ca5375204bacdc1df30285d3c5d179b1
+md5sum = af5d9aeac2bae695220465a4348ae592
 
 [template-zeo]
 filename = instance-zeo.cfg.in

stack/erp5/instance-erp5.cfg.in

@@ -212,7 +212,7 @@ config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
 {%   if server_type == 'neo' -%}
 config-neo-cluster = ${publish-early:neo-cluster}
 config-neo-name = {{ server_dict.keys()[0] }}
-config-neo-masters = ${neo-0-final:connection-masters}
+config-neo-masters = ${publish-early:neo-masters}
 {%   else -%}
 config-zodb-zeo = ${request-zodb:connection-storage-dict}
 config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
@@ -368,10 +368,6 @@ return = site_url
 <= monitor-publish
 recipe = slapos.cookbook:publish.serialised
 -extends = publish-early
-{% if 'neo' in storage_dict -%}
-neo-masters = ${neo-0-final:connection-masters}
-neo-admins = ${neo-0-final:connection-admins}
-{% endif  -%}
 {% if zope_address_list_id_dict -%}
 {#
 Pick any published hosts-dict, they are expected to be identical - and there is
@@ -388,7 +384,6 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-
 {%   endfor -%}
 {% endif -%}
 
-
 [publish-early]
 recipe = slapos.cookbook:publish-early
 -init =
@@ -404,6 +399,8 @@ recipe = slapos.cookbook:publish-early
 {%- endif %}
 {%- if neo %}
   neo-cluster gen-neo-cluster:name
+  neo-admins neo-cluster:admins
+  neo-masters neo-cluster:masters
 {%-  if neo[0] %}
 neo-cluster = {{ dumps(neo[0]) }}
 {%-  endif %}

stack/slapos.cfg

@@ -135,7 +135,7 @@ pyOpenSSL = 18.0.0
 pyparsing = 2.2.0
 pytz = 2016.10
 requests = 2.13.0
-six = 1.11.0
+six = 1.12.0
 slapos.cookbook = 1.0.119
 slapos.core = 1.4.26
 slapos.extension.strip = 0.4