Update Release Candidate

component/gnupg/buildout.cfg

+[buildout]
+extends =
+  ../bzip2/buildout.cfg
+  ../file/buildout.cfg
+  ../gnutls/buildout.cfg
+  ../sqlite3/buildout.cfg
+parts = gnupg
+
+[npth]
+<= gpg-common
+version = 1.5
+md5sum = 9ba2dc4302d2f32c66737c43ed191b1b
+configure-options-extra =
+  --disable-tests
+
+[libassuan]
+<= with-gpg-error
+version = 2.4.3
+md5sum = 8e01a7c72d3e5d154481230668e6eb5a
+
+[libksba]
+<= with-gpg-error
+version = 1.3.5
+md5sum = 8302a3e263a7c630aa7dea7d341f07a2
+
+[gnupg]
+<= libgcrypt
+version = 2.2.1
+md5sum = f781efae8756f6cf5d500aad8e4b33e2
+configure-options +=
+  --disable-ldap
+  --disable-card-support
+  --disable-photo-viewers
+  --with-bzip2=${bzip2:location}
+  --with-ksba-prefix=${libksba:location}
+  --with-libassuan-prefix=${libassuan:location}
+  --with-libgcrypt-prefix=${libgcrypt:location}
+  --with-npth-prefix=${npth:location}
+  --with-readline=${readline:location}
+environment =
+  PATH=${pkgconfig:location}/bin:${gettext:location}/bin:%(PATH)s
+  PKG_CONFIG_PATH=${gnutls:pkg-config-path}:${gnutls:location}/lib/pkgconfig:${sqlite3:location}/lib/pkgconfig
+  CPPFLAGS=-I${file:location}/include -I${libgcrypt:location}/include
+  LDFLAGS=-L${file:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${sqlite3:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib -Wl,-rpath=${libgpg-error:location}/lib -Wl,-rpath -Wl,${gnutls:location}/lib -Wl,-rpath=${libksba:location}/lib -Wl,-rpath=${libassuan:location}/lib -Wl,-rpath=${libgcrypt:location}/lib -Wl,-rpath=${npth:location}/lib

component/gnutls/buildout.cfg

@@ -14,7 +14,6 @@ url = http://www.gnupg.org/ftp/gcrypt/${:_buildout_section_name_}/${:_buildout_s
 configure-options = ${:configure-options-extra}
   --enable-option-checking=fatal
   --disable-dependency-tracking
-  --disable-doc
 environment = ${:environment-extra}
   PATH=${gettext:location}/bin:%(PATH)s
 environment-extra =
@@ -24,22 +23,30 @@ environment-extra =
 version = 1.27
 md5sum = 5217ef3e76a7275a2a3b569a12ddc989
 configure-options-extra =
+  --disable-doc
   --disable-tests
 
-[libgcrypt]
+[with-gpg-error]
 <= gpg-common
-version = 1.8.1
-md5sum = b21817f9d850064d2177285f1073ec55
-configure-options-extra =
+configure-options-extra = ${:configure-options-extra2}
   --with-gpg-error-prefix=${libgpg-error:location}
+configure-options-extra2 =
 environment-extra =
   LDFLAGS=-Wl,-rpath=${libgpg-error:location}/lib
 
+[libgcrypt]
+<= with-gpg-error
+version = 1.8.1
+md5sum = b21817f9d850064d2177285f1073ec55
+configure-options-extra2 =
+  --disable-doc
+
 [gnutls]
 <= gpg-common
 url = http://www.gnupg.org/ftp/gcrypt/gnutls/v3.5/gnutls-3.5.15.tar.xz
 md5sum = bcdcbc65c50a7499617ad9f4d0058de9
 configure-options-extra =
+  --disable-doc
   --disable-static
   --disable-libdane
   --disable-guile

component/golang/buildout.cfg

@@ -29,6 +29,7 @@ revision = go1.4.3-16-g4d5426a
 <= golang-common
 path = ${golang14-repository:location}
 environment-extra =
+make-targets= cd src && ./make.bash && cp -alf .. ${:location}
 
 
 [golang18]

component/openssl/buildout.cfg

@@ -16,8 +16,8 @@ parts =
 
 [openssl]
 recipe = slapos.recipe.cmmi
-url = https://www.openssl.org/source/openssl-1.0.2k.tar.gz
-md5sum = f965fc0bf01bf882b31314b61391ae65
+url = https://www.openssl.org/source/openssl-1.0.2m.tar.gz
+md5sum = 10e9e37f492094b9ef296f68f24a7666
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 # 'prefix' option to override --openssldir/--prefix (which is useful
 # when combined with INSTALL_PREFIX). Used by slapos.package.git/obs

setup.py

@@ -59,7 +59,7 @@ setup(name=name,
         'hexagonit.recipe.download',
         'netaddr', # to manipulate on IP addresses
         'setuptools', # namespaces
-        'inotifyx', # to watch filesystem changes (used in lockfile)
+        'inotifyx', # XXX use pyinotify instead
         'lock_file', #another lockfile implementation for multiprocess
         'slapos.core', # uses internally
         'zc.buildout', # plays with buildout
@@ -133,7 +133,6 @@ setup(name=name,
           'lamp.static = slapos.recipe.lamp:Static',
           'libcloud = slapos.recipe.libcloud:Recipe',
           'libcloudrequest = slapos.recipe.libcloudrequest:Recipe',
-          'lockfile = slapos.recipe.lockfile:Recipe',
           'logrotate = slapos.recipe.logrotate:Recipe',
           'logrotate.d = slapos.recipe.logrotate:Part',
           'memcached = slapos.recipe.memcached:Recipe',

slapos/recipe/generic_mysql/__init__.py

@@ -80,7 +80,7 @@ class Recipe(GenericBaseRecipe):
     mysql_script_list = []
 
     # user defined functions
-    udf_registration = ""
+    udf_registration = "DROP FUNCTION IF EXISTS last_insert_grn_id;\nDROP FUNCTION IF EXISTS mroonga_snippet;\nDROP FUNCTION IF EXISTS mroonga_command;\n"
     mroonga = self.options.get('mroonga', 'ha_mroonga.so')
     if mroonga:
       udf_registration += "CREATE FUNCTION last_insert_grn_id RETURNS " \

slapos/recipe/lockfile.py

-##############################################################################
-#
-# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-import os
-import sys
-import subprocess
-
-import inotifyx
-
-from slapos.recipe.librecipe import GenericBaseRecipe
-
-class LockFile(object):
-
-  class LockException(Exception):
-    pass
-
-  def __init__(self, filename, wait=True, exit=False):
-    self.filename = filename
-    if wait:
-      self.callback = lambda: self.waitDeletion()
-    elif not exit:
-      self.callback = lambda: self.raiseException()
-    else:
-      self.callback = lambda: sys.exit(1)
-
-  def raiseException(self):
-    raise LockFile.LockException("Not able to lock the file")
-
-  def waitDeletion(self):
-    inotify_fd = inotifyx.init()
-    try:
-      inotifyx.add_watch(inotify_fd, self.filename, inotifyx.IN_DELETE)
-      inotifyx.get_events(inotify_fd)
-    except IOError: # add_watch failed
-      pass
-    finally:
-      os.close(inotify_fd)
-
-    self.__enter__()
-
-  def __enter__(self):
-    try:
-      # Atomic file acquisition
-      self._fd = os.open(self.filename, os.O_CREAT | os.O_EXCL)
-    except OSError:
-      self.callback()
-
-  def __exit__(self, exc_type, exc_value, traceback):
-    os.close(self._fd)
-    os.unlink(self.filename)
-
-def locked_run(args):
-  with LockFile(args['filename'], wait=args['wait'], exit=True):
-    subprocess.check_call([args['binary']])
-
-class Recipe(GenericBaseRecipe):
-
-  def install(self):
-    wrapper = self.createPythonScript(self.options['wrapper'],
-                                      __name__ + '.locked_run',
-                                      dict(
-                                        filename=self.options['lock-file'],
-                                        wait=self.optionIsTrue('wait', False),
-                                        binary=self.options['binary'],
-                                      )
-                                     )
-    return [wrapper]

software/apache-frontend/buildout.hash.cfg

@@ -39,7 +39,7 @@ md5sum = 665e83d660c9b779249b2179d7ce4b4e
 
 [template-apache-frontend-configuration]
 filename = templates/apache.conf.in
-md5sum = 82cdb4ab02fec36285b9c1ce502f82f0
+md5sum = a56045e7b53ff00ab34d2a8f911fc1a1
 
 [template-custom-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in

software/apache-frontend/templates/apache.conf.in

@@ -166,8 +166,8 @@ SSLProxyCheckPeerCN off
 SSLProxyCheckPeerExpire off
 
 include {{frontend_configuration.get('log-access-configuration')}}
-include {{ slave_configuration_directory }}/*.conf
-include {{ slave_with_cache_configuration_directory }}/*.conf
+includeoptional {{ slave_configuration_directory }}/*.conf
+includeoptional {{ slave_with_cache_configuration_directory }}/*.conf
 
 ErrorDocument 404 /notfound.html
 RewriteRule (.*) /notfound.html [R=404,L]

software/buildout-testing/runTestSuite.in

+#!{{runTestSuite_py}}
+from __future__ import print_function
+import argparse, os, re, subprocess, sys
+from time import gmtime, strftime, time
+from erp5.util import taskdistribution
+from erp5.util.testsuite import SubprocessError, TestSuite
+from zc.buildout.buildout import Buildout
+
+slapos_buildout = {{repr(slapos_buildout)}}
+test_dict = {
+  'zc.buildout': slapos_buildout,
+  'zc.recipe.egg': os.path.join(slapos_buildout, 'zc.recipe.egg_'),
+}
+
+class DummyTestResult:
+
+  class DummyTestResultLine:
+    def stop(self, **kw):
+      pass
+
+  done = 0
+
+  def __init__(self, test_name_list):
+    self.test_name_list = test_name_list
+
+  def start(self):
+    test_result_line = self.DummyTestResultLine()
+    try:
+      test_result_line.name = self.test_name_list[self.done]
+    except IndexError:
+      return
+    self.done += 1
+    return test_result_line
+
+class BuildoutTestSuite(TestSuite):
+
+  RUN_RE = re.compile(
+    r'Ran (?P<all_tests>\d+) tests with'
+    ' (?P<failures>\d+) failures,'
+    ' (?P<errors>\d+) errors and'
+    ' (?P<skips>\d+) skipped in')
+
+  def run(self, test):
+    start = time()
+    try:
+      status_dict = self.spawn(os.path.join('bin', 'zope-testrunner'),
+        '--test-path', os.path.join(test_dict[test], 'src'))
+    except SubprocessError, e:
+      status_dict = e.status_dict
+    end = time()
+    status_dict.update(
+      date = strftime("%Y/%m/%d %H:%M:%S", gmtime(end)),
+      duration = end - start)
+    search = self.RUN_RE.search(status_dict['stdout'])
+    if search:
+      groupdict = search.groupdict()
+      status_dict.update(
+        test_count = int(groupdict['all_tests']),
+        error_count = int(groupdict['errors']),
+        failure_count = int(groupdict['failures']),
+        skip_count = int(groupdict['skips']))
+    return status_dict
+
+def main():
+  os.environ['TEMP'] = {{repr(temp_directory)}}
+  parser = argparse.ArgumentParser(description='Run a test suite.')
+  parser.add_argument('--test_suite', help='The test suite name')
+  parser.add_argument('--test_suite_title', help='The test suite title')
+  parser.add_argument('--test_node_title', help='The test node title')
+  parser.add_argument('--project_title', help='The project title')
+  parser.add_argument('--revision', help='The revision to test',
+                      default='dummy_revision')
+  parser.add_argument('--master_url',
+                      help='The Url of Master controling many suites')
+
+  args = parser.parse_args()
+
+  test_title = args.test_suite_title or args.test_suite
+  if args.master_url:
+    tool = taskdistribution.TaskDistributionTool(args.master_url)
+    test_result = tool.createTestResult(args.revision,
+                                        list(test_dict),
+                                        args.test_node_title,
+                                        test_title=test_title,
+                                        project_title=args.project_title)
+    if test_result is None:
+      return
+  else:
+    test_result = DummyTestResult(list(test_dict))
+
+  fd = os.open('buildout.cfg', os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0666)
+  try:
+    os.write(fd, """\
+[buildout]
+extends = %s
+develop =%s
+parts = test
+
+[versions]
+%s
+
+[bootstrap]
+recipe = zc.recipe.egg
+eggs = zc.buildout
+
+[test]
+recipe = zc.recipe.egg
+eggs +=
+  zope.testrunner
+scripts =
+  zope-testrunner
+""" % (os.path.join(slapos_buildout, 'buildout.cfg'),
+       ''.join('\n  ' + x for x in test_dict.itervalues()),
+       '\n'.join(x + ' =' for x in test_dict)))
+  finally:
+    os.close(fd)
+  Buildout('buildout.cfg', {}).install(['bootstrap'])
+  subprocess.check_call((os.path.join('bin', 'buildout'),))
+
+  test_suite = BuildoutTestSuite(1)
+  while 1:
+    test_result_line = test_result.start()
+    if not test_result_line:
+      break
+    test_result_line.stop(**test_suite.run(test_result_line.name))
+
+if __name__ == "__main__":
+    main()

software/buildout-testing/software-next.cfg

+[buildout]
+extends = software.cfg
+
+[versions]
+setuptools = 36.6.0

software/buildout-testing/software.cfg

+[buildout]
+extends =
+  ../../stack/slapos.cfg
+parts =
+  slapos-cookbook
+  template
+
+[slapos.buildout-repository]
+recipe = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/slapos.buildout.git
+git-executable = ${git:location}/bin/git
+
+[runTestSuite_py]
+recipe = zc.recipe.egg
+eggs = erp5.util
+  zc.buildout
+scripts = ${:interpreter}
+interpreter = ${:_buildout_section_name_}
+
+[template]
+recipe = slapos.recipe.template:jinja2
+# XXX: "template.cfg" is hardcoded in instanciation recipe
+rendered = ${buildout:directory}/template.cfg
+template =
+  inline:
+  [buildout]
+  eggs-directory = ${buildout:eggs-directory}
+  develop-eggs-directory = ${buildout:develop-eggs-directory}
+  parts = runTestSuite
+
+  [slap-configuration]
+  recipe = slapos.cookbook:slapconfiguration.serialised
+  computer = $${slap-connection:computer-id}
+  partition = $${slap-connection:partition-id}
+  url = $${slap-connection:server-url}
+  key = $${slap-connection:key-file}
+  cert = $${slap-connection:cert-file}
+
+  [directory]
+  recipe = slapos.cookbook:mkdirectory
+  bin = $${buildout:directory}/bin
+  tmp = $${buildout:directory}/tmp
+
+  [slapos.buildout-repository]
+  recipe = slapos.recipe.build:gitclone
+  repository = ${slapos.buildout-repository:location}
+  git-executable = ${git:location}/bin/git
+  shared = true
+
+  [runTestSuite]
+  recipe = slapos.recipe.template:jinja2
+  rendered = $${directory:bin}/$${:_buildout_section_name_}
+  template = ${:_profile_base_location_}/$${:_buildout_section_name_}.in
+  mode = 0755
+  context =
+    key slapparameter_dict slap-configuration:configuration
+    key slapos_buildout slapos.buildout-repository:location
+    key temp_directory directory:tmp
+    raw runTestSuite_py ${buildout:bin-directory}/${runTestSuite_py:interpreter}

software/erp5/instance-erp5-input-schema.json

@@ -21,6 +21,10 @@
       "description": "Business Template to install at automatic site creation. By default, all configurators are installed.",
       "type": "string"
     },
+    "id-store-interval": {
+      "description": "Set Store Interval of default SQL Non Continuous Increasing Id Generator at automatic site creation. If unset, the value from the erp5_core Business Template is not touched.",
+      "type": "integer"
+    },
     "timezone": {
       "description": "Zope's timezone. Possible values are determined by host's libc, and typically come from a separate package (tzdata, ...)",
       "default": "UTC",

software/erp5testnode/software.cfg

@@ -27,10 +27,9 @@ eggs =
   zc.buildout
   slapos.libnetworkcache
   slapos.core
+  supervisor
   jsonschema
   hexagonit.recipe.download
-  netaddr
-  inotifyx
   lock_file
   pytz
   erp5.util

software/slapos-in-partition/common.cfg

@@ -71,21 +71,3 @@ mode = 0644
 recipe = zc.recipe.egg
 eggs =
   collective.recipe.template
-
-# Add slapos.libnetworkcache to path of slapos.core.
-[slapos-cookbook]
-eggs =
-  ${lxml-python:egg}
-  slapos.cookbook
-  cliff
-  hexagonit.recipe.download
-  inotifyx
-  netaddr
-  netifaces
-  requests
-  slapos.core
-  supervisor
-  xml_marshaller
-  pytz
-  slapos.libnetworkcache
-

software/slapos-master/buildout.hash.cfg

@@ -15,7 +15,7 @@
 # not need these here).
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 95f7341ff490cb70df61fad35d08bc30
+md5sum = 251c1fcec1817fe70ec0cf7da69ccf3a
 
 [template-balancer]
 filename = instance-balancer.cfg.in

software/slapos-master/instance-erp5.cfg.in

@@ -155,6 +155,7 @@ config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
 config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
 config-hosts-dict = {{ dumps(slapparameter_dict.get('hosts-dict', {})) }}
 config-hostalias-dict = {{ dumps(slapparameter_dict.get('hostalias-dict', {})) }}
+config-id-store-interval = {{ dumps(slapparameter_dict.get('id-store-interval')) }}
 config-inituser-login = {{ dumps(inituser_login) }}
 config-inituser-password = ${publish-early:inituser-password}
 config-kumofs-url = ${request-memcached-persistent:connection-url}

software/slaprunner/buildout.hash.cfg

@@ -19,11 +19,11 @@ md5sum = 713db528880282d568278f09458d2aab
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = baf70a9f2f433ea0f2f5139700e53b1f
+md5sum = 7df86928aeef0fbada832b0283b9a454
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2
-md5sum = 130193114cbbcd014af9704851410605
+md5sum = 1edd9c7a20e208b6cb647886bfb6d1bb
 
 [instance-runner-import]
 filename = instance-runner-import.cfg.in
@@ -55,7 +55,7 @@ md5sum = 525e37ea8b2acf6209869999b15071a6
 
 [template-slapos-cfg]
 filename = template/slapos.cfg.in
-md5sum = ef16446d432e1397182b1654fe920ffb
+md5sum = 035e027e9cb9bbdca0509ac895fc4696
 
 [template-parameters]
 filename = parameters.xml.in

software/slaprunner/instance-runner.cfg

@@ -137,6 +137,8 @@ working-directory = $${runnerdirectory:home}
 project-directory = $${runnerdirectory:project}
 instance_root = $${runnerdirectory:instance-root}
 software_root = $${runnerdirectory:software-root}
+pidfile-software = $${directory:run}/slapgrid-cp.pid
+pidfile-instance = $${directory:run}/slapgrid-sr.pid
 ssh_client = ${openssh:location}/bin/ssh
 public_key = $${runner-sshd-raw-server:rsa-keyfile}.pub
 private_key = $${runner-sshd-raw-server:rsa-keyfile}
@@ -728,14 +730,12 @@ ip = $${slaprunner:ipv4}
 server = $${:ip}:$${:port}
 port = 39986
 slapgrid-cp = slapgrid-cp
-slapgrid-cp-command = $${slaprunner:slapos} node instance --all --cfg $${:slapos-cfg} --pidfile $${:slapgrid-cp-pid} --verbose --logfile $${:slapgrid-cp-log}
+slapgrid-cp-command = $${slaprunner:slapos} node instance --all --cfg $${:slapos-cfg} --verbose --logfile $${:slapgrid-cp-log}
 slapgrid-cp-log = $${runnerdirectory:home}/instance.log
-slapgrid-cp-pid = $${directory:run}/slapgrid-cp.pid
 slapgrid-cp-startretries = 0
 slapgrid-sr = slapgrid-sr
-slapgrid-sr-command = $${slaprunner:slapos} node software --all --cfg $${:slapos-cfg} --pidfile $${:slapgrid-sr-pid} --verbose --logfile $${:slapgrid-sr-log}
+slapgrid-sr-command = $${slaprunner:slapos} node software --all --cfg $${:slapos-cfg} --verbose --logfile $${:slapgrid-sr-log}
 slapgrid-sr-log = $${runnerdirectory:home}/software.log
-slapgrid-sr-pid = $${directory:run}/slapgrid-sr.pid
 slapgrid-sr-startretries = 0
 slapproxy = slapproxy
 slapproxy-autorestart = true

software/slaprunner/template/runner-import.sh.jinja2

@@ -155,22 +155,20 @@ rm '{{ instance_folder }}'/etc/supervisord.conf.d/* || true
 
 SLAPOSCFG='{{ supervisord["slapos-cfg"] }}'
 SLAPGRIDSRLOG='{{ supervisord["slapgrid-sr-log"] }}'
-SLAPGRIDSRPID='{{ supervisord["slapgrid-sr-pid"] }}'
 SLAPGRIDCPLOG='{{ supervisord["slapgrid-cp-log"] }}'
-SLAPGRIDCPPID='{{ supervisord["slapgrid-cp-pid"] }}'
 
 log_message "Building newest Software Release..."
-"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" --pidfile "$SLAPGRIDSRPID" >/dev/null 2>&1 ||
-"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" --pidfile "$SLAPGRIDSRPID" >/dev/null 2>&1 ||
-"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" --pidfile "$SLAPGRIDSRPID" >/dev/null 2>&1 ||
+"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 ||
+"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 ||
+"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 ||
 (tail -n 200 "$SLAPGRIDSRLOG" && false)
 # Remove defined scripts to force buildout to recreate them to have updated paths
 rm "$srv_directory"/runner/instance/slappart*/srv/runner-import-restore || true
 log_message "Fixing Instances as needed after import..."
 # XXX hardcoded
-"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" --pidfile "$SLAPGRIDCPPID" >/dev/null 2>&1 ||
-"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" --pidfile "$SLAPGRIDCPPID" >/dev/null 2>&1 ||
-"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" --pidfile "$SLAPGRIDCPPID" >/dev/null 2>&1 ||
+"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" >/dev/null 2>&1 ||
+"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" >/dev/null 2>&1 ||
+"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" >/dev/null 2>&1 ||
 (tail -n 200 "$SLAPGRIDCPLOG" && false)
 
 # Invoke defined scripts for each partition inside of slaprunner

software/slaprunner/template/slapos.cfg.in

@@ -6,6 +6,8 @@ computer_id = slaprunner
 maximal_delay = 0
 root_check = {{ slaprunner['root_check'] }}
 forbid_supervisord_automatic_launch = true
+pidfile_software =  {{slaprunner['pidfile-software']}}
+pidfile_instance =  {{slaprunner['pidfile-instance']}}
 
 [slapformat]
 partition_amount = {{ slaprunner['partition-amount'] }}

software/test-slave-instance-deployment/software.cfg

@@ -106,23 +106,11 @@ versions = versions
 networkcache-section = networkcache
 
 # Install slapos.cookbook containing all officials recipes
-# Explicitely define dependencies as well, because of buildout limitation
-# if using "develop"
 [slapos-cookbook]
 recipe = zc.recipe.egg
 eggs =
   ${lxml-python:egg}
   slapos.cookbook
-  cliff
-  hexagonit.recipe.download
-  inotifyx
-  netaddr
-  netifaces
-  requests
-  slapos.core
-  supervisor
-  xml_marshaller
-  pytz
 
 [versions]
 # Use SlapOS patched zc.buildout

software/wendelin-scalability/test-fluentd-1pb.cfg

+[buildout]
+extends = test-fluentd-common.cfg
+
+[template-erp5-patched]
+extra =
+  {%- for family_name, zope_section_id_list in zope_family_dict.items() %}
+  {%-   for zope_section_id in zope_section_id_list %}
+  [{{zope_section_id}}]
+  config-family = {{family_name}}
+  {%-   endfor %}
+  {%- endfor %}
+
+[template-zope]
+recipe =
+target = ${template-zope-patched:location}
+
+[template-zope-base]
+<= template-zope
+recipe = slapos.recipe.build:download
+
+[template-zope-patched]
+<= template-fluentd
+base = ${template-zope-base:target}
+method = unpack
+tags =
+  {%- if slapparameter_dict['family'] == 'fluentd' %}
+  {%- for i, zope in enumerate(publish_list) %}
+  {%-   do tags.__setitem__(
+      node_id_base ~ (node_id_index_format % i) ~ '-stream', zope[0]) %}
+  {%- endfor %}
+  {%- endif %}
+
+[feeder]
+feeder =
+  #
+    import os, struct
+    from random import lognormvariate
+    pack = struct.Struct('!d').pack
+    size = 65536
+    data = os.urandom(size - 8)
+    while True:
+      emit('',
+        ''.join((pack(time.time()) + data[:int(lognormvariate(10, 1))]
+                ).ljust(size, '\0') for i in xrange(16)))
+      time.sleep(interval)

software/wendelin-scalability/test-fluentd-common.cfg

+[buildout]
+extends =
+  ../../component/gnupg/buildout.cfg
+  ../fluentd/software.cfg
+  ../wendelin/software.cfg
+  ../../component/mariadb/mariarocks.cfg
+
+[local-bt5-repository]
+list += ${slapos.cookbook-repository:location}/software/wendelin-scalability
+
+[patch-template]
+recipe = slapos.recipe.build
+location = ${buildout:directory}/${:_buildout_section_name_}.cfg.in
+script =
+  with open(self.options['location'], 'w') as dst, \
+       open(self.options['base']) as src:
+    src = src.read()
+    i = src.index('[buildout]')
+    dst.write(src[:i] + self.options['extra'] + '\n' + src[i:])
+
+[template-erp5]
+recipe =
+target = ${template-erp5-patched:location}
+
+[template-erp5-base]
+<= template-erp5
+recipe = slapos.recipe.build:download
+
+[template-erp5-patched]
+<= patch-template
+base = ${template-erp5-base:target}
+
+[template-fluentd]
+<= patch-template
+extra =
+  {%- set tags = {} %}
+  ${:tags}
+  {%- if tags %}
+  [feeder]
+  recipe = slapos.cookbook:wrapper
+  wrapper-path = $${directory:services}/$${:_buildout_section_name_}
+  command-line = ${feeder:bin-directory}/feeder $${fluentd-agent-conf:rendered}
+    $${:_buildout_section_name_} .4
+  {%- for tag in tags %}
+  [{{ section(tag) }}]
+  <= feeder
+  {%- endfor %}
+
+  [{{ section('fluentd') }}]
+  recipe = slapos.cookbook:wrapper
+  wrapper-path = $${directory:services}/$${:_buildout_section_name_}
+  command-line = ${fluentd:location}/bin/fluentd
+    -v -c $${fluentd-agent-conf:rendered}
+  environment =
+    GEM_PATH=${fluentd:location}/lib/ruby/gems/1.8
+
+  [fluentd-agent-conf]
+  recipe = slapos.recipe.template:jinja2
+  rendered = $${directory:etc}/fluentd-agent.conf
+  template =
+    inline:{% raw -%}
+    <match debug.**>
+      @type stdout
+      @id stdout_output
+    </match>
+    <source>
+# in_exec is unusable because it logs dropped events in case of buffer overflow
+      @type unix
+      path $${directory:run}/fluentd.sock
+    </source>
+    {%- for tag, netloc in tags.iteritems() %}
+    <match {{ tag }}>
+      @type          wendelin
+      @id            {{ tag }}
+      streamtool_uri http://{{ netloc }}/erp5/portal_ingestion_policies/scalability_test_${:method}
+      user           {{ user }}
+      password       {{ password }}
+      buffer_type    memory
+      buffer_queue_limit 4
+    </match>
+    {%- endfor %}
+    {%- endraw %}
+  context =
+    key tags :tags
+    raw user {{ slapparameter_dict['inituser-login'] }}
+    raw password {{ slapparameter_dict['inituser-password'] }}
+  tags = {{ dumps(tags) }}
+  {%- endif %}
+
+[feeder]
+recipe = zc.recipe.egg
+eggs = fluent-logger
+entry-points = ${:_buildout_section_name_}=__main__:main
+initialization =
+  import re, time, urlparse
+  from urllib import FancyURLopener
+  from fluent.sender import FluentSender
+  def main():
+    conf, tag, interval = sys.argv[1:]
+    with open(conf) as f:
+      for k, v in re.compile(r"<(.+?)>\s*(.*?)\s*</.+?>",
+                             re.S).findall(f.read()):
+        v = dict(v.split() for v in v.splitlines())
+        if k == 'source':
+          sock = v['path']
+        elif v['@type'] == 'wendelin':
+          if k.split(None, 1)[1] == tag:
+            conf = v
+    emit = FluentSender(tag, host='unix://'+sock, timeout=None,
+      msgpack_kwargs={'use_bin_type': True}).emit
+    url = urlparse.urlsplit(conf['streamtool_uri'])
+    url = urlparse.urlunsplit(url._replace(
+      netloc='%s:%s@%s' % (conf['user'], conf['password'], url.netloc),
+      path='/erp5/isPortalBeingCreated'))
+    urlopen = FancyURLopener().open
+    while True:
+      try:
+        f = urlopen(url)
+      except IOError:
+        pass
+      else:
+        try:
+          if not f.read():
+            break
+        finally:
+          f.close()
+      time.sleep(5)
+    interval = float(interval)
+    ${:feeder}
+
+[versions]
+fluent-logger = 0.5.3

software/wendelin-scalability/test-much-deduplication-high-compressibility.cfg

+[buildout]
+extends = test-fluentd-common.cfg
+
+[template-erp5-patched]
+extra =
+  [request-balancer]
+  config-inituser-login = {{ dumps(inituser_login) }}
+  config-inituser-password = $${publish-early:inituser-password}
+
+[template-balancer]
+recipe =
+target = ${template-balancer-patched:location}
+
+[template-balancer-base]
+<= template-balancer
+recipe = slapos.recipe.build:download
+
+[template-balancer-patched]
+<= template-fluentd
+base = ${template-balancer-base:target}
+method = raw_lf
+tags =
+  {%- set port, backend_list = haproxy_dict['fluentd'] %}
+  {%- for i in range(len(backend_list)) %}
+  {%-   do tags.__setitem__('wendelin_tag_' ~ i, ipv4 ~ ':' ~ port) %}
+  {%- endfor %}
+
+[data.gpg]
+recipe = slapos.recipe.build:download
+url = http://download.shacache.org/0f56a59ab3bc3193e70c1996fe69260e1b225bc53bbb7509cb3ae036006653dec69b3384cd624eefed81c85e692e4b4b7cb3d25c6244c298070569dddf7216b1
+md5sum = 374e00dfbd28f27819ccfff31d169c9c
+
+[gpg-decrypt]
+recipe = slapos.recipe.build
+gpg = ${gnupg:location}/bin/gpg
+script =
+  import subprocess, zc.buildout
+  options = self.options
+  args = (options['gpg'], '-d', '--batch', '--passphrase-fd', '0',
+          '-o', options['location'], options['input'])
+  p = subprocess.Popen(args, stdin=subprocess.PIPE)
+  p.communicate(options['password'])
+  retcode = p.poll()
+  if retcode:
+    raise subprocess.CalledProcessError(retcode, ' '.join(args))
+
+[data]
+<= gpg-decrypt
+location = ${data.gpg:location}/data.log
+input = ${data.gpg:target}
+
+[feeder]
+feeder =
+  #
+    import mmap, os, random
+    data = mmap.mmap(os.open('${data:location}', os.O_RDONLY),
+                     0, prot=mmap.PROT_READ)
+    size = 1024 * 1024
+    max_seek = len(data) - size
+    while True:
+      a = random.randint(0, max_seek)
+      a = {'data': data[a:a+size]}
+      for i in xrange(20):
+        emit('', a)
+        time.sleep(interval)

software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_ingestion_policies/scalability_test_raw_lf.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Ingestion Policy" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_Access_contents_information_Permission</string> </key>
+            <value>
+              <tuple>
+                <string>Assignee</string>
+                <string>Assignor</string>
+                <string>Associate</string>
+                <string>Auditor</string>
+                <string>Manager</string>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>_Add_portal_content_Permission</string> </key>
+            <value>
+              <tuple>
+                <string>Assignee</string>
+                <string>Assignor</string>
+                <string>Associate</string>
+                <string>Manager</string>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>_Modify_portal_content_Permission</string> </key>
+            <value>
+              <tuple>
+                <string>Assignee</string>
+                <string>Assignor</string>
+                <string>Associate</string>
+                <string>Manager</string>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>_View_Permission</string> </key>
+            <value>
+              <tuple>
+                <string>Assignee</string>
+                <string>Assignor</string>
+                <string>Associate</string>
+                <string>Auditor</string>
+                <string>Manager</string>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>scalability_test_raw_lf</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Ingestion Policy</string> </value>
+        </item>
+        <item>
+            <key> <string>script_id</string> </key>
+            <value> <string>ERP5Site_handleRawDataFluentdIngestion</string> </value>
+        </item>
+        <item>
+            <key> <string>workflow_history</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>validation_workflow</string> </key>
+                    <value>
+                      <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+                    </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
+    <pickle>
+      <global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
+    </pickle>
+    <pickle>
+      <tuple>
+        <none/>
+        <list>
+          <dictionary>
+            <item>
+                <key> <string>action</string> </key>
+                <value> <string>validate</string> </value>
+            </item>
+            <item>
+                <key> <string>actor</string> </key>
+                <value> <string>zope</string> </value>
+            </item>
+            <item>
+                <key> <string>comment</string> </key>
+                <value> <string></string> </value>
+            </item>
+            <item>
+                <key> <string>error_message</string> </key>
+                <value> <string></string> </value>
+            </item>
+            <item>
+                <key> <string>time</string> </key>
+                <value>
+                  <object>
+                    <klass>
+                      <global name="DateTime" module="DateTime.DateTime"/>
+                    </klass>
+                    <tuple>
+                      <none/>
+                    </tuple>
+                    <state>
+                      <tuple>
+                        <float>1507057895.87</float>
+                        <string>UTC</string>
+                      </tuple>
+                    </state>
+                  </object>
+                </value>
+            </item>
+            <item>
+                <key> <string>validation_state</string> </key>
+                <value> <string>validated</string> </value>
+            </item>
+          </dictionary>
+        </list>
+      </tuple>
+    </pickle>
+  </record>
+</ZopeData>

software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_ingestion_policies/scalability_test_unpack.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Ingestion Policy" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_Access_contents_information_Permission</string> </key>
+            <value>
+              <tuple>
+                <string>Assignee</string>
+                <string>Assignor</string>
+                <string>Associate</string>
+                <string>Auditor</string>
+                <string>Manager</string>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>_Add_portal_content_Permission</string> </key>
+            <value>
+              <tuple>
+                <string>Assignee</string>
+                <string>Assignor</string>
+                <string>Associate</string>
+                <string>Manager</string>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>_Modify_portal_content_Permission</string> </key>
+            <value>
+              <tuple>
+                <string>Assignee</string>
+                <string>Assignor</string>
+                <string>Associate</string>
+                <string>Manager</string>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>_View_Permission</string> </key>
+            <value>
+              <tuple>
+                <string>Assignee</string>
+                <string>Assignor</string>
+                <string>Associate</string>
+                <string>Auditor</string>
+                <string>Manager</string>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>scalability_test_unpack</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Ingestion Policy</string> </value>
+        </item>
+        <item>
+            <key> <string>script_id</string> </key>
+            <value> <string>ERP5Site_handleRawDataFluentdIngestion</string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>workflow_history</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>validation_workflow</string> </key>
+                    <value>
+                      <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+                    </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
+    <pickle>
+      <global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
+    </pickle>
+    <pickle>
+      <tuple>
+        <none/>
+        <list>
+          <dictionary>
+            <item>
+                <key> <string>action</string> </key>
+                <value> <string>validate</string> </value>
+            </item>
+            <item>
+                <key> <string>actor</string> </key>
+                <value> <string>zope</string> </value>
+            </item>
+            <item>
+                <key> <string>comment</string> </key>
+                <value> <string></string> </value>
+            </item>
+            <item>
+                <key> <string>error_message</string> </key>
+                <value> <string></string> </value>
+            </item>
+            <item>
+                <key> <string>time</string> </key>
+                <value>
+                  <object>
+                    <klass>
+                      <global name="DateTime" module="DateTime.DateTime"/>
+                    </klass>
+                    <tuple>
+                      <none/>
+                    </tuple>
+                    <state>
+                      <tuple>
+                        <float>1507057913.93</float>
+                        <string>UTC</string>
+                      </tuple>
+                    </state>
+                  </object>
+                </value>
+            </item>
+            <item>
+                <key> <string>validation_state</string> </key>
+                <value> <string>validated</string> </value>
+            </item>
+          </dictionary>
+        </list>
+      </tuple>
+    </pickle>
+  </record>
+</ZopeData>

software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/DataStreamModule_getTotalSize.py

+size = 0
+for ds in context.objectValues():
+  size += ds.getSize()
+return size

software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/DataStreamModule_getTotalSize.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>Script_magic</string> </key>
+            <value> <int>3</int> </value>
+        </item>
+        <item>
+            <key> <string>_bind_names</string> </key>
+            <value>
+              <object>
+                <klass>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                </klass>
+                <tuple/>
+                <state>
+                  <dictionary>
+                    <item>
+                        <key> <string>_asgns</string> </key>
+                        <value>
+                          <dictionary>
+                            <item>
+                                <key> <string>name_container</string> </key>
+                                <value> <string>container</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_context</string> </key>
+                                <value> <string>context</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_m_self</string> </key>
+                                <value> <string>script</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_subpath</string> </key>
+                                <value> <string>traverse_subpath</string> </value>
+                            </item>
+                          </dictionary>
+                        </value>
+                    </item>
+                  </dictionary>
+                </state>
+              </object>
+            </value>
+        </item>
+        <item>
+            <key> <string>_params</string> </key>
+            <value> <string></string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>DataStreamModule_getTotalSize</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion.py

+portal = context.getPortalObject()
+request = portal.REQUEST
+reference = request['reference']
+data_chunk = request['data_chunk']
+
+module = portal.data_stream_module
+try:
+  data_stream = module[reference]
+except KeyError:
+  data_stream = module.newContent(reference, 'Data Stream')
+
+append_method = context.getId()
+if append_method == 'scalability_test_raw_lf':
+  data_stream.appendData(data_chunk + '\n')
+else:
+  assert append_method == 'scalability_test_unpack', append_method
+  for time, data_chunk in context.unpack(data_chunk):
+    data_stream.appendData(data_chunk)

software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>Script_magic</string> </key>
+            <value> <int>3</int> </value>
+        </item>
+        <item>
+            <key> <string>_bind_names</string> </key>
+            <value>
+              <object>
+                <klass>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                </klass>
+                <tuple/>
+                <state>
+                  <dictionary>
+                    <item>
+                        <key> <string>_asgns</string> </key>
+                        <value>
+                          <dictionary>
+                            <item>
+                                <key> <string>name_container</string> </key>
+                                <value> <string>container</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_context</string> </key>
+                                <value> <string>context</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_m_self</string> </key>
+                                <value> <string>script</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_subpath</string> </key>
+                                <value> <string>traverse_subpath</string> </value>
+                            </item>
+                          </dictionary>
+                        </value>
+                    </item>
+                  </dictionary>
+                </state>
+              </object>
+            </value>
+        </item>
+        <item>
+            <key> <string>_params</string> </key>
+            <value> <string></string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>ERP5Site_handleRawDataFluentdIngestion</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

software/wendelin-scalability/test_scalability_fluentd/bt/dependency_list

+erp5_wendelin
\ No newline at end of file

software/wendelin-scalability/test_scalability_fluentd/bt/template_format_version

+1
\ No newline at end of file

software/wendelin-scalability/test_scalability_fluentd/bt/template_keep_last_workflow_history_only_path_list

+portal_ingestion_policies/scalability_test_*
\ No newline at end of file

software/wendelin-scalability/test_scalability_fluentd/bt/template_path_list

+portal_ingestion_policies/scalability_test_*
+portal_skins/custom/DataStreamModule_getTotalSize
+portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion
\ No newline at end of file

software/wendelin-scalability/test_scalability_fluentd/bt/title

+test_scalability_fluentd
\ No newline at end of file

stack/caddy/Caddyfile.in

-[$${caddy-configuration:ipv6}]:$${caddy-configuration:port}
+{%- if parameter_dict['domain']  -%}
 
-log $${caddy-configuration:access_log}
+{{ parameter_dict['domain'] }} {
 
-root  $${directory:public_html}
+tls {{ custom_cert_dict['cert-file'] }} {{ custom_cert_dict['key-file'] }} {
\ No newline at end of file
+  
+  ca {{ca_custom_frontend_dict['rendered']}}
+
+}
+
+log {{caddy_configuration_dict['access_log']}}
+
+root {{ directory_dict['public_html'] }}
+
+{% if parameter_dict['enable-basic-auth'] == 'true'  -%}
+basicauth / {{ parameter_dict['username'] }} {{parameter_dict['password']}}
+{%- endif %}
+
+}
+
+{%- endif %}
+
+[{{caddy_configuration_dict['ipv6']}}]:{{parameter_dict['port-ipv6']}} {
+
+  tls {{ custom_cert_dict['cert-file'] }} {{ custom_cert_dict['key-file'] }} {
+  
+    ca {{ certificate_authority_dict['ca-cert-file'] }}
+  
+  }
+
+  log {{caddy_configuration_dict['access_log']}}
+
+  root {{ directory_dict['public_html'] }}
+
+  {% if parameter_dict['enable-basic-auth'] == 'true'  -%}
+  basicauth / {{ parameter_dict['username'] }} {{parameter_dict['password']}}
+  {%- endif %}
+
+}
+
+[{{caddy_configuration_dict['local_ip']}}]:{{parameter_dict['port-ipv4']}} {
+
+  log {{caddy_configuration_dict['access_log']}}
+
+  root {{ directory_dict['public_html'] }}
+
+  {% if parameter_dict['enable-basic-auth'] == 'true'  -%}
+  basicauth / {{ parameter_dict['username'] }} {{parameter_dict['password']}}
+  {%- endif %}
+
+}
\ No newline at end of file

stack/caddy/buildout.cfg

@@ -2,6 +2,7 @@
 extends =
   ../../component/golang/buildout.cfg
   ../../stack/slapos.cfg
+  ../../component/dash/buildout.cfg
   gowork.cfg
 
 parts =
@@ -14,36 +15,44 @@ parts =
 [gowork]
 install =
   github.com/mholt/caddy
-  
+
 [instance-profile]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = ae83c35995ce14432ded78ca30cab61b
+md5sum = 403f86b667f7a5d397993735bcd162ab
 output =${buildout:directory}/instance.cfg
 filename = instance.cfg
 mode = 0644
 
 [template-caddyfile]
-recipe = slapos.recipe.template
+recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/Caddyfile.in
-md5sum = 616f9c7cb788e1f1d7cee2093a7dc6ef
-output = ${buildout:directory}/Caddyfile.in
+#md5sum = bb3b314ebeb58e9875d547a053a9f268
+filename = Caddyfile.in
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
 mode = 0644
 
 [template-caddy]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-caddy.cfg.in
-md5sum = 045022e5698badca5dbdd6b95518f8e1
+md5sum = 3b796dee5c509393fd396fd019914a2e
 output = ${buildout:directory}/instance-caddy.cfg.in
 mode = 0644
 
 [template-public-html]
 recipe = slapos.recipe.template
-url =  ${:_profile_base_location_}/index.html
+url =  ${:_profile_base_location_}/templates/index.html
 md5sum = b5794ac8b10ed90173ad566e6e324b35
 output =  ${buildout:directory}/index.html
 mode = 0644
 
+[template-caddy-service]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/template-caddy-service.sh.in
+md5sum = 5467fac7b95acde681e613ae98ce990d
+output = ${buildout:directory}/template-caddy-service.sh.in
+mode = 0644
+
 [caddy]
 recipe  = slapos.recipe.cmmi
 path    = ${go_github.com_mholt_caddy:location}

stack/caddy/instance-caddy.cfg.in

@@ -2,10 +2,12 @@
 parts =
   caddy-service
   caddy-configuration
+  certificate-authority
+  custom-cert
+  htpasswd
   public-html
   publish-connection-information
 
-
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
@@ -20,29 +22,89 @@ service = $${:etc}/service
 public_html =  $${buildout:directory}/public_html
 run = $${:var}/run
 log = $${:var}/log
-
-#www = $${:srv}/www
-#ssl = $${:etc}/ssl
+ca-dir = $${:srv}/ssl
 
 #################################
 # caddy service
 #################################
 [caddy-service]
-recipe = slapos.cookbook:wrapper 
-command-line = ${caddy:output}  -conf $${caddy-configuration:output}
-wrapper-path = $${directory:service}/caddy
-output = $${:wrapper-path}
+recipe = slapos.recipe.template:jinja2
+template = ${template-caddy-service:output}
+rendered = $${directory:service}/caddy
+mode = 0700
+context =
+  key caddy_exec caddy-exec-dict:caddy-exec-file
+  section caddy_configuration_dict caddy-configuration
+  section parameter_dict slap-parameter
+
+[caddy-exec-dict]
+caddy-exec-file = ${caddy:output}
 
 [caddy-configuration]
-recipe = slapos.recipe.template
-url = ${template-caddyfile:output}
-output = $${directory:etc}/Caddyfile
+recipe = slapos.recipe.template:jinja2
+template = ${template-caddyfile:location}/${template-caddyfile:filename}
+rendered = $${directory:etc}/Caddyfile
 mode = 0600
 access_log = $${directory:log}/caddy-access.log
 error_log = $${directory:log}/caddy-error.log
 ipv6 = $${slap-network-information:global-ipv6}
 local_ip = $${slap-network-information:local-ipv4}
-port = 9443
+context =
+  section parameter_dict slap-parameter
+  section directory_dict directory
+  section caddy_configuration_dict caddy-configuration
+  section certificate_authority_dict certificate-authority
+  key htpasswd_dict htpasswd:passwd
+  section custom_cert_dict custom-cert
+  section ca_custom_frontend_dict ca-custom-frontend
+
+[ca-directory]
+recipe = slapos.cookbook:mkdirectory
+requests = $${directory:ca-dir}/requests/
+private = $${directory:ca-dir}/private/
+certs = $${directory:ca-dir}/certs
+newcerts = $${directory:ca-dir}/newcerts/
+crl = $${directory:ca-dir}/crl/
+
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = ${openssl:location}/bin/openssl
+ca-dir = $${directory:ca-dir}
+requests-directory = $${ca-directory:requests}
+wrapper = $${directory:service}/certificate_authority
+ca-private = $${ca-directory:private}
+ca-certs = $${ca-directory:certs}
+ca-newcerts = $${ca-directory:newcerts}
+ca-crl = $${ca-directory:crl}
+ca-cert-file = $${:ca-dir}/cacert.pem
+ca-key-file = $${:ca-private}/cakey.pem
+
+[custom-cert]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+executable = $${directory:service}/caddy
+wrapper = $${directory:service}/caddy
+key-file = $${ca-directory:private}/custom.key
+cert-file = $${ca-directory:certs}/custom.crt
+key-content = $${slap-parameter:key-content}
+cert-content = $${slap-parameter:cert-content}
+
+[ca-custom-frontend]
+recipe = slapos.recipe.template:jinja2
+template = $${template-empty:target}
+rendered = $${ca-directory:certs}/caddy_frontend.ca.crt
+context =
+    key content slap-parameter:caddy-ca-certificate
+
+[template-empty]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/templates/$${:filename}
+filename = empty.in
+
+[htpasswd]
+recipe = slapos.cookbook:generate.password
+storage-path = $${directory:etc}/.pwd
+bytes = 8
 
 [public-html]
 recipe = slapos.recipe.template
@@ -52,4 +114,18 @@ mode = 0600
 
 [publish-connection-information]
 recipe = slapos.cookbook:publish
-url = http://[$${caddy-configuration:ipv6}]:$${caddy-configuration:port}
+password = $${slap-parameter:password}
\ No newline at end of file
+user = $${slap-parameter:username}
+secure_access = https://[$${caddy-configuration:ipv6}]:$${slap-parameter:port-ipv6}
+
+[slap-parameter]
+domain =
+key-content =
+cert-content =
+caddy-ca-certificate =
+port-ipv6 = 9443
+port-ipv4 = 4443
+enable-quic = true
+enable-basic-auth =
+username = admin
+password = $${htpasswd:passwd}
\ No newline at end of file

stack/caddy/instance.cfg.in

@@ -6,7 +6,7 @@
 [buildout]
 parts =
   switch-softwaretype
-  
+
 #  publish-connection-parameter
 
 # Define egg directories to be the one from Software Release
@@ -28,7 +28,7 @@ rendered = $${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
 filename = instance-caddy.cfg
 
 [slap-connection]
-computer-id = {slap_connection:computer_id}
+computer-id = $${slap_connection:computer_id}
 partition-id = $${slap_connection:partition_id}
 server-url = $${slap_connection:server_url}
 software-release-url = $${slap_connection:software_release_url}
@@ -48,4 +48,4 @@ computer = $${slap_connection:computer_id}
 partition = $${slap_connection:partition_id}
 url = $${slap_connection:server_url}
 key = $${slap_connection:key_file}
-cert = $${slap_connection:cert_file}
+cert = $${slap_connection:cert_file}
\ No newline at end of file

stack/caddy/template-caddy-service.sh.in

+#!${dash-output:dash}
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+
+{{ caddy_exec }} -conf {{caddy_configuration_dict['rendered']}}
+ {%- if parameter_dict['enable-quic'] != 'false'  %}
+ -quic
+ {%- endif -%}
+ 
\ No newline at end of file

stack/caddy/templates/empty.in

+{{ content }}
\ No newline at end of file

stack/caucase/buildout.cfg

@@ -87,7 +87,7 @@ context =
 
 
 [versions]
-Flask-User = 0.6.11
+Flask-User = 0.6.19
 apache-libcloud = 2.1.0
 bcrypt = 3.1.3
 caucase = 0.1.4
@@ -103,24 +103,24 @@ smmap2 = 2.0.3
 Flask-AlchemyDumps = 0.0.10
 
 # Required by:
-# Flask-User==0.6.11
+# Flask-User==0.6.19
 Flask-Login = 0.4.0
 
 # Required by:
-# Flask-User==0.6.11
+# Flask-User==0.6.19
 Flask-Mail = 0.9.1
 
 # Required by:
 # Flask-AlchemyDumps==0.0.10
-# Flask-User==0.6.11
-Flask-SQLAlchemy = 2.2
+# Flask-User==0.6.19
+Flask-SQLAlchemy = 2.3.2
 
 # Required by:
 # Flask-AlchemyDumps==0.0.10
-Flask-Script = 2.0.5
+Flask-Script = 2.0.6
 
 # Required by:
-# Flask-User==0.6.11
+# Flask-User==0.6.19
 Flask-WTF = 0.14.2
 
 # Required by:
@@ -133,7 +133,7 @@ PyRSS2Gen = 1.1
 
 # Required by:
 # Flask-AlchemyDumps==0.0.10
-SQLAlchemy = 1.1.11
+SQLAlchemy = 1.1.15
 
 # Required by:
 # Flask-AlchemyDumps==0.0.10
@@ -173,8 +173,9 @@ passlib = 1.7.1
 
 # Required by:
 # caucase==0.1.4
-pem = 16.1.0
+pem = 17.1.0
 
+# Required by:
 # caucase==0.1.4
 pyasn1 = 0.2.3
 
@@ -183,5 +184,5 @@ pyasn1 = 0.2.3
 pyasn1-modules = 0.0.9
 
 # Required by:
-# Flask-User==0.6.11
-pycryptodome = 3.4.6
+# Flask-User==0.6.19
+pycryptodome = 3.4.7

stack/erp5/buildout.cfg

@@ -476,7 +476,6 @@ eggs = ${neoppod:eggs}
   collective.recipe.template
   coverage
   erp5diff
-  inotifyx
   interval
   ipdb
   Jinja2

stack/erp5/buildout.hash.cfg

@@ -39,7 +39,7 @@ md5sum = 76f9e8c8cdc352081e34539d8fc17026
 
 [template-zope-conf]
 filename = zope.conf.in
-md5sum = bbea91bc8f0b2d455d9824928abfad5b
+md5sum = 3524ef2e14cea4a5bd40fdc9e95cfc0c
 
 [site-zcml]
 filename = site.zcml
@@ -51,7 +51,7 @@ md5sum = d814b984abf2dc444af2a0bc6287e7f5
 
 [template-mariadb-initial-setup]
 filename = mariadb_initial_setup.sql.in
-md5sum = 6465212fdc7fe9076a0c929d9f14da14
+md5sum = dec33a617fa1b307c8ddb883efcfe3ce
 
 [template-postfix]
 filename = instance-postfix.cfg.in
@@ -79,7 +79,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 33c8f97afe58058d428108fb933523ac
+md5sum = 2465af81147af322056cee9f6c7de14f
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -87,7 +87,7 @@ md5sum = d1f33d406d528ae27d973e2dd0efb1ba
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = 27d26c6380883cf3bd7b2f003f7888d8
+md5sum = fd7e8c507cef1950e6c0347ce2a01021
 
 [template-balancer]
 filename = instance-balancer.cfg.in

stack/erp5/instance-erp5.cfg.in

@@ -155,6 +155,7 @@ config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
 config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
 config-hosts-dict = {{ dumps(slapparameter_dict.get('hosts-dict', {})) }}
 config-hostalias-dict = {{ dumps(slapparameter_dict.get('hostalias-dict', {})) }}
+config-id-store-interval = {{ dumps(slapparameter_dict.get('id-store-interval')) }}
 config-inituser-login = {{ dumps(inituser_login) }}
 config-inituser-password = ${publish-early:inituser-password}
 config-kumofs-url = ${request-memcached-persistent:connection-url}

stack/erp5/instance-zope.cfg.in

@@ -230,6 +230,7 @@ sql-connection-string = {{ '%s@erp5-catalog-0:%s %s %s' % (
     mysql.path.split('/')[1], mysql.port, mysql.username, mysql.password) }}
 bt5 = {{ slapparameter_dict['bt5'] }}
 bt5-repository-url = {{ slapparameter_dict['bt5-repository-url'] }}
+id-store-interval = {{ dumps(slapparameter_dict['id-store-interval']) }}
 home = ${buildout:directory}
 # We only want to change the hostname to 'erp5-cloudooo' if we use the internal
 # cloudooo. We plan to remove the ability to have an internal one, so this

stack/erp5/mariadb_initial_setup.sql.in

 USE mysql;
 DROP FUNCTION IF EXISTS last_insert_grn_id;
+DROP FUNCTION IF EXISTS mroonga_snippet;
+DROP FUNCTION IF EXISTS mroonga_command;
 {% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%}
-{% if mroonga %}CREATE FUNCTION last_insert_grn_id RETURNS INTEGER SONAME '{{ mroonga }}';{% endif %}
+{% if mroonga %}CREATE FUNCTION last_insert_grn_id RETURNS INTEGER SONAME '{{ mroonga }}';
+CREATE FUNCTION mroonga_snippet RETURNS STRING SONAME '{{ mroonga }}';
+CREATE FUNCTION mroonga_command RETURNS STRING SONAME '{{ mroonga }}';{% endif %}
 DROP FUNCTION IF EXISTS sphinx_snippets;
 #CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
 

stack/erp5/zope.conf.in

@@ -86,6 +86,9 @@ products {{ parameter_dict['instance-products'] }}
   cmf_activity_sql_connection_string {{ sql_connection_string }}
   bt5_repository_url {{ ' '.join(bt5_repository_url) }}
   bt5 {{ parameter_dict['bt5'] }}
+{%- if parameter_dict['id-store-interval'] != None %}
+  id_store_interval {{ parameter_dict['id-store-interval'] }}
+{%- endif %}
   cloudooo_url {{ parameter_dict['cloudooo-url'] }}
 </product-config>
 

stack/slapos.cfg

@@ -78,8 +78,6 @@ recipe = zc.recipe.egg:develop
 setup = ${slapos.cookbook-repository:location}
 
 # Install slapos.cookbook containing all officials recipes
-# Explicitely define dependencies as well, because of buildout limitation
-# if using "develop". XXX really ?
 [slapos-cookbook]
 recipe = zc.recipe.egg
 eggs =
@@ -91,15 +89,6 @@ eggs =
   pyOpenSSL
   slapos.cookbook
   slapos.libnetworkcache
-  hexagonit.recipe.download
-  inotifyx
-  netaddr
-  netifaces
-  requests
-  slapos.core
-  supervisor
-  xml_marshaller
-  pytz
 
 [versions]
 # Use SlapOS patched zc.buildout
@@ -139,7 +128,7 @@ requests = 2.13.0
 setuptools = 33.1.1
 six = 1.10.0
 slapos.cookbook = 1.0.53
-slapos.core = 1.4.2
+slapos.core = 1.4.3
 slapos.extension.strip = 0.4
 slapos.libnetworkcache = 0.15
 slapos.rebootstrap = 4.1
@@ -151,7 +140,7 @@ xml-marshaller = 0.9.7
 paramiko = 2.1.3
 
 # Required by:
-# slapos.core==1.4.2
+# slapos.core==1.4.3
 Flask = 0.12
 
 # Required by:
@@ -171,7 +160,7 @@ ipaddress = 1.0.18
 jsonschema = 2.6.0
 
 # Required by:
-# slapos.core==1.4.2
+# slapos.core==1.4.3
 # XXX 'slapos node format' raises an exception with netifaces 0.10.5.
 netifaces = 0.10.4
 
@@ -184,15 +173,15 @@ packaging = 16.8
 pycparser = 2.17
 
 # Required by:
-# slapos.core==1.4.2
+# slapos.core==1.4.3
 supervisor = 3.3.3
 
 # Required by:
-# slapos.core==1.4.2
+# slapos.core==1.4.3
 uritemplate = 3.0.0
 
 # Required by:
-# slapos.core==1.4.2
+# slapos.core==1.4.3
 zope.interface = 4.3.3
 
 [networkcache]
@@ -200,7 +189,6 @@ download-cache-url = http://download.shacache.org/
 download-dir-url = http://dir.shacache.org/
 
 # signature certificates of the following uploaders.
-#   Romain Courteaud
 #   Sebastien Robin
 #   Kazuhiko Shiozaki
 #   Gabriel Monnerat
@@ -210,19 +198,6 @@ download-dir-url = http://dir.shacache.org/
 #   Ivan Tyagov
 #   Julien Muchembled
 signature-certificate-list =
-  -----BEGIN CERTIFICATE-----
-  MIIB4DCCAUkCADANBgkqhkiG9w0BAQsFADA5MQswCQYDVQQGEwJGUjEZMBcGA1UE
-  CBMQRGVmYXVsdCBQcm92aW5jZTEPMA0GA1UEChMGTmV4ZWRpMB4XDTExMDkxNTA5
-  MDAwMloXDTEyMDkxNTA5MDAwMlowOTELMAkGA1UEBhMCRlIxGTAXBgNVBAgTEERl
-  ZmF1bHQgUHJvdmluY2UxDzANBgNVBAoTBk5leGVkaTCBnzANBgkqhkiG9w0BAQEF
-  AAOBjQAwgYkCgYEApYZv6OstoqNzxG1KI6iE5U4Ts2Xx9lgLeUGAMyfJLyMmRLhw
-  boKOyJ9Xke4dncoBAyNPokUR6iWOcnPHtMvNOsBFZ2f7VA28em3+E1JRYdeNUEtX
-  Z0s3HjcouaNAnPfjFTXHYj4um1wOw2cURSPuU5dpzKBbV+/QCb5DLheynisCAwEA
-  ATANBgkqhkiG9w0BAQsFAAOBgQBCZLbTVdrw3RZlVVMFezSHrhBYKAukTwZrNmJX
-  mHqi2tN8tNo6FX+wmxUUAf3e8R2Ymbdbn2bfbPpcKQ2fG7PuKGvhwMG3BlF9paEC
-  q7jdfWO18Zp/BG7tagz0jmmC4y/8akzHsVlruo2+2du2freE8dK746uoMlXlP93g
-  QUUGLQ==
-  -----END CERTIFICATE-----
   -----BEGIN CERTIFICATE-----
   MIIB8jCCAVugAwIBAgIJAPu2zchZ2BxoMA0GCSqGSIb3DQEBBQUAMBIxEDAOBgNV
   BAMMB3RzeGRldjMwHhcNMTExMDE0MTIxNjIzWhcNMTIxMDEzMTIxNjIzWjASMRAw