Update Release Candidate

component/fontconfig/buildout.cfg

@@ -7,14 +7,15 @@ extends =
   ../bzip2/buildout.cfg
   ../zlib/buildout.cfg
   ../bzip2/buildout.cfg
+  ../gperf/buildout.cfg
 
 parts =
   fontconfig
 
 [fontconfig]
 recipe = slapos.recipe.cmmi
-url = http://fontconfig.org/release/fontconfig-2.12.1.tar.bz2
-md5sum = b5af5a423ee3b5cfc34846838963c058
+url = http://fontconfig.org/release/fontconfig-2.12.6.tar.bz2
+md5sum = 733f5e2371ca77b69707bd7b30cc2163
 pkg_config_depends = ${freetype:pkg_config_depends}:${freetype:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig
 # XXX-Cedric : should we use --with-add-fonts={somefont:location}/share,{someotherfont:location}/share?
 configure-options =
@@ -23,7 +24,7 @@ configure-options =
   --enable-libxml2
   --with-default-fonts=${fonts:location}
 environment =
-  PATH=${pkgconfig:location}/bin:%(PATH)s
+  PATH=${pkgconfig:location}/bin:${gperf:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${:pkg_config_depends}
   CPPFLAGS=-I${zlib:location}/include -I${bzip2:location}/include
   LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib

component/haproxy/buildout.cfg

@@ -12,8 +12,8 @@ parts = haproxy
 
 [haproxy]
 recipe = slapos.recipe.cmmi
-url = http://www.haproxy.org/download/1.7/src/haproxy-1.7.9.tar.gz
-md5sum = a2bbbdd45ffe18d99cdcf26aa992f92d
+url = http://www.haproxy.org/download/1.8/src/haproxy-1.8.1.tar.gz
+md5sum = e42892d4b6ee33200fccaa1d81837e49
 configure-command = true
 # If the system is running on Linux 2.6, we use "linux26" as the TARGET,
 # otherwise use "generic".

component/jemalloc/buildout.cfg

@@ -5,7 +5,8 @@ parts =
 
 [jemalloc]
 recipe = slapos.recipe.cmmi
-url = http://www.canonware.com/download/jemalloc/jemalloc-3.6.0.tar.bz2
+url = https://github.com/jemalloc/jemalloc/releases/download/${:version}/jemalloc-${:version}.tar.bz2
+version = 3.6.0
 md5sum = e76665b63a8fddf4c9f26d2fa67afdf2
 configure-options =
   --disable-static

component/keras/buildout.cfg

@@ -25,7 +25,7 @@ interpreter = keras-python
 scripts = keras-python
 
 [versions]
-Keras = 2.0.8
-tensorflow = 1.4.0rc1
+Keras = 2.1.0
+tensorflow = 1.4.0
 h5py = 2.7.0rc2
 Cython = 0.25.2

component/mariadb/buildout.cfg

@@ -12,7 +12,6 @@ extends =
   ../ncurses/buildout.cfg
   ../openssl/buildout.cfg
   ../patch/buildout.cfg
-  ../pcre/buildout.cfg
   ../pkgconfig/buildout.cfg
   ../readline/buildout.cfg
   ../xz-utils/buildout.cfg
@@ -27,8 +26,9 @@ parts =
 
 [mariadb]
 recipe = slapos.recipe.cmmi
-url = https://downloads.mariadb.org/f/mariadb-10.1.28/source/mariadb-10.1.28.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
-md5sum = 38acd5b44c56791701d80fddf088ef38
+url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
+version = 10.1.30
+md5sum = c424fd12bdff388e3da1bdecf42626c9
 patch-options = -p0
 patches =
   ${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3
@@ -45,7 +45,7 @@ configure-options =
   -DWITH_ZLIB=system
   -DWITH_READLINE=0
   -DWITH_PIC=1
-  -DWITH_PCRE=system
+  -DWITH_PCRE=bundled
   -DENABLE_DTRACE=0
   -DWITH_EXTRA_CHARSETS=complex
   -DWITH_EMBEDDED_SERVER=0
@@ -56,22 +56,22 @@ configure-options =
   -DWITH_INNODB_LZO=OFF
   -DWITH_INNODB_SNAPPY=OFF
   -DWITH_SAFEMALLOC=OFF
-  -DWITHOUT_EXAMPLE_STORAGE_ENGINE=1
-  -DWITHOUT_MROONGA_STORAGE_ENGINE=1
-  -DWITHOUT_DAEMON_EXAMPLE=1
+  -DPLUGIN_DAEMON_EXAMPLE=NO
+  -DPLUGIN_EXAMPLE=NO
+  -DPLUGIN_MROONGA=NO
   -DCMAKE_C_FLAGS="${:CMAKE_CFLAGS}"
   -DCMAKE_CXX_FLAGS="${:CMAKE_CFLAGS}"
   -DCMAKE_INSTALL_RPATH=${:CMAKE_LIBRARY_PATH}
-CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include ${:extra_cflags}
-CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib${:extra_library_path}
+CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include ${:extra_cflags}
+CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib${:extra_library_path}
 extra_cflags =
 extra_include_path =
 extra_library_path =
 environment =
   CMAKE_PROGRAM_PATH=${cmake:location}/bin
-  CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${pcre:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include${:extra_include_path}
+  CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include${:extra_include_path}
   CMAKE_LIBRARY_PATH=${:CMAKE_LIBRARY_PATH}
-  LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${pcre:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib
+  LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib
   PATH=${patch:location}/bin:%(PATH)s
 post-install =
   mkdir -p ${:location}/include/wsrep &&
@@ -108,6 +108,6 @@ pre-build =
   sed -i -e "s,${mariadb:location}/include,$(pwd)/fake_mariadb_source/include,g" Makefile */Makefile
 environment =
   PATH=${groonga:location}/bin:${patch:location}/bin:${pkgconfig:location}/bin:%(PATH)s
-  CPPFLAGS=-I${groonga:location}/include/groonga -I${pcre:location}/include
+  CPPFLAGS=-I${groonga:location}/include/groonga
   LDFLAGS=-L${groonga:location}/lib
   PKG_CONFIG_PATH=${groonga:location}/lib/pkgconfig:${groonga-normalizer-mysql:location}/lib/pkgconfig

component/mariadb/mariarocks.cfg

 # Do not extend any file that touch buildout:parts.
 
 [mariadb]
-url = https://downloads.mariadb.org/f/mariadb-10.2.9/source/mariadb-10.2.9.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
-md5sum = c59999bd182ddeb3db3d55250aecd8f8
+version = 10.2.12
+md5sum = c5bce588e3c53ebc417e37ecb0967aee
 stable-patches =
 configure-options +=
-  -DPLUGIN_DAEMON_EXAMPLE=NO
-  -DPLUGIN_EXAMPLE=NO
-  -DPLUGIN_MROONGA=NO
   -DCMAKE_C_COMPILER=${gcc:location}/bin/gcc
   -DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++
 extra_cflags = -I${zstd:location}/include

component/nginx/buildout.cfg

@@ -18,6 +18,7 @@ patch-options = -p0
 patch-binary = ${patch:location}/bin/patch
 patches =
   http://nginx.org/download/patch.2017.ranges.txt#40bf9f37c881cb3b10cfefd84ca92f6a
+  ${:_profile_base_location_}/fix-gcc7-implicit-fallthrough-errors.patch
 
 [nginx]
 <= nginx-common

component/nginx/fix-gcc7-implicit-fallthrough-errors.patch

+commit 8449f750e62cd229026e9df3bd023ec7e073a7d4
+Author: Maxim Dounin <mdounin@mdounin.ru>
+Date:   Thu Apr 27 16:57:18 2017 +0300
+
+    Added missing "fall through" comments (ticket #1259).
+    
+    Found by gcc7 (-Wimplicit-fallthrough).
+
+diff --git src/core/ngx_murmurhash.c src/core/ngx_murmurhash.c
+index c31e0e03..5ade658d 100644
+--- src/core/ngx_murmurhash.c
++++ src/core/ngx_murmurhash.c
+@@ -35,8 +35,10 @@ ngx_murmur_hash2(u_char *data, size_t len)
+     switch (len) {
+     case 3:
+         h ^= data[2] << 16;
++        /* fall through */
+     case 2:
+         h ^= data[1] << 8;
++        /* fall through */
+     case 1:
+         h ^= data[0];
+         h *= 0x5bd1e995;
+diff --git src/http/ngx_http_parse.c src/http/ngx_http_parse.c
+index 36220fdc..e8e51563 100644
+--- src/http/ngx_http_parse.c
++++ src/http/ngx_http_parse.c
+@@ -1396,6 +1396,7 @@ ngx_http_parse_complex_uri(ngx_http_request_t *r, ngx_uint_t merge_slashes)
+                 goto done;
+             case '+':
+                 r->plus_in_uri = 1;
++                /* fall through */
+             default:
+                 state = sw_usual;
+                 *u++ = ch;
+@@ -1437,6 +1438,7 @@ ngx_http_parse_complex_uri(ngx_http_request_t *r, ngx_uint_t merge_slashes)
+                 goto done;
+             case '+':
+                 r->plus_in_uri = 1;
++                /* fall through */
+             default:
+                 state = sw_usual;
+                 *u++ = ch;
+@@ -1484,6 +1486,7 @@ ngx_http_parse_complex_uri(ngx_http_request_t *r, ngx_uint_t merge_slashes)
+                 goto done;
+             case '+':
+                 r->plus_in_uri = 1;
++                /* fall through */
+             default:
+                 state = sw_usual;
+                 *u++ = ch;
+diff --git src/os/unix/ngx_process.c src/os/unix/ngx_process.c
+index dd50b5ca..993c032a 100644
+--- src/os/unix/ngx_process.c
++++ src/os/unix/ngx_process.c
+@@ -413,6 +413,7 @@ ngx_signal_handler(int signo, siginfo_t *siginfo, void *ucontext)
+                 break;
+             }
+             ngx_debug_quit = 1;
++            /* fall through */
+         case ngx_signal_value(NGX_SHUTDOWN_SIGNAL):
+             ngx_quit = 1;
+             action = ", shutting down";

component/ocropy/buildout.cfg

@@ -32,8 +32,8 @@ setup-eggs =
   ${matplotlib:egg}
   ${pillow-python:egg}
 patches =
-  ${:_profile_base_location_}/ocropy.patch
-patch-options = -p0
+  ${:_profile_base_location_}/ocropy.patch#dd7a02e1e63ed9df68e3a539b3e919eb
+patch-options = -p1
 patch-binary = ${patch:location}/bin/patch
 environment = ocropy-env
 find-links = https://github.com/tmbdev/ocropy/tarball/4efbddca22bb2f0c639af0694e7a1386f2f097b5/ocropy-1.0.tar.gz

component/ocropy/ocropy.patch

-diff --git ocrolib/__init__.py ocrolib/__init__.py
+From 1bb1546b12b0c08b1b32b293207de2d58d43ff1c Mon Sep 17 00:00:00 2001
+From: Francois Le Corre <francois.lecorre@nexedi.com>
+Date: Thu, 6 Apr 2017 11:32:27 +0200
+Subject: [PATCH] WIP
+
+- New OCROPY_MODEL_PATH environment variable to specify the position of models
+- Import for the first time a module that compiles C code from python,
+  allowing us to have the static library built before runtime and
+  preventing us from using gcc through zope.
+---
+ ocrolib/__init__.py |  3 ++-
+ ocrolib/common.py   |  7 ++++---
+ ocrolib/native.py   |  1 +
+ setup.py            | 15 ++++++++++++---
+ 4 files changed, 19 insertions(+), 7 deletions(-)
+
+diff --git a/ocrolib/__init__.py b/ocrolib/__init__.py
 index 1e0d627..81e85fb 100644
---- ocrolib/__init__.py
-+++ ocrolib/__init__.py
+--- a/ocrolib/__init__.py
++++ b/ocrolib/__init__.py
 @@ -1,7 +1,7 @@
  __all__ = [
      "binnednn","cairoextras","common","components","dbtables",
@@ -11,18 +27,18 @@ index 1e0d627..81e85fb 100644
  ]
  
  ################################################################
-@@ -9,5 +9,6 @@ __all__ = [
+@@ -9,5 +9,6 @@
  ################################################################
  
  import default
 +from psegutils import *
  from common import *
  from default import traceback as trace
-diff --git ocrolib/common.py ocrolib/common.py
+diff --git a/ocrolib/common.py b/ocrolib/common.py
 index 27c0f26..14f088f 100644
---- ocrolib/common.py
-+++ ocrolib/common.py
-@@ -14,6 +14,7 @@ import unicodedata
+--- a/ocrolib/common.py
++++ b/ocrolib/common.py
+@@ -14,6 +14,7 @@
  import inspect
  import glob
  import cPickle
@@ -38,7 +54,7 @@ index 27c0f26..14f088f 100644
  def load_object(fname,zip=0,nofind=0,verbose=0):
      """Loads an object from disk. By default, this handles zipped files
      and searches in the usual places for OCRopus. It also handles some
-@@ -439,8 +441,7 @@ def load_object(fname,zip=0,nofind=0,verbose=0):
+@@ -439,8 +441,7 @@ class names that have changed."""
      if zip==0 and fname.endswith(".gz"):
          zip = 1
      if zip>0:
@@ -57,10 +73,10 @@ index 27c0f26..14f088f 100644
  
      possible_prefixes.append("/usr/local/share/ocropus")
  
-diff --git ocrolib/native.py ocrolib/native.py
+diff --git a/ocrolib/native.py b/ocrolib/native.py
 index b7a207f..240450b 100644
---- ocrolib/native.py
-+++ ocrolib/native.py
+--- a/ocrolib/native.py
++++ b/ocrolib/native.py
 @@ -44,6 +44,7 @@ class CompileError(Exception):
  
  def compile_and_find(c_string,prefix=".pynative",opt="-g -O4",libs="-lm",
@@ -69,22 +85,22 @@ index b7a207f..240450b 100644
      if not os.path.exists(prefix):
          os.mkdir(prefix)
      m = hashlib.md5()
-diff --git setup.py setup.py
-index 2ec5832..6697b12 100644
---- setup.py
-+++ setup.py
-@@ -10,7 +10,9 @@ assert sys.version_info[0]==2 and sys.version_info[1]>=7,\
+diff --git a/setup.py b/setup.py
+index 2ec5832..0ad4d85 100644
+--- a/setup.py
++++ b/setup.py
+@@ -10,7 +10,9 @@
  from distutils.core import setup #, Extension, Command
  #from distutils.command.install_data import install_data
  
 -if not os.path.exists("models/en-default.pyrnn.gz"):
-+models = os.environ.get('OCROPY_MODEL_PATH', '').split(':') or \
-+    [c for c in glob.glob("models/*pyrnn.gz")]
++models = os.environ.get('OCROPY_MODEL_PATH')
++models = models.split(':') if models else glob.glob("models/*pyrnn.gz")
 +if not models:
      print()
      print("You should download the default model 'en-default.pyrnn.gz'")
      print("and put it into ./models.")
-@@ -18,16 +20,23 @@ if not os.path.exists("models/en-default.pyrnn.gz"):
+@@ -18,16 +20,23 @@
      print("Check https://github.com/tmbdev/ocropy for the location")
      print("of model files.")
      print()
@@ -110,3 +126,6 @@ index 2ec5832..6697b12 100644
 +    data_files= [('share/ocropus', models), ("", ["LICENSE"])],
      scripts = scripts,
      )
+-- 
+2.14.1
+

component/openssh/buildout.cfg

@@ -27,4 +27,13 @@ environment =
 configure-options = 
   --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
   --exec-prefix=${buildout:parts-directory}/${:_buildout_section_name_}
-  --with-privsep-path=${buildout:parts-directory}/${:_buildout_section_name_}/var/empty
\ No newline at end of file
+  --with-privsep-path=${buildout:parts-directory}/${:_buildout_section_name_}/var/empty
+
+[openssh-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${coreutils-output:test} -x ${:ssh} -a -x ${:keygen}
+ssh = ${openssh:location}/bin/ssh
+keygen = ${openssh:location}/bin/ssh-keygen
\ No newline at end of file

component/openssl/buildout.cfg

@@ -16,8 +16,8 @@ parts =
 
 [openssl]
 recipe = slapos.recipe.cmmi
-url = https://www.openssl.org/source/openssl-1.0.2m.tar.gz
-md5sum = 10e9e37f492094b9ef296f68f24a7666
+url = https://www.openssl.org/source/openssl-1.0.2n.tar.gz
+md5sum = 13bdc1b1d1ff39b6fd42a255e74676a4
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 # 'prefix' option to override --openssldir/--prefix (which is useful
 # when combined with INSTALL_PREFIX). Used by slapos.package.git/obs

component/rdiff-backup/buildout.cfg

@@ -22,6 +22,13 @@ rpath =
 # and pin the egg in the [versions] section of the stack or SR.
 find-links = http://pkgs.fedoraproject.org/repo/pkgs/rdiff-backup/rdiff-backup-1.0.5.tar.gz/fa2a165fa07a94be52c52e3545bc7758/rdiff-backup-1.0.5.tar.gz
 
+[rdiff-backup-build-1.3.4]
+<= rdiff-backup-build
+# use our own version
+find-links = http://www.nexedi.org/static/packages/source/rdiff-backup-1.3.4nxd2.tar.gz
+patches =
+  ${:_profile_base_location_}/rdiff-backup-1.3.4-librsync-1.0.0.patch#31fafc8bc4a00f002f52008a9f3b671f
+
 [rdiff-backup]
 # Scripts only generation part for rdiff-backup
 recipe = zc.recipe.egg

component/rdiff-backup/rdiff-backup-1.3.4-librsync-1.0.0.patch

+Patch by Roman Tereshonkov and Kari Hautio for rdiff-backup <= 1.2.8 to avoid a build failure with
+librsync >= 1.0.0 (which is a security bugfix release). The discussion and solution finding can be
+found at https://bugs.launchpad.net/duplicity/+bug/1416344 (for duplicity).
+
+--- rdiff-backup-1.3.4/rdiff_backup/_librsyncmodule.c			2009-03-16 15:36:21.000000000 +0100
++++ rdiff-backup-1.3.4/rdiff_backup/_librsyncmodule.c.librsync-1.0.0		2015-03-02 00:54:24.000000000 +0100
+@@ -59,8 +59,13 @@
+   if (sm == NULL) return NULL;
+   sm->x_attr = NULL;
+ 
++#ifdef RS_DEFAULT_STRONG_LEN
+   sm->sig_job = rs_sig_begin((size_t)blocklen,
+ 							 (size_t)RS_DEFAULT_STRONG_LEN);
++#else
++  sm->sig_job = rs_sig_begin((size_t)blocklen,
++							 (size_t)8, RS_MD4_SIG_MAGIC);
++#endif
+   return (PyObject*)sm;
+ }
+ 

component/tensorboard/buildout.cfg

@@ -56,14 +56,16 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
 need-tensorboard-build = ${tensorboard-build:location}
 egg = tensorflow-tensorboard
 bazel-bin = ${bazel:location}/bin
+java_home_bin = ${bazel:java_home}/bin
 numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter}
 script =
   os.makedirs(location)
   workdir = self.options['tensorboard-repository-path']
-  egg_name = 'tensorflow_tensorboard-0.4.0rc1-py2.7.egg'
+  egg_name = 'tensorflow_tensorboard-0.4.0rc3-py2.7.egg'
   dist_dir = os.path.join(workdir, 'dist')
   dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
   env = {'PATH':':'.join([self.options['bazel-bin'],
+                          self.options['java_home_bin'],
                           os.environ['PATH']]),
          'PYTHON_BIN_PATH':self.options['numpy-python-command'],
          }

component/tensorflow/buildout.cfg

@@ -27,7 +27,7 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
 recipe = plone.recipe.command
 stop-on-error = true
 repository = https://github.com/tensorflow/tensorflow
-tag = r1.4
+tag = v1.4.0
 git-binary = ${git:location}/bin/git
 patch-binary = ${patch:location}/bin/patch
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
@@ -110,7 +110,7 @@ egg = tensorflow
 script =
   os.makedirs(location)
   workdir = self.options['tensorflow-repository-path']
-  egg_name = 'tensorflow-1.4.0rc1-py2.7-linux-x86_64.egg'
+  egg_name = 'tensorflow-1.4.0-py2.7-linux-x86_64.egg'
   dist_dir = os.path.join(workdir, 'dist')
   dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
   call(['bazel-bin/tensorflow/tools/pip_package/build_pip_package', dist_dir], cwd=workdir)

setup.py

@@ -55,11 +55,12 @@ setup(name=name,
       packages=find_packages(),
       include_package_data=True,
       install_requires=[
+        'enum34',  # for inotify-simple
         'jsonschema',
         'hexagonit.recipe.download',
         'netaddr', # to manipulate on IP addresses
         'setuptools', # namespaces
-        'inotifyx', # XXX use pyinotify instead
+        'inotify_simple',
         'lock_file', #another lockfile implementation for multiprocess
         'slapos.core', # uses internally
         'zc.buildout', # plays with buildout

slapos/recipe/erp5_test/__init__.py

@@ -85,7 +85,7 @@ class Recipe(GenericBaseRecipe):
             mysql_connection_string_list),
           ] + common_list, **common_dict)]))
     path_list.append(self.createPythonScript(self.options['run-test-suite'],
-        __name__ + '.test.runUnitTest', [dict(
+        __name__ + '.test.runTestSuite', [dict(
         call_list=[self.options['run-test-suite-binary'],
           '--db_list', ','.join(mysql_connection_string_list),
           ] + common_list, **common_dict)]))

slapos/recipe/erp5_test/test.py

@@ -34,7 +34,10 @@ def runTestSuite(args):
   if 'test_ca_path' in d:
     env['TEST_CA_PATH'] = d['test_ca_path']
   if 'prepend_path' in d:
-    env['PATH'] = ':'.join([d['prepend_path']] + os.environ.get('PATH', '').split(':'))
+    try:
+      env['PATH'] = d['prepend_path'] + ':' + env['PATH']
+    except KeyError:
+      env['PATH'] = d['prepend_path']
   if 'instance_home' in d:
     env['INSTANCE_HOME'] = d['instance_home']
     env['REAL_INSTANCE_HOME'] = d['instance_home']
@@ -57,29 +60,4 @@ def runTestSuite(args):
   argument_list.append(env)
   os.execle(executable_filepath, *argument_list)
 
-def runUnitTest(args):
-  env = os.environ.copy()
-  d = args[0]
-  if 'openssl_binary' in d:
-    env['OPENSSL_BINARY'] = d['openssl_binary']
-  if 'test_ca_path' in d:
-    env['TEST_CA_PATH'] = d['test_ca_path']
-  if 'prepend_path' in d:
-    env['PATH'] = ':'.join([d['prepend_path']] + os.environ.get('PATH', '').split(':'))
-  if 'instance_home' in d:
-    env['INSTANCE_HOME'] = d['instance_home']
-    env['REAL_INSTANCE_HOME'] = d['instance_home']
-  # Deal with Shebang size limitation
-  executable_filepath = d['call_list'][0]
-  file_object = open(executable_filepath, 'r')
-  line = file_object.readline()
-  file_object.close()
-  argument_list = []
-  if line[:2] == '#!':
-    executable_filepath = line[2:].strip()
-    argument_list.append(executable_filepath)
-  argument_list.extend(d['call_list'])
-  argument_list.extend(sys.argv[1:])
-  argument_list.append(env)
-  os.execle(executable_filepath, *argument_list)
-
+runUnitTest = runTestSuite

slapos/recipe/librecipe/execute.py

@@ -3,47 +3,31 @@ import os
 import signal
 import subprocess
 import time
-
-import inotifyx
+from collections import defaultdict
+from inotify_simple import INotify, flags
 
 def _wait_files_creation(file_list):
-  # Etablish a list of directory and subfiles
-  directories = dict()
-  for dirname, filename in [os.path.split(f) for f in file_list]:
-    directories.setdefault(dirname, dict())
-    directories[dirname][filename] = False
+  # Establish a list of directory and subfiles.
+  # and test existence before watching, so that we don't miss an event.
+  directories = defaultdict(dict)
+  for f in file_list:
+    dirname, filename = os.path.split(f)
+    directories[dirname][filename] = os.path.lexists(f)
 
   def all_files_exists():
-    return all([all(files.values()) for files in directories.values()])
+    return all(all(files.itervalues()) for files in directories.itervalues())
 
-  fd = inotifyx.init()
-  try:
-    # Watch every directories where the file are
-    watchdescriptors = dict()
-    for dirname in directories.keys():
-      wd = inotifyx.add_watch(fd,
-                      dirname,
-                      inotifyx.IN_CREATE | inotifyx.IN_DELETE | inotifyx.IN_MOVE)
-      watchdescriptors[wd] = dirname
-
-    # Set to True the file wich exists
-    for dirname, filename in [os.path.split(f) for f in file_list]:
-      directories[dirname][filename] = os.path.exists(os.path.join(dirname,
-                                                                   filename))
-    # Let's wait for every file creation
-    while not all_files_exists():
-      events_list = inotifyx.get_events(fd)
-      for event in events_list:
-        dirname = watchdescriptors[event.wd]
-        if event.name in directories[dirname]:
-          # One of watched file was created or deleted
-          if event.mask & inotifyx.IN_DELETE:
-            directories[dirname][event.name] = False
-          else:
-            directories[dirname][event.name] = True
+  with INotify() as inotify:
+    watchdescriptors = {inotify.add_watch(dirname,
+        flags.CREATE | flags.DELETE | flags.MOVED_TO | flags.MOVED_FROM
+        ): dirname
+      for dirname in directories}
 
-  finally:
-    os.close(fd)
+    while not all_files_exists():
+      for event in inotify.read():
+        directory = directories[watchdescriptors[event.wd]]
+        if event.name in directory:
+          directory[event.name] = event.mask & (flags.CREATE | flags.MOVED_TO)
 
 def execute(args):
   """Portable execution with process replacement"""
@@ -83,8 +67,8 @@ def generic_exec(args):
 
   os.execve(exec_list[0], exec_list + sys.argv[1:], exec_env)
 
-def sig_handler(signal, frame):
-  print 'Received signal %r, killing children and exiting' % signal
+def sig_handler(sig, frame):
+  print 'Received signal %r, killing children and exiting' % sig
   if child_pg is not None:
     os.killpg(child_pg, signal.SIGHUP)
     os.killpg(child_pg, signal.SIGTERM)
@@ -97,6 +81,7 @@ signal.signal(signal.SIGTERM, sig_handler)
 
 def execute_with_signal_translation(args):
   """Run process as children and translate from SIGTERM to another signal"""
+  global child_pg
   child = subprocess.Popen(args, close_fds=True, preexec_fn=os.setsid)
   child_pg = child.pid
   try:

slapos/recipe/librecipe/inotify.py

@@ -25,8 +25,7 @@
 #
 ##############################################################################
 import os
-
-import inotifyx
+from inotify_simple import INotify, flags
 
 def subfiles(directory):
   """Return the list of subfiles of a directory, and wait for the newly created
@@ -34,18 +33,12 @@ def subfiles(directory):
 
   CAUTION : *DONT TRY TO CONVERT THE RESULT OF THIS FUNCTION INTO A LIST !
   ALWAYS ITERATE OVER IT !!!*"""
-  watchfd = inotifyx.init()
-  inotifyx.add_watch(watchfd, directory, inotifyx.IN_CREATE)
-  try:
 
-    subfiles = set(os.listdir(directory))
-    subfiles |= set([file_.name for file_ in inotifyx.get_events(watchfd, 0)])
+  with INotify() as inotify:
+    inotify.add_watch(directory, flags.CLOSE_WRITE | flags.MOVED_TO)
 
+    names = os.listdir(directory)
     while True:
-      for file_ in subfiles:
-        yield os.path.join(directory, file_)
-
-      subfiles = [file_.name for file_ in inotifyx.get_events(watchfd)]
-
-  finally:
-    os.close(watchfd)
+      for name in names:
+        yield os.path.join(directory, name)
+      names = (event.name for event in inotify.read())

slapos/recipe/neoppod.py

@@ -92,6 +92,8 @@ class Storage(NeoBaseRecipe):
     engine = self.options.get('engine')
     if engine: # old versions of NEO don't support -e
       r  += '-e', engine
+    if self.options.get('dedup'):
+      r.append('--dedup')
     if self.options.get('disable-drop-partitions'):
       r.append('--disable-drop-partitions')
     return r

slapos/test/recipe/test_inotify.py

+import os, shutil, tempfile, threading, unittest
+from slapos.recipe.librecipe import execute, inotify
+
+class TestInotify(unittest.TestCase):
+
+  def setUp(self):
+    self.tmp = tempfile.mkdtemp()
+
+  def tearDown(self):
+    shutil.rmtree(self.tmp)
+
+  def test_subfiles(self):
+    p = lambda x: os.path.join(self.tmp, x)
+    def create(name, text):
+      a = open(p(name), 'w')
+      a.write(text)
+      a.flush()
+      return a
+    def check(name, text):
+      path = next(notified)
+      self.assertEqual(path, p(name))
+      with open(path) as f:
+        self.assertEqual(f.read(), text)
+    a = create('first', 'blah')
+    a.write('...')
+    notified = inotify.subfiles(self.tmp)
+    check('first', 'blah')
+    os.link(p(a.name), p('a hard link')) # ignored
+    b = create('other', 'hello')
+    b.close()
+    check('other', 'hello')
+    c = create('last', '!!!')
+    a.close()
+    check('first', 'blah...')
+    os.rename(p(a.name), p(b.name))
+    check('other', 'blah...')
+    c.close()
+    check('last', '!!!')
+
+  def test_wait_files_creation(self):
+    file_list = (
+      'foo',
+      'bar',
+      'hello/world',
+      'hello/world!',
+      'a/b/c',
+    )
+    create = lambda x: open(x, 'w').close()
+    p = lambda x: os.path.join(self.tmp, x)
+    P = lambda x: p(file_list[x])
+    create(P(1))
+    os.mkdir(p('hello'))
+    os.makedirs(p('a/b'))
+    t = threading.Thread(target=execute._wait_files_creation,
+                         args=(map(p, file_list),))
+    t.daemon = True
+    t.start()
+    def check():
+      t.join(.2)
+      self.assertTrue(t.is_alive())
+    check()
+    for x in P(3), p('a/b/d'), P(0):
+      create(x)
+    check()
+    os.rename(P(3), P(2))
+    os.rename(p('a/b/d'), P(4))
+    check()
+    os.remove(P(1))
+    for x in P(3), P(1):
+      create(x)
+    t.join(10)
+    self.assertFalse(t.is_alive())

software/agent/software.cfg

@@ -50,7 +50,7 @@ gitdb = 0.6.4
 pycrypto = 2.6.1
 pycurl = 7.43.0
 slapos.recipe.download = 1.0
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 slapos.toolbox = 0.73
 smmap = 0.9.0
 

software/apache-frontend/software.cfg

@@ -10,7 +10,7 @@ gitdb = 0.6.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
 rdiff-backup = 1.0.5+SlapOSPatched001
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 slapos.toolbox = 0.73
 smmap = 0.9.0
 numpy = 1.11.2

software/backupserver/instance-pullrdiffbackup.cfg.in

@@ -27,6 +27,7 @@ statistic = $${:srv}/statistic
 backupscript = $${:etc}/backup
 www = $${:srv}/www
 home = $${:etc}/home
+promises = $${:etc}/promise
 ssl = $${:etc}/ssl
 ssh = $${:home}/.ssh
 
@@ -49,12 +50,13 @@ logfile = $${directory:log}/crond.log
 {%   set frequency = slave_instance.get('frequency', '') -%}
 {%   set hostname = slave_instance.get('hostname', '') -%}
 {%   set connection = slave_instance.get('connection', '') -%}
+{%   set connection_port = slave_instance.get('connection_port', '22') -%}
 {%   set include = slave_instance.get('include', '') -%}
 {%   set include_string = "' --include='".join(include.split(' ')) -%}
 {%   set exclude = slave_instance.get('exclude', '') -%}
 {%   set exclude_string = '' -%}
 {%   set sudo = slave_instance.get('sudo', 'False') -%}
-{%   set remote_schema = 'rdiff-backup --server --restrict-read-only / -- "$@"' -%}
+{%   set remote_schema = slave_instance.get('remote_rdiff_path', 'rdiff-backup') + ' --server --restrict-read-only / -- "$@"' -%}
 
 {%   if (exclude != '') -%}
 {%     set exclude_string = "' --exclude='".join(exclude.split(' ')) -%}
@@ -73,23 +75,19 @@ directory = $${directory:backup}/$${:_buildout_section_name_}
 
 [{{ slave_reference }}-backup-private_key]
 recipe = plone.recipe.command
-stop-on-error = false
-command = ${dropbear-output:keygen} -t $${:type} -s 2048 -f $${:key} 
+stop-on-error = true
+command = ${coreutils-output:rm} -f $${:key} $${:public_key} && ${openssh-output:keygen} -t $${:type} -b 2048 -f $${:key} -q -N ""
 key = $${directory:ssh}/$${:_buildout_section_name_}
+public_key = $${:key}.pub
+location = $${:public_key}
 type = rsa
 
-[{{ slave_reference }}-backup-public_key]
-recipe = plone.recipe.command
-stop-on-error = true
-command = ${coreutils-output:rm} -f $${:key} && ${dropbear-output:keygen} -y -f {{ '$${' ~ slave_reference }}-backup-private_key:key} | ${grep-output:grep} {{ '$${' ~ slave_reference }}-backup-private_key:type} > $${:key}
-key = {{ '$${' ~ slave_reference }}-backup-private_key:key}.pub
-location = $${:key}
 # Insert as a beginning part, to ensure that all public keys are generated before trying to publish. This will reduce the number of slapgrid-cp run.
-{%   do part_list.insert(0, "%s-backup-public_key" % slave_reference) -%}
+{%   do part_list.insert(0, "%s-backup-private_key" % slave_reference) -%}
 
 [{{ slave_reference }}-backup-read-public_key]
 recipe = slapos.cookbook:readline
-storage-path = {{ '$${' ~ slave_reference }}-backup-public_key:key}
+storage-path = {{ '$${' ~ slave_reference }}-backup-private_key:public_key}
 
 # Publish slave {{ slave_reference }} information
 [{{ slave_reference }}-backup-publish]
@@ -107,6 +105,7 @@ mode = 0700
 datadirectory = {{ '$${' ~ slave_reference }}-backup-directory:directory}
 sshkey = {{ '$${' ~ slave_reference }}-backup-private_key:key}
 connection = {{ connection }}
+connection_port = {{ connection_port }}
 hostname = {{ hostname }}
 include = {{ include_string }}
 exclude_string = {{ exclude_string }}
@@ -169,6 +168,12 @@ mode = 0700
 virtual-depends = 
   $${nginx-configuration:ip}
 
+[nginx-listen-promise]
+recipe = slapos.cookbook:check_port_listening
+hostname = $${nginx-configuration:ip}
+port = $${nginx-configuration:port}
+path = $${directory:promises}/nginx_listen
+
 [nginx-configuration]
 recipe = slapos.recipe.template
 url = ${template-nginx-configuration:output}
@@ -187,6 +192,7 @@ ssl_crt = $${directory:ssl}/nginx.crt
 parts =
   dcron-service
   nginx-service
+  nginx-listen-promise
   activate-crontab-file
   publish-global-rss
 {% for part in part_list -%}

software/backupserver/software.cfg

@@ -10,7 +10,7 @@ extends =
 #   ../../component/git/buildout.cfg
 #   ../../component/subversion/buildout.cfg
   ../../component/rsync/buildout.cfg
-  ../../component/dropbear/buildout.cfg
+  ../../component/openssh/buildout.cfg
   ../../component/grep/buildout.cfg
   ../../component/findutils/buildout.cfg
 #   ../../stack/flask.cfg
@@ -67,7 +67,7 @@ mode = 0644
 [template-backup-script]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/template-backup-script.sh.in
-md5sum = 47b20031db3b575651d8515d5add23e6
+md5sum = fa79e0307e12e2f5b1f2adbd261995fc
 output = ${buildout:directory}/template-backup-script.sh.in
 mode = 0644
 
@@ -88,7 +88,7 @@ mode = 0644
 [status2rss]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/status2rss.py
-md5sum = 0cd1cf97b199dd18fc0168c7281890ea
+md5sum = a023694817975e73998fb9187a6015d6
 output = ${buildout:directory}/status2rss.py
 mode = 0644
 
@@ -105,7 +105,7 @@ mode = 0644
 [template-pullrdiffbackup]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-pullrdiffbackup.cfg.in
-md5sum = 061b98d001b501c9e1beb424e8802d3d
+md5sum = a2fb7b0cdd944be99da4122eb6f07749
 output = ${buildout:directory}/template-pullrdiffbackup.cfg
 mode = 0644
 
@@ -116,8 +116,14 @@ md5sum = 42021b325159dff29e4bd4e33b8ff2f3
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
+[rdiff-backup]
+eggs =
+  ${rdiff-backup-build-1.3.4:egg}
+
 [versions]
-rdiff-backup = 1.0.5+SlapOSPatched001
+# 1.3.4nxd2 is invalid version string, thus pached version string is not '1.3.4nxd2+SlapOSPatched001'
+# but '1.3.4nxd2-SlapOSPatched001'.
+rdiff-backup = 1.3.4nxd2-SlapOSPatched001
 gunicorn = 19.1.1
 plone.recipe.command = 1.1
 slapos.recipe.template = 2.4.2

software/backupserver/status2rss.py

@@ -29,9 +29,8 @@ while 1:
     title = desc,
     description = "<p>%s</p>" % "<br/>".join(("%s, %s\n<a href='http://www.nongnu.org/rdiff-backup/FAQ.html#statistics'>Lastest statistic</a>\n%s" % (time, desc,
       open(statistic).read())).split("\n")),
-    link = LINK,
     pubDate = datetime.datetime.fromtimestamp(mktime_tz(parsedate_tz(time))),
-    guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (time, desc)))
+    guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (time, desc)), isPermaLink=0)
     )
   items.append(rss_item)
 

software/backupserver/template-backup-script.sh.in

@@ -18,7 +18,7 @@ ${rdiff-backup-output:rdiff-backup} \
   $${:exclude_string} \
   --include='$${:include}' \
   --exclude='**' \
-  --remote-schema '${dropbear-output:ssh} -T -y -i $${:sshkey} %s $${:remote_schema}' \
+  --remote-schema '${openssh-output:ssh} -6 -q -T -y -o "StrictHostKeyChecking no" -i $${:sshkey} -p $${:connection_port} %s $${:remote_schema}' \
   $${:connection}::/ ./
 
 RESULT=$?

software/buildout-testing/runTestSuite.in

@@ -77,7 +77,7 @@ def main():
 
   test_title = args.test_suite_title or args.test_suite
   if args.master_url:
-    tool = taskdistribution.TaskDistributionTool(args.master_url)
+    tool = taskdistribution.TaskDistributor(args.master_url)
     test_result = tool.createTestResult(args.revision,
                                         list(test_dict),
                                         args.test_node_title,

software/cdn-me/software.cfg

@@ -40,4 +40,4 @@ cns.recipe.symlink = 0.2.3
 collective.recipe.environment = 0.2.0
 erp5.util = 0.4.49
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2

software/dream/software.cfg

@@ -53,7 +53,7 @@ mysqlclient = 1.3.12
 # indirect dependancies
 cp.recipe.cmd = 0.5
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 zope.exceptions = 4.0.7
 zope.testing = 4.1.3
 zc.recipe.testrunner = 2.0.0

software/erp5/README.rst

@@ -3,12 +3,8 @@ Available ``software-type`` values
 
 - ``default``
 
-  Recommended for production use.
-
-- ``create-erp5-site``
-
-  Automated creation of ERP5Site instance, for easy deployment.
-  Usage in production discouraged due to the increased risk of data loss.
+  Recommended for developemnt and production use. Automatic creation of
+  erp5-site.
 
 Notes
 =====

software/erp5/software.cfg.json

@@ -9,13 +9,6 @@
       "request": "instance-erp5-input-schema.json",
       "response": "instance-erp5-output-schema.json",
       "index": 0
-    },
-    "create-erp5-site": {
-      "title": "Create ERP5 Site", 
-      "description": "Automated ERP5Site creation on instanciation when ZODB is found empty.",
-      "request": "instance-erp5-input-schema.json",
-      "response": "instance-erp5-output-schema.json",
-      "index": 1
     }
   }
 }

software/erp5testnode/software.cfg

@@ -58,8 +58,8 @@ mode = 0644
 
 [versions]
 PyXML = 0.8.5
-erp5.util = 0.4.49
-slapos.recipe.template = 4.1
+erp5.util = 0.4.50
+slapos.recipe.template = 4.2
 ipython = 5.3.0
 apache-libcloud = 2.1.0
 gitdb2 = 2.0.2

software/erp5testnode/testsuite/deploy-test/runTestSuite.py

@@ -158,10 +158,10 @@ def main():
     access_url_https = 'https://%s:10443' % (args.partition_ipv4,)
     os.environ['TEST_ACCESS_URL_HTTP'] = access_url_http
     os.environ['TEST_ACCESS_URL_HTTPS'] = access_url_https
-  tool = taskdistribution.TaskDistributionTool(
+  distributor = taskdistribution.TaskDistributor(
       args.master_url,
       logger=logger)
-  test_result = tool.createTestResult(
+  test_result = distributor.createTestResult(
       revision, suite.getTestList(), args.test_node_title,
       suite.allow_restart, test_suite_title, args.project_title)
   if test_result is None:

software/erp5testnode/testsuite/deploy-test/software.cfg

@@ -71,4 +71,4 @@ output = ${buildout:directory}/template.cfg
 mode = 0644
 
 [versions]
-erp5.util = 0.4.49
+erp5.util = 0.4.50

software/erp5testnode/testsuite/dummy/software.cfg

@@ -5,6 +5,7 @@ extends =
 
 parts =
   instance-template
+  slapos-cookbook
 
 [instance-template]
 recipe = slapos.recipe.template
@@ -14,4 +15,4 @@ md5sum = efd3b712a2294207f265a9c45648d5cf
 mode = 0644
 
 [versions]
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2

software/gitlab/software.cfg

@@ -381,5 +381,5 @@ cns.recipe.symlink = 0.2.3
 docutils = 0.12
 plone.recipe.command = 1.1
 rubygemsrecipe  = 0.2.2+slapos001
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 z3c.recipe.scripts = 1.0.1

software/html5as/software.cfg

@@ -72,7 +72,7 @@ async = 0.6.1
 gitdb = 0.5.4
 pycrypto = 2.6
 rdiff-backup = 1.0.5+SlapOSPatched001
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 slapos.toolbox = 0.40.4
 smmap = 0.8.2
 plone.recipe.command = 1.1

software/html5ide/software.cfg

@@ -48,4 +48,4 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2

software/jstestnode/runTestSuite.in

@@ -91,8 +91,8 @@ def main():
     if args.target == 'firefox':
       firefox_capabilities = webdriver.common.desired_capabilities.DesiredCapabilities.FIREFOX
       firefox_capabilities['marionette'] = True
-      firefox_capabilities['binary'] = '${firefox:location}/firefox-slapos'
       browser = webdriver.Firefox(capabilities=firefox_capabilities,
+                                  firefox_binary='${firefox:location}/firefox-slapos',
                                   executable_path='${firefox:location}/geckodriver')
     elif args.target in ['iOS', 'Android']:
       # parameters for mobile emulators have different names then parameters for
@@ -169,7 +169,7 @@ def main():
     browser.quit()
     is_browser_running = False
 
-    tool = taskdistribution.TaskDistributionTool(portal_url=args.master_url)
+    tool = taskdistribution.TaskDistributor(portal_url=args.master_url)
     test_result = tool.createTestResult(revision = revision,
                                         test_name_list = test_line_dict.keys(),
                                         node_title = args.test_node_title,

software/jstestnode/software.cfg

@@ -107,11 +107,11 @@ mode = 0644
 [template-runTestSuite]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/runTestSuite.in
-md5sum = fcf15b2a90340e0afe8f8b9921a4ffae
+md5sum = ff66d13f73982e8257eb5535cdb541c7
 output = ${buildout:directory}/runTestSuite.in
 mode = 0644
 
 [versions]
-erp5.util = 0.4.49
-slapos.recipe.template = 4.1
-selenium = 2.53.1
+erp5.util = 0.4.50
+slapos.recipe.template = 4.2
+selenium = 3.8.0

software/jupyter/software.cfg

@@ -85,7 +85,7 @@ pyzmq = 16.0.2
 scikit-learn = 0.18.1
 seaborn = 0.7.1
 simplegeneric = 0.8.1
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 statsmodels = 0.8.0
 terminado = 0.6
 tornado = 4.4.2

software/kvm/software.cfg

@@ -11,7 +11,7 @@ apache-libcloud = 1.1.0
 collective.recipe.environment = 0.2.0
 gitdb = 0.6.4
 pycurl = 7.43.0
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 smmap = 0.9.0
 # websockify = 0.8.0
 

software/nayuos/software.cfg

@@ -17,7 +17,7 @@ parts +=
 versions = versions
 
 [versions]
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 
 
 [template-instance]

software/neoppod/instance-neo-input-schema.json

@@ -70,6 +70,11 @@
             "default": 1,
             "type": "integer"
           },
+          "data-deduplication": {
+              "description": "Set the --dedup option for storage nodes.",
+              "default": false,
+              "type": "boolean"
+          },
           "disable-drop-partitions": {
               "description": "Set the --disable-drop-partitions option for storage nodes.",
               "default": false,

software/neoppod/instance-neo-storage-mysql.cfg.in

@@ -87,6 +87,7 @@ masters = ${publish:masters}
 database-adapter = MySQL
 wait-database = -1
 engine = {{ slapparameter_dict.get('engine', '') }}
+dedup = {{ dumps(bool(slapparameter_dict.get('data-deduplication'))) }}
 disable-drop-partitions = {{ dumps(bool(slapparameter_dict.get('disable-drop-partitions'))) }}
 
 {% for i in range(slapparameter_dict.get('storage-count', 1)) -%}

software/neoppod/software-common.cfg

@@ -98,7 +98,7 @@ md5sum = 1fee10f02c2fa2a581e21878ca0fd704
 
 [instance-neo-storage-mysql]
 <= download-base-neo
-md5sum = 67d623d631c2f99e33bcabc79fc9cccf
+md5sum = 366e51c0dbd85e511a31e403b8704735
 
 [template-neo-my-cnf]
 <= download-base-neo
@@ -116,7 +116,7 @@ mysqlclient = 1.3.12
 persistent = 4.2.3
 pycrypto = 2.6.1
 pycurl = 7.43.0
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 slapos.toolbox = 0.73
 smmap2 = 2.0.1
 transaction = 1.7.0

software/nginx-push-stream/software.cfg

@@ -45,5 +45,5 @@ output = ${buildout:directory}/instance-nginx.cfg.in
 mode = 0644
 
 [versions]
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 slapos.toolbox = 0.73

software/powerdns/software.cfg

@@ -65,4 +65,4 @@ mode = 0644
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2

software/re6stnet/software.cfg

@@ -110,7 +110,7 @@ gitdb = 0.6.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
 pycurl = 7.43.0
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 slapos.toolbox = 0.73
 smmap = 0.9.0
 

software/seleniumrunner/software.cfg

@@ -47,4 +47,4 @@ mode = 0644
 
 [versions]
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2

software/slapos-in-partition/software.cfg

@@ -11,4 +11,4 @@ extends = common.cfg
 Pygments = 1.6
 collective.recipe.environment = 0.2.0
 collective.recipe.template = 1.10
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2

software/slapos-master/buildout.hash.cfg

@@ -15,7 +15,7 @@
 # not need these here).
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 251c1fcec1817fe70ec0cf7da69ccf3a
+md5sum = 4a77ee4a6367fee27552f8bfe9d87aab
 
 [template-balancer]
 filename = instance-balancer.cfg.in

software/slapos-master/instance-erp5.cfg.in

@@ -143,7 +143,7 @@ return =
   zope-address-list
   hosts-dict
   monitor-base-url
-{% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_run_my_doc slapos_configurator' -%}
+{% set bt5_default_list = 'erp5_full_text_myisam_catalog slapos_configurator' -%}
 {% if has_jupyter -%}
 {%   set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%}
 {% endif -%}

software/slapos-testing/software.cfg

@@ -97,4 +97,4 @@ mode = 640
 Pygments = 2.1.3
 collective.recipe.template = 1.10
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2

software/slaprunner/common.cfg

@@ -152,6 +152,7 @@ eggs =
   gunicorn==19.7.1
   futures
   ${slapos-cookbook:eggs}
+  slapos.core # listed explicitly for scripts generation
 
 [extra-eggs]
 recipe = zc.recipe.egg

software/slaprunner/software.cfg

@@ -15,7 +15,7 @@ gitdb = 0.6.4
 gunicorn = 19.7.1
 prettytable = 0.7.2
 pycurl = 7.43.0
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 slapos.toolbox = 0.73
 smmap = 0.9.0
 

software/test-slave-instance-deployment/software.cfg

@@ -59,7 +59,7 @@ eggs = collective.recipe.template
 collective.recipe.template = 1.11
 plone.recipe.command = 1.1
 slapos.recipe.build = 0.28
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 
 # Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.
 [buildout]

software/unstable/condor/software.cfg

@@ -51,7 +51,6 @@ MarkupSafe = 0.18
 Werkzeug = 0.8.3
 buildout-versions = 1.7
 hexagonit.recipe.cmmi = 2.0
-inotifyx = 0.2.0-1
 lxml = 3.2.1
 meld3 = 0.6.10
 netaddr = 0.7.10

software/unstable/zimbra-kvm/software.cfg

@@ -44,7 +44,6 @@ hexagonit.recipe.download = 1.6nxd002
 
 # Required by:
 # slapos.cookbook==0.73.1
-inotifyx = 0.2.0
 
 # Required by:
 # slapos.cookbook==0.73.1

software/varnish/software.cfg

@@ -62,4 +62,4 @@ md5sum = 0ea12a4ad2d2e3d406476e35b8d3e3fb
 mode = 640
 
 [versions]
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2

software/wendelin-scalability/test-common.cfg

+[buildout]
+extends =
+  ../wendelin/software.cfg
+  ../../component/mariadb/mariarocks.cfg
+
+[local-bt5-repository]
+list += ${slapos.cookbook-repository:location}/software/wendelin-scalability
+
+[patch-template]
+recipe = slapos.recipe.build
+location = ${buildout:directory}/${:_buildout_section_name_}.cfg.in
+script =
+  with open(self.options['location'], 'w') as dst, \
+       open(self.options['base']) as src:
+    src = src.read()
+    i = src.index('[buildout]')
+    dst.write(src[:i] + self.options['extra'] + '\n' + src[i:])

software/wendelin-scalability/test-fluentd-1pb.cfg

+# ERP5: id-store-interval must be big enough to avoid conflicts (e.g. 1000)
+
 [buildout]
 extends = test-fluentd-common.cfg
 
@@ -21,7 +23,6 @@ recipe = slapos.recipe.build:download
 [template-zope-patched]
 <= template-fluentd
 base = ${template-zope-base:target}
-method = unpack
 tags =
   {%- if slapparameter_dict['family'] == 'fluentd' %}
   {%- for i, zope in enumerate(publish_list) %}
@@ -32,7 +33,7 @@ tags =
 
 [feeder]
 feeder =
-  #
+  # Same algorithm as ERP5Site_simulateFluentdIngestion
     import os, struct
     from random import lognormvariate
     pack = struct.Struct('!d').pack

software/wendelin-scalability/test-fluentd-common.cfg

 [buildout]
 extends =
-  ../../component/gnupg/buildout.cfg
   ../fluentd/software.cfg
-  ../wendelin/software.cfg
-  ../../component/mariadb/mariarocks.cfg
-
-[local-bt5-repository]
-list += ${slapos.cookbook-repository:location}/software/wendelin-scalability
-
-[patch-template]
-recipe = slapos.recipe.build
-location = ${buildout:directory}/${:_buildout_section_name_}.cfg.in
-script =
-  with open(self.options['location'], 'w') as dst, \
-       open(self.options['base']) as src:
-    src = src.read()
-    i = src.index('[buildout]')
-    dst.write(src[:i] + self.options['extra'] + '\n' + src[i:])
+  test-common.cfg
 
 [template-erp5]
 recipe =
@@ -32,6 +17,7 @@ base = ${template-erp5-base:target}
 
 [template-fluentd]
 <= patch-template
+method = unpack
 extra =
   {%- set tags = {} %}
   ${:tags}

software/wendelin-scalability/test-many-small-transactions.cfg

+# ERP5: id-store-interval must be big enough to avoid conflicts (e.g. 1000)
+
+# XXX: Because supervisord can't handle too many processes
+#      ("too many open files to spawn" errors), these SR
+#      should instantiate the source processes differently.
+
+[buildout]
+extends = test-fluentd-common.cfg
+
+[template-erp5-patched]
+extra =
+  [request-balancer]
+  config-inituser-login = {{ dumps(inituser_login) }}
+  config-inituser-password = $${publish-early:inituser-password}
+
+[template-balancer]
+recipe =
+target = ${template-balancer-patched:location}
+
+[template-balancer-base]
+<= template-balancer
+recipe = slapos.recipe.build:download
+
+[template-balancer-patched]
+<= template-fluentd
+base = ${template-balancer-base:target}
+tags =
+  {%- set port, backend_list = haproxy_dict['fluentd'] %}
+  {%- for i in range(100*len(backend_list)) %}
+  {%-   do tags.__setitem__('wendelin_tag_' ~ i, ipv4 ~ ':' ~ port) %}
+  {%- endfor %}
+
+[feeder]
+feeder =
+  #
+    import collections, random, struct
+    pack = struct.Struct('!d').pack
+    data = collections.deque(
+      ''.join(chr(int(random.gauss(0, .68)) % 256) for _ in xrange(2500))
+      # With a period greater than 64kiB (zlib dictionary size),
+      # we avoid extra compression due to repetition.
+      for _ in xrange(30))
+    interval = 60 # XXX: same as fluentd flush internal
+    time.sleep(interval * random.random())
+    while True:
+      emit('', pack(time.time()) + data[0])
+      data.rotate()
+      time.sleep(interval)

software/wendelin-scalability/test-much-deduplication-high-compressibility.cfg

+# NEO: data deduplication must be enabled
+
 [buildout]
-extends = test-fluentd-common.cfg
+extends =
+  ../../component/gnupg/buildout.cfg
+  test-fluentd-common.cfg
 
 [template-erp5-patched]
 extra =

software/wendelin-scalability/test-neo.cfg

+# The 'start_ingest' command causes a zope to fill NEO as fast as possible,
+# as if fluentd pushed data.
+#
+# Use sigma > 0 to have oids of variable size inside NEO.
+# To know average compression ratio:
+#   x=test_scalability_fluentd/ExtensionTemplateItem/portal_components/extension.erp5.ScalabilityFluentd.py
+#   $x 10 1
+#   0.434851958247
+#   2155 - 65536 (99th percentile)
+#   $x 8.787 0
+#   0.100036621094
+#   6556
+
+[buildout]
+extends = test-common.cfg
+parts += start_ingest
+
+[start_ingest]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
+mode = 0755
+template =
+  inline:#!${buildout:executable}
+  import argparse, base64, httplib, sys
+  parser = argparse.ArgumentParser()
+  _ = parser.add_argument
+  _('--site-id', default='erp5')
+  _('hostport', metavar='host[:port]', help='Zope address')
+  _('password', help="'zope' user password")
+  _('reference', help='Data Stream reference')
+  _('mu', type=float)
+  _('sigma', type=float)
+  _('chunks_per_transaction', nargs='?', type=int, help='default: 128 (8 MiB)')
+  args = parser.parse_args()
+  qs = []
+  for k in 'reference', 'mu', 'sigma', 'chunks_per_transaction':
+    v = getattr(args, k)
+    if v is not None:
+      t = type(v)
+      qs.append('%s=%s' % (k if t is str else k + ':' + t.__name__, v))
+  c = httplib.HTTPConnection(args.hostport)
+  c.putrequest('GET', '/%s/ERP5Site_simulateFluentdIngestion?%s'
+                      % (args.site_id, '&'.join(qs)))
+  c.putheader('Authorization',
+    'Basic ' + base64.b64encode('zope:'+args.password))
+  c.endheaders()

software/wendelin-scalability/test_scalability_fluentd/ExtensionTemplateItem/portal_components/extension.erp5.ScalabilityFluentd.py

+#!/usr/bin/python
+from __future__ import division, print_function
+import os, struct
+from random import lognormvariate
+
+bigfile_chunk_size = 65536
+
+def simulateFluentdIngestion(self, reference, mu, sigma,
+                             chunks_per_transaction=128):
+  from time import time
+  import transaction
+  note = (self['portal_ingestion_policies']['scalability_test_unpack'].getPath()
+          + '/ingest')
+  module = self['data_stream_module']
+  try:
+    data_stream = module[reference]
+  except KeyError:
+    data_stream = module.newContent(reference, 'Data Stream')
+    transaction.commit()
+
+  pack = struct.Struct('!d').pack
+  data = os.urandom(bigfile_chunk_size - 8)
+
+  while 1:
+    txn = transaction.begin()
+    data_stream.appendData(''.join(
+      (pack(time()) + data[:int(lognormvariate(mu, sigma))]
+      ).ljust(bigfile_chunk_size, '\0')
+      for _ in xrange(chunks_per_transaction)))
+    txn.note(note)
+    txn.commit()
+
+if __name__ == '__main__':
+  import sys
+  mu, sigma = map(float, sys.argv[1:3])
+  if sigma:
+    try:
+      n = int(sys.argv[3])
+    except IndexError:
+      n = 1000000
+  else:
+    n = 1
+  x = sorted(min(int(lognormvariate(mu, sigma)), bigfile_chunk_size - 8)
+             for _ in xrange(n))
+  print((8 * n + sum(x)) / (bigfile_chunk_size * n))
+  if n == 1:
+    print(x[0] + 8)
+  else:
+    n //= 100
+    if n:
+      print(8 + x[n], '-', 8 + x[-n-1], '(99th percentile)')
+    else:
+      print(8 + x[0], '-', 8 + x[-1])

software/wendelin-scalability/test_scalability_fluentd/ExtensionTemplateItem/portal_components/extension.erp5.ScalabilityFluentd.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Extension Component" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_recorded_property_dict</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+        <item>
+            <key> <string>default_reference</string> </key>
+            <value> <string>ScalabilityFluentd</string> </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>extension.erp5.ScalabilityFluentd</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Extension Component</string> </value>
+        </item>
+        <item>
+            <key> <string>sid</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_error_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_warning_message</string> </key>
+            <value>
+              <tuple>
+                <string>W:  8, 46: Redefining name \'mu\' from outer scope (line 35) (redefined-outer-name)</string>
+                <string>W:  8, 50: Redefining name \'sigma\' from outer scope (line 35) (redefined-outer-name)</string>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>version</string> </key>
+            <value> <string>erp5</string> </value>
+        </item>
+        <item>
+            <key> <string>workflow_history</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary/>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>component_validation_workflow</string> </key>
+                    <value>
+                      <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
+                    </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="4" aka="AAAAAAAAAAQ=">
+    <pickle>
+      <global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
+    </pickle>
+    <pickle>
+      <tuple>
+        <none/>
+        <list>
+          <dictionary>
+            <item>
+                <key> <string>action</string> </key>
+                <value> <string>validate</string> </value>
+            </item>
+            <item>
+                <key> <string>validation_state</string> </key>
+                <value> <string>validated</string> </value>
+            </item>
+          </dictionary>
+        </list>
+      </tuple>
+    </pickle>
+  </record>
+</ZopeData>

software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/ERP5Site_simulateFluentdIngestion.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_function</string> </key>
+            <value> <string>simulateFluentdIngestion</string> </value>
+        </item>
+        <item>
+            <key> <string>_module</string> </key>
+            <value> <string>ScalabilityFluentd</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>ERP5Site_simulateFluentdIngestion</string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string></string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

software/wendelin-scalability/test_scalability_fluentd/bt/template_extension_id_list

+extension.erp5.ScalabilityFluentd
\ No newline at end of file

software/wendelin-scalability/test_scalability_fluentd/bt/template_path_list

 portal_ingestion_policies/scalability_test_*
 portal_skins/custom/DataStreamModule_getTotalSize
-portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion
\ No newline at end of file
+portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion
+portal_skins/custom/ERP5Site_simulateFluentdIngestion
\ No newline at end of file

stack/caddy/Caddyfile.in

@@ -10,6 +10,8 @@ tls {{ custom_cert_dict['cert-file'] }} {{ custom_cert_dict['key-file'] }} {
 
 log {{caddy_configuration_dict['access_log']}}
 
+errors {{caddy_configuration_dict['error_log']}}
+
 root {{ directory_dict['public_html'] }}
 
 {% if parameter_dict['enable-basic-auth'] == 'true'  -%}
@@ -18,6 +20,8 @@ basicauth / {{ parameter_dict['username'] }} {{parameter_dict['password']}}
 
 }
 
+bind {{caddy_configuration_dict['ipv6']}}
+
 {%- endif %}
 
 [{{caddy_configuration_dict['ipv6']}}]:{{parameter_dict['port-ipv6']}} {
@@ -29,23 +33,31 @@ basicauth / {{ parameter_dict['username'] }} {{parameter_dict['password']}}
   }
 
   log {{caddy_configuration_dict['access_log']}}
+  
+  errors {{caddy_configuration_dict['error_log']}}
 
   root {{ directory_dict['public_html'] }}
 
   {% if parameter_dict['enable-basic-auth'] == 'true'  -%}
   basicauth / {{ parameter_dict['username'] }} {{parameter_dict['password']}}
   {%- endif %}
+  
+  bind {{caddy_configuration_dict['ipv6']}}
 
 }
 
 [{{caddy_configuration_dict['local_ip']}}]:{{parameter_dict['port-ipv4']}} {
 
   log {{caddy_configuration_dict['access_log']}}
+  
+  errors {{caddy_configuration_dict['error_log']}}
 
   root {{ directory_dict['public_html'] }}
 
   {% if parameter_dict['enable-basic-auth'] == 'true'  -%}
   basicauth / {{ parameter_dict['username'] }} {{parameter_dict['password']}}
   {%- endif %}
+  
+  bind {{caddy_configuration_dict['local_ip']}}
 
 }
\ No newline at end of file

stack/caddy/buildout.cfg

@@ -27,7 +27,7 @@ mode = 0644
 [template-caddyfile]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/Caddyfile.in
-#md5sum = bb3b314ebeb58e9875d547a053a9f268
+md5sum = 88c4c33e374ea3f61cdd36b2816d24ba
 filename = Caddyfile.in
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 mode = 0644

stack/caucase/buildout.cfg

@@ -94,7 +94,7 @@ caucase = 0.1.4
 futures = 3.1.1
 gitdb2 = 2.0.2
 gunicorn = 19.7.1
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 slapos.toolbox = 0.73
 smmap2 = 2.0.3
 

stack/cloudooo.cfg

@@ -89,7 +89,7 @@ PasteScript = 2.0.2
 WSGIUtils = 0.7
 python-magic = 0.4.6
 rdiff-backup = 1.0.5+SlapOSPatched001
-slapos.recipe.template = 4.1
+slapos.recipe.template = 4.2
 
 # Required by:
 # PasteScript==2.0

stack/erp5/buildout.cfg

@@ -736,7 +736,7 @@ uuid = 1.30
 validictory = 1.1.0
 xfw = 0.10
 xupdate-processor = 0.4
-selenium = 2.53.1
+selenium = 3.8.0
 
 # Required by:
 # Products.CMFCore==2.2.10

stack/erp5/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = c1f1083bf6c911a0e65dcb841fba327d
 
 [mariadb-slow-query-report-script]
 filename = mysql-querydigest.sh.in
-md5sum = cfe6ab8ae54a521ecb269e9d9762cbeb
+md5sum = 0c0d98a68230cd0ad36046bb25b35f4a
 
 [mariadb-start-clone-from-backup]
 filename = instance-mariadb-start-clone-from-backup.sh.in
@@ -27,7 +27,7 @@ md5sum = 1af531c51f575a1d1362f2ca2d61620d
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = 8ea5033142f450a2e90431817771cb44
+md5sum = da7c36ecb490b67360d2afda94b41bff
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
@@ -79,7 +79,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 2465af81147af322056cee9f6c7de14f
+md5sum = 02ed5d9b74c70789004d01dd2ecde7b1
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -91,7 +91,7 @@ md5sum = fd7e8c507cef1950e6c0347ce2a01021
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = f64c568f1365eb1164f12f48fede9a99
+md5sum = a71ad387eab681b9020e271cba2c7a79
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -287,7 +287,7 @@ apachedex = ${monitor-directory:private}/apachedex
 [{{ section('monitor-generate-apachedex-report') }}]
 recipe = slapos.cookbook:wrapper
 wrapper-path = ${monitor-directory:reports}/${:command}
-command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --config "${apachedex-parameters:configuration}"
+command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration "${apachedex-parameters:configuration}"
 command = apachedex_every_23_hour
 
 [apachedex-parameters]

stack/erp5/instance-erp5.cfg.in

@@ -284,7 +284,7 @@ config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
 config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
 config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
 config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
-'--erp5-base "/erp5(/|$|/\?)" --skip-user-agent Zabbix  --error-detail --js-embed --quiet')) }}
+'--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
 
 [request-frontend-base]
 {% if has_frontend -%}

stack/erp5/instance-mariadb.cfg.in

@@ -107,7 +107,7 @@ time = {{ dumps(backup_periodicity) }}
 #   can be fully restored.
 # master-data: use value "2" as we are not in a replication case
 #}
-command = "${binary-wrap-mysqldump:wrapper-path}" -u root --all-databases --single-transaction {% if incremental_backup_retention_days > -1 %}--flush-logs --master-data=2 {% endif %}| {{ parameter_dict['gzip-location'] }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ parameter_dict['coreutils-location'] }}/bin/date "+%Y%m%d%H%M%S").sql.gz"
+command = "${binary-wrap-mysqldump:wrapper-path}" -u root --all-databases --flush-privileges --single-transaction {% if incremental_backup_retention_days > -1 %}--flush-logs --master-data=2 {% endif %}| {{ parameter_dict['gzip-location'] }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ parameter_dict['coreutils-location'] }}/bin/date "+%Y%m%d%H%M%S").sql.gz"
 {# KEEP GLOB PATTERN IN SYNC with generated filenames above
 #           YYYYmmddHHMMSS -#}
 file-glob = ??????????????.sql.gz
@@ -252,7 +252,7 @@ mariadb-ssl = ${:etc}/mariadb-ssl
 var = ${buildout:directory}/var
 log = ${:var}/log
 run = ${:var}/run
-slowquery =  ${monitor-directory:private}/slowquerydex
+slowquery = ${monitor-directory:private}/slowquery_digest
 
 [{{ section('resiliency-exclude-file') }}]
 # Generate rdiff exclude file in case of resiliency

stack/erp5/mysql-querydigest.sh.in

@@ -7,19 +7,20 @@ PT_QUERY_EXEC='{{pt_query_exec}}'
 
 if [ ! -d "$OUTPUT_FOLDER" ]; then
   echo "ERROR: output_folder don't exists"
-  exit 0
+  exit 1
 fi
 
-OUTPUT_FILE=${OUTPUT_FOLDER}/slowquery_digest.txt
-
-TODAY=`date +%Y%m%d`
+dashed_today=$(date +%Y-%m-%d)
+today=$(date -d "$dashed_today" +%Y%m%d)
 
-SLOW_LOG=$SLOW_QUERY_PATH-$TODAY
+SLOW_LOG="$SLOW_QUERY_PATH-$today"
+OUTPUT_FILE="$OUTPUT_FOLDER/slowquery_digest.txt-$dashed_today"
 
 if [ ! -f "$SLOW_LOG" ]; then
   echo "ERROR: cannot read mysql slow query log file $SLOW_LOG. Exiting."
   exit 1
 fi
 
-$PT_QUERY_EXEC $SLOW_LOG >  $OUTPUT_FILE
-echo "ok"
+"$PT_QUERY_EXEC" "$SLOW_LOG" > "$OUTPUT_FILE" && \
+echo "Report generated successfully." || \
+echo "Report failed with code $?"

stack/monitor/buildout.cfg

@@ -132,4 +132,5 @@ PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 pycurl = 7.43.0
 slapos.toolbox = 0.73
+pyasn1 = 0.3.7
 

stack/slapos.cfg

@@ -53,6 +53,9 @@ allow-hosts +=
   www.dabeaz.com
   www.owlfish.com
 
+# Use an https index
+index = https://pypi.python.org/simple/
+
 # XXX: Workaround of SlapOS limitation
 # Unzippig of eggs is required, as SlapOS do not yet provide nicely working
 # development / fast switching environment for whole software
@@ -88,6 +91,7 @@ eggs =
   slapos.libnetworkcache
 
 [versions]
+setuptools = 33.1.1
 # Use SlapOS patched zc.buildout
 zc.buildout = 2.5.2+slapos011
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
@@ -108,7 +112,7 @@ collective.recipe.template = 2.0
 cryptography = 2.1.1
 decorator = 4.0.11
 idna = 2.2
-inotifyx = 0.2.2
+inotify-simple = 1.1.1
 itsdangerous = 0.24
 lock-file = 2.0
 lxml = 3.7.3
@@ -122,7 +126,6 @@ pyOpenSSL = 17.2.0
 pyparsing = 2.2.0
 pytz = 2016.10
 requests = 2.13.0
-setuptools = 33.1.1
 six = 1.10.0
 slapos.cookbook = 1.0.53
 slapos.core = 1.4.3