Merge branch 'master+ZODB4-wc2' into y/wc2-next

* master+ZODB4-wc2: (106 commits)
  component/perl-Image-ExifTool: version up 12.30
  fixup! stack/erp5: use %Ta for timing of haproxy requests
  stack/erp5: use %Ta for timing of haproxy requests
  Revert "kvm: Bang immediately in case of vnc_promise problem"
  kvm: Bang immediately in case of vnc_promise problem
  kvm: Switch mouse to tablet mode
  software/slapos-sr-testing: update doc
  software/slapos-sr-testing: run ERP5 SR test with a python3 slapos.core
  component/depot_tools: check out a fixed revision
  software/erp5/test: add python3 support
  software/galene: use python3
  stack/caucase: Version up caucase 0.9.12
  galene: version up 0.4
  kvm: Increase default VM parameters to match nowadays requirement
  hugo: create tempfiles in a dedicated temporary directory
  component/java-jdk: add missing rpaths
  component/alsa: enable slapos.recipe.cmmi's shared
  kvm: Upgrade default ISO for vm to debian bullseye
  componant/slapos/obs.cfg: update OBS build
  caddy-frontend: Pick up recent kedifa from the distribution
  ...

component/alsa/buildout.cfg

@@ -5,6 +5,7 @@ parts =
 [alsa]
 # Contains libasound
 recipe = slapos.recipe.cmmi
+shared = true
 url = ftp://ftp.alsa-project.org/pub/lib/alsa-lib-1.1.3.tar.bz2
 md5sum = eefe5992567ba00d6110a540657aaf5c
 configure-options =

component/apache/buildout.cfg

@@ -39,9 +39,9 @@ configure-options =
 [apache]
 recipe = slapos.recipe.cmmi
 shared = true
-version = 2.4.46
+version = 2.4.49
 url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
-md5sum = 7d661ea5e736dac5e2761d9f49fe8361
+md5sum = f294efbeabcf6027fccc7983a6daa55f
 configure-options = --disable-static
                     --enable-authn-alias
                     --enable-bucketeer

component/cclient/buildout.cfg

@@ -10,7 +10,9 @@ parts =
 
 [cclient]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.cac.washington.edu/imap/imap-2007f.tar.gz
+#url = ftp://ftp.cac.washington.edu/imap/imap-2007f.tar.gz
+url = https://src.fedoraproject.org/lookaside/pkgs/uw-imap/${:filename}/${:md5sum}/${:filename}
+filename = imap-2007f.tar.gz
 md5sum = 2126fd125ea26b73b20f01fcd5940369
 configure-command = true
 # cclient does not support parallel compilation
@@ -23,11 +25,11 @@ make-options =
   SSLLIB=${openssl:location}/lib
   EXTRACFLAGS=-fPIC
   EXTRALDFLAGS="-Wl,-rpath -Wl,${openssl:location}/lib"
-  CCLIENT=${buildout:parts-directory}
+  PREFIX=%(location)s
   -j1
 
 patches =
-  ${:_profile_base_location_}/imap-2007f.patch#42c77fdd5d7a976fc302b93aadb3da98
+  ${:_profile_base_location_}/imap-2007f.patch#5d1f2f95472f6be465ef7e152a011100
   ${:_profile_base_location_}/imap-2007f-openssl-1.1.patch#c726354e888f2f3b3954e334903cef80
 
 patch-options = -p1

component/cclient/imap-2007f.patch

---- old/Makefile	2011-09-22 13:19:53.000000000 +0100
-+++ new/Makefile	2011-11-09 15:02:54.038306922 +0100
-@@ -280,7 +280,11 @@
+diff -u old/Makefile new/Makefile
+--- old/Makefile
++++ new/Makefile
+@@ -280,6 +280,7 @@
  SYSTEM=unix
  TOOLS=tools
  TOUCH=touch
--
-+IMAPDIR=$(CCLIENT)/cclient
-+COMPILEDIR = $(CCLIENT)/cclient__compile__/imap-2007f
-+CP=cp -r
 +INSTALL=install
-+FOR=for
  
- # Primary build command
  
-@@ -580,7 +584,6 @@
+ # Primary build command
+@@ -580,7 +581,6 @@
  	@echo +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  	@echo
  	@echo Do you want to continue this build anyway?  Type y or n please:
@@ -21,36 +17,26 @@
  
  nounenc:
  	@echo +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-@@ -639,7 +642,7 @@
+@@ -639,7 +639,6 @@
  	@echo +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
  	@echo
  	@echo Do you want to build with IPv6 anyway?  Type y or n please:
 -	@$(SH) -c 'read x; case "$$x" in y) exit 0;; *) (make noip6;exit 1);; esac'
-+	#@$(SH) -c 'read x; case "$$x" in y) exit 0;; *) (make noip6;exit 1);; esac'
  	@echo OK, I will remember that you really want to build with IPv6.
  	@echo You will not see this message again.
  	@$(TOUCH) ip6
-@@ -731,6 +734,24 @@
+@@ -731,6 +730,15 @@
  	$(SH) -c '$(RM) an ua OSTYPE SPECIALS c-client mtest imapd ipopd mailutil mlock dmail tmail || true'
  	$(CD) tools;$(MAKE) clean
  
-+install:	
-+	$(INSTALL) -v -d $(IMAPDIR)/include
-+	$(INSTALL) -v -d $(IMAPDIR)/lib
-+	$(INSTALL) -v -m 644 $(COMPILEDIR)/c-client/*.h $(IMAPDIR)/include
-+	$(INSTALL) -v -m 644 $(COMPILEDIR)/c-client/c-client.a $(IMAPDIR)/lib
-+	$(CP) $(COMPILEDIR)/ipopd $(COMPILEDIR)/mailutil $(COMPILEDIR)/imapd \
-+	$(COMPILEDIR)/dmail $(COMPILEDIR)/mlock $(COMPILEDIR)/mtest $(COMPILEDIR)/tmail $(IMAPDIR)
-+	$(RM) $(IMAPDIR)/ipopd/*.h $(IMAPDIR)/ipopd/*.c $(IMAPDIR)/ipopd/Makefile;
-+	$(RM) $(IMAPDIR)/mailutil/*.h $(IMAPDIR)/mailutil/*.c $(IMAPDIR)/mailutil/Makefile;
-+	$(RM) $(IMAPDIR)/imapd/*.h $(IMAPDIR)/imapd/*.c $(IMAPDIR)/imapd/Makefile;
-+	$(RM) $(IMAPDIR)/tmail/*.h $(IMAPDIR)/tmail/*.c $(IMAPDIR)/tmail/Makefile;
-+	$(RM) $(IMAPDIR)/mlock/*.h $(IMAPDIR)/mlock/*.c $(IMAPDIR)/mlock/Makefile;
-+	$(RM) $(IMAPDIR)/mtest/*.h $(IMAPDIR)/mtest/*.c $(IMAPDIR)/mtest/Makefile;
-+	$(RM) $(IMAPDIR)/dmail/*.h $(IMAPDIR)/dmail/*.c $(IMAPDIR)/dmail/Makefile;
-+	$(INSTALL) -v -m 644 $(COMPILEDIR)/ip6 $(IMAPDIR)
-+	$(INSTALL) -v -m 644 $(COMPILEDIR)/OSTYPE $(IMAPDIR)
-+	$(INSTALL) -v -m 644 $(COMPILEDIR)/SPECIALS $(IMAPDIR)
++install:
++	$(INSTALL) -v -d $(PREFIX)/bin $(PREFIX)/include $(PREFIX)/lib
++	$(INSTALL) -v c-client/*.h $(PREFIX)/include
++	$(INSTALL) -v c-client/c-client.a $(PREFIX)/lib
++	for x in ipopd mailutil imapd dmail mlock mtest tmail; \
++	do for x in $$x/*; do [ ! -x $$x ] || echo $$x; done; \
++	done |xargs $(INSTALL) -v -t $(PREFIX)/bin
++	$(INSTALL) -v -t $(PREFIX) ip6 OSTYPE SPECIALS
 +
  
  # A monument to a hack of long ago and far away...

component/depot_tools/buildout.cfg

 [buildout]
+extends =
+  ../git/buildout.cfg
 parts =
   depot_tools
 
 [depot_tools]
 recipe = slapos.recipe.build:gitclone
 repository = https://chromium.googlesource.com/chromium/tools/depot_tools.git
-branch = master
+revsion = e023d4482012d89690f6a483e877eceb47c4501e
 git-executable = ${git:location}/bin/git

component/firewalld/buildout.cfg

@@ -73,7 +73,7 @@ environment =
   PATH=${pkgconfig:location}/bin:${gettext:location}/bin:${glib:location}/bin:${xz-utils:location}/bin:${flex:location}/bin:${bison:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
   CPPFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include
-  LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${libffi:location}/lib -Wl,-rpath=${libffi:location}/lib -lffi
+  LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${libffi:location}/lib -Wl,-rpath=${libffi:location}/lib -lffi -L${zlib:location}/lib/ -Wl,-rpath=${zlib:location}/lib/
   GLIB_CFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include
   GLIB_LIBS=-L${glib:location}/lib -lglib-2.0 -lintl -lgobject-2.0
   FFI_CFLAGS=-I${libffi:location}/include

component/fluentd/buildout.cfg

@@ -11,20 +11,21 @@ url = https://rubygems.org/rubygems/rubygems-2.4.8.zip
 ruby-executable = ${ruby:location}/bin/ruby
 gems =
   msgpack==1.3.3
-  cool.io==1.4.6
+  concurrent-ruby==1.1.9
+  cool.io==1.7.1
+  dig_rb==1.0.1
   http_parser.rb==0.6.0
   sigdump==0.2.4
-  serverengine==2.2.3
-  strptime==0.1.9
+  serverengine==2.2.4
+  strptime==0.2.5
   thread_safe==0.3.6
-  tzinfo==1.2.9
+  tzinfo==2.0.4
   tzinfo-data==1.2021.1
   yajl-ruby==1.4.1
-  fluentd==0.14.14
+  fluentd==1.8.1
   httpclient==2.8.3
-  json==2.5.1
-  td-client==0.8.85
-  fluent-plugin-td==0.10.29
+  td-client==1.0.8
+  fluent-plugin-td==1.1.0
 gem-options =
   --with-icu-lib=${icu:location}/lib/
   --with-icu-dir=${icu:location}/

component/fonts/buildout.cfg

@@ -10,19 +10,13 @@ parts =
   ipaex-fonts
   liberation-fonts
   ocrb-fonts
+  source-code-pro-fonts
+  jetbrains-mono-fonts
 
 
 [fonts-base]
-# XXX download and unpack, with shared parts support
-# we could make slapos.recipe.build:download-unpacked really support shared
-# parts and use it here. Current version of slapos.recipe.build ( 0.44 ) looks
-# for buildout:shared-parts , but this is not what shared parts are using.
-recipe = slapos.recipe.cmmi
+recipe = slapos.recipe.build:download-unpacked
 shared = true
-configure-command = :
-make-binary = :
-post-install = cp -ra . ${:location}
-location = @@LOCATION@@
 environment =
   PATH=${xz-utils:location}/bin:%(PATH)s
 
@@ -77,27 +71,19 @@ md5sum = 62f02985bfef43a27dbdd17641fec210
 # Microsoft's TrueType core fonts
 # non-free so not enabled by default
 [msttcore-fonts]
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 recipe = slapos.recipe.build
+shared = true
+p7z = ${p7zip:location}/bin/7z
 install =
   import os, subprocess
-  from zc.buildout.download import Download
-  d = location
-  fonts = []
-  download = lambda x, dl=Download(self.buildout['buildout']): (
-    dl("http://downloads.sf.net/corefonts/%s32.exe" % name, md5sum=md5sum)
-    for md5sum, name in (x.split() for x in x.splitlines() if x))
-  extract = lambda x, d=d, p7z="${p7zip:location}/bin/7z": any(
-    subprocess.check_call((p7z, "x", "-ssc-", path, "*.ttf"), cwd=d)
-    for path, is_temp in x)
-  try:
-    fonts += download(options['fonts'])
-    os.makedirs(d)
-    extract(fonts)
-  finally:
-    for path, is_temp in fonts:
-      if is_temp:
-        os.remove(path)
+  os.makedirs(location)
+  cmd = [options['p7z'], "x", "-ssc-", None, "*.ttf"]
+  for x in options['fonts'].splitlines():
+    md5sum, name = x.split()
+    cmd[3] = self.download(
+      "http://downloads.sf.net/corefonts/%s32.exe" % name,
+      md5sum)
+    subprocess.check_call(cmd, cwd=location)
 slapos_promise =
 slapos_update_promise = ${:slapos_promise}
 fonts =

component/geoip2/buildout.cfg

@@ -16,6 +16,7 @@ rpath = ${:library-dirs}
 
 [geolite2-country]
 recipe = slapos.recipe.build:download-unpacked
+shared = true
 url = http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz#${:md5sum}
 md5sum = dc6224c648350d90f344a0c5c3ca5474
 strip-top-level-dir = true

component/ghostscript/buildout.cfg

@@ -14,8 +14,8 @@ parts = ghostscript
 [ghostscript]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs9540/ghostscript-9.54.0.tar.gz
-md5sum = 5d571792a8eb826c9f618fb69918d9fc
+url = https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs9550/ghostscript-9.55.0.tar.xz
+md5sum = 92aa46e75c4f32eb11d9c975053d876c
 pkg_config_depends = ${libtiff:location}/lib/pkgconfig:${libjpeg:location}/lib/pkgconfig:${fontconfig:location}/lib/pkgconfig:${fontconfig:pkg_config_depends}
 # XXX --with-tessdata work arounds a slaprunner bug of having softwares installed in a path containing //
 configure-options =

component/git/buildout.cfg

@@ -18,8 +18,8 @@ parts =
 [git]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://www.kernel.org/pub/software/scm/git/git-2.25.1.tar.xz 
-md5sum = 92bf65673b4fc08b64108d807f36f4d9
+url = https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.33.0.tar.xz
+md5sum = 0990ff97af1511be0d9f0d3223dd4359
 configure-options =
   --with-curl=${curl:location}
   --with-openssl=${openssl:location}

component/java-jdk/buildout.cfg

 [buildout]
-
 extends = 
   ../patchelf/buildout.cfg
   ../zlib/buildout.cfg
-
+  ../alsa/buildout.cfg
+  ../libpng/buildout.cfg
+  ../freetype/buildout.cfg
+  ../xorg/buildout.cfg
 parts = 
   java-jdk
 
 
 [java-jdk]
-recipe = plone.recipe.command
-command = echo "Error: unsupported platform" && false
-stop-on-error = true
-location =
+recipe = slapos.recipe.build
+update =
+  from zc.buildout import UserError
+  raise UserError("unsupported platform")
 
 
-[java-jdk:linux and bits64]
+[java-jdk:linux and platform.machine() == 'x86_64']
 recipe = slapos.recipe.cmmi
 shared = true
 url = https://download.java.net/java/GA/jdk12.0.2/e482c34c86bd4bf8b56c0b35558996b9/10/GPL/openjdk-12.0.2_linux-x64_bin.tar.gz
 md5sum = f5da6f4dec81bdd2a096184ec1d69216
 configure-command = :
 make-binary = :
-pre-install =
-  mkdir -p @@LOCATION@@
-  cp -r * @@LOCATION@@
 post-install =
-  for file in @@LOCATION@@/bin/* ; do
+  mv * %(location)s
+  for file in %(location)s/bin/* %(location)s/lib/*.so ; do
     echo appending rpath to $file
-	  ${patchelf:location}/bin/patchelf --set-rpath ${:rpath} $file
+    ${patchelf:location}/bin/patchelf --set-rpath %(rpath)s $file
   done
-rpath = ${zlib:location}/lib:@@LOCATION@@/lib
-location = @@LOCATION@@
+rpath = ${zlib:location}/lib:${alsa:location}/lib:${freetype:location}/lib:${libpng:location}/lib:${libXrender:location}/lib:${libXtst:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXext:location}/lib:${libXdmcp:location}/lib:${libXi:location}/lib:${libxcb:location}/lib:@@LOCATION@@/lib:@@LOCATION@@/lib/server

component/java/buildout.cfg

@@ -9,42 +9,34 @@ parts =
 [java-re]
 <= java-re-7
 
+[java-common]
+recipe = slapos.recipe.build:download-unpacked
+url = http://javadl.sun.com/webapps/download/AutoDL?BundleId=${:bundle-id}
+
 [java-re-7]
-recipe = slapos.recipe.build
-slapos_promisee =
-  directory:bin
-  directory:lib
-  directory:man
-  directory:plugin
-  file:lib/rt.jar
-  file:bin/java
+<= java-common
 # http://java.com/en/download/manual_java7.jsp
-x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=97798 90a6b9e2a32d06c18a3f16b485f0d1ea
-x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=97800 7605134662f6c87131eca5745895fe84
-install =
-  url, md5sum = options[guessPlatform()].split()
-  extract_dir = self.extract(self.download(url, md5sum))
-  workdir = guessworkdir(extract_dir)
-  self.copyTree(workdir, location)
+
+[java-re-7:linux and platform.machine() == 'i686']
+bundle-id = 97798
+md5sum = 90a6b9e2a32d06c18a3f16b485f0d1ea
+
+[java-re-7:linux and platform.machine() == 'x86_64']
+bundle-id = 97800
+md5sum = 7605134662f6c87131eca5745895fe84
 
 [java-re-8]
-recipe = slapos.recipe.build
-slapos_promisee =
-  directory:bin
-  directory:lib
-  directory:man
-  directory:plugin
-  file:lib/rt.jar
-  file:bin/java
+<= java-common
 # https://www.java.com/en/download/manual.jsp
 # Update 161
-x86 = http://javadl.oracle.com/webapps/download/AutoDL?BundleId=230530_2f38c3b165be4555a1fa6e98c45e0808 32db95dd417fd7949922206b2a61aa19
-x86-64 = http://javadl.oracle.com/webapps/download/AutoDL?BundleId=230532_2f38c3b165be4555a1fa6e98c45e0808 4385bc121b085862be623f4a31e7e0b4
-install =
-  url, md5sum = options[guessPlatform()].split()
-  extract_dir = self.extract(self.download(url, md5sum))
-  workdir = guessworkdir(extract_dir)
-  self.copyTree(workdir, location)
+
+[java-re-8:linux and platform.machine() == 'i686']
+bundle-id = 230530_2f38c3b165be4555a1fa6e98c45e0808
+md5sum = 90a6b9e2a32d06c18a3f16b485f0d1ea
+
+[java-re-8:linux and platform.machine() == 'x86_64']
+bundle-id = 230532_2f38c3b165be4555a1fa6e98c45e0808
+md5sum = 4385bc121b085862be623f4a31e7e0b4
 
 [java-re-8-output]
 # Shared binary location to ease migration

component/java/jdk-6u27-no-user-interaction.patch

---- jdk-6u27-linux-x64.bin.orig	2011-09-27 11:02:14.000000000 +0200
-+++ jdk-6u27-linux-x64.bin	2011-09-27 10:38:01.000000000 +0200
-@@ -81,7 +81,7 @@
- trap 'rm -f $outname; exit 1' HUP INT QUIT TERM
- echo "Unpacking..."
- tail ${tail_args} +189 "$0" > $outname
--if [ -x /usr/bin/sum ]; then
-+if [ -x /usr/bin/null ]; then
-     echo "Checksumming..."
- 
-     sum=`/usr/bin/sum $outname`
-@@ -169,7 +169,7 @@
-     fi
- 
-     # Service Tag support and JDK product registration
--    register_JDK "$javahome" "${BINARY_NAME}" "$1"
-+    # register_JDK "$javahome" "${BINARY_NAME}" "$1"
- 
- else
-     if [ "$1" = "-x" ]; then

component/lua/buildout.cfg

@@ -8,12 +8,23 @@ parts =
 [lua]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.lua.org/ftp/lua-5.3.1.tar.gz
-md5sum = 797adacada8d85761c079390ff1d9961
+url = http://www.lua.org/ftp/lua-5.4.3.tar.gz
+md5sum = ef63ed2ecfb713646a7fcc583cf5f352
 configure-command = true
 make-options =
   "$(uname -sr 2>/dev/null|grep -Eq '^Linux' && echo linux || echo posix)"
-  MYCFLAGS="-I${readline:location}/include"
+  MYCFLAGS="-I${readline:location}/include -fPIC"
   MYLDFLAGS="-L${readline:location}/lib -Wl,-rpath=${readline:location}/lib"
-make-targets =
-  install INSTALL_TOP=@@LOCATION@@
+  INSTALL_TOP=@@LOCATION@@
+post-install =
+  mkdir -p %(location)s/lib/pkgconfig
+  {
+    make pc INSTALL_TOP=%(location)s
+    echo '%(pc)s'
+  } > %(location)s/lib/pkgconfig/lua.pc
+pc =
+  Name: Lua
+  Description: Lua language engine
+  Version: $${version}
+  Libs: -L$${libdir} -llua
+  Cflags: -I$${includedir}

component/m4/01-fix-ftbfs-with-glibc-2.28.patch

-From: Santiago Vila <sanvila@debian.org>
-Subject: Fix FTBFS with glibc 2.28
-Bug-Debian: https://bugs.debian.org/915152
-X-Debian-version: 1.4.18-2
-
-Based on this gnulib commit by Paul Eggert:
-
-https://lists.gnu.org/r/bug-gnulib/2018-03/msg00002.html
-
---- a/lib/fflush.c
-+++ b/lib/fflush.c
-@@ -33,7 +33,7 @@
- #undef fflush
- 
- 
--#if defined _IO_ftrylockfile || __GNU_LIBRARY__ == 1 /* GNU libc, BeOS, Haiku, Linux libc5 */
-+#if defined _IO_EOF_SEEN || __GNU_LIBRARY__ == 1 /* GNU libc, BeOS, Haiku, Linux libc5 */
- 
- /* Clear the stream's ungetc buffer, preserving the value of ftello (fp).  */
- static void
-@@ -72,7 +72,7 @@
- 
- #endif
- 
--#if ! (defined _IO_ftrylockfile || __GNU_LIBRARY__ == 1 /* GNU libc, BeOS, Haiku, Linux libc5 */)
-+#if ! (defined _IO_EOF_SEEN || __GNU_LIBRARY__ == 1 /* GNU libc, BeOS, Haiku, Linux libc5 */)
- 
- # if (defined __sferror || defined __DragonFly__ || defined __ANDROID__) && defined __SNPT
- /* FreeBSD, NetBSD, OpenBSD, DragonFly, Mac OS X, Cygwin, Android */
-@@ -148,7 +148,7 @@
-   if (stream == NULL || ! freading (stream))
-     return fflush (stream);
- 
--#if defined _IO_ftrylockfile || __GNU_LIBRARY__ == 1 /* GNU libc, BeOS, Haiku, Linux libc5 */
-+#if defined _IO_EOF_SEEN || __GNU_LIBRARY__ == 1 /* GNU libc, BeOS, Haiku, Linux libc5 */
- 
-   clear_ungetc_buffer_preserving_position (stream);
- 
---- a/lib/fpending.c
-+++ b/lib/fpending.c
-@@ -32,7 +32,7 @@
-   /* Most systems provide FILE as a struct and the necessary bitmask in
-      <stdio.h>, because they need it for implementing getc() and putc() as
-      fast macros.  */
--#if defined _IO_ftrylockfile || __GNU_LIBRARY__ == 1 /* GNU libc, BeOS, Haiku, Linux libc5 */
-+#if defined _IO_EOF_SEEN || __GNU_LIBRARY__ == 1 /* GNU libc, BeOS, Haiku, Linux libc5 */
-   return fp->_IO_write_ptr - fp->_IO_write_base;
- #elif defined __sferror || defined __DragonFly__ || defined __ANDROID__
-   /* FreeBSD, NetBSD, OpenBSD, DragonFly, Mac OS X, Cygwin, Android */
---- a/lib/fpurge.c
-+++ b/lib/fpurge.c
-@@ -62,7 +62,7 @@
-   /* Most systems provide FILE as a struct and the necessary bitmask in
-      <stdio.h>, because they need it for implementing getc() and putc() as
-      fast macros.  */
--# if defined _IO_ftrylockfile || __GNU_LIBRARY__ == 1 /* GNU libc, BeOS, Haiku, Linux libc5 */
-+# if defined _IO_EOF_SEEN || __GNU_LIBRARY__ == 1 /* GNU libc, BeOS, Haiku, Linux libc5 */
-   fp->_IO_read_end = fp->_IO_read_ptr;
-   fp->_IO_write_ptr = fp->_IO_write_base;
-   /* Avoid memory leak when there is an active ungetc buffer.  */
---- a/lib/freadahead.c
-+++ b/lib/freadahead.c
-@@ -25,7 +25,7 @@
- size_t
- freadahead (FILE *fp)
- {
--#if defined _IO_ftrylockfile || __GNU_LIBRARY__ == 1 /* GNU libc, BeOS, Haiku, Linux libc5 */
-+#if defined _IO_EOF_SEEN || __GNU_LIBRARY__ == 1 /* GNU libc, BeOS, Haiku, Linux libc5 */
-   if (fp->_IO_write_ptr > fp->_IO_write_base)
-     return 0;
-   return (fp->_IO_read_end - fp->_IO_read_ptr)
---- a/lib/freading.c
-+++ b/lib/freading.c
-@@ -31,7 +31,7 @@
-   /* Most systems provide FILE as a struct and the necessary bitmask in
-      <stdio.h>, because they need it for implementing getc() and putc() as
-      fast macros.  */
--# if defined _IO_ftrylockfile || __GNU_LIBRARY__ == 1 /* GNU libc, BeOS, Haiku, Linux libc5 */
-+# if defined _IO_EOF_SEEN || __GNU_LIBRARY__ == 1 /* GNU libc, BeOS, Haiku, Linux libc5 */
-   return ((fp->_flags & _IO_NO_WRITES) != 0
-           || ((fp->_flags & (_IO_NO_READS | _IO_CURRENTLY_PUTTING)) == 0
-               && fp->_IO_read_base != NULL));
---- a/lib/fseeko.c
-+++ b/lib/fseeko.c
-@@ -47,7 +47,7 @@
- #endif
- 
-   /* These tests are based on fpurge.c.  */
--#if defined _IO_ftrylockfile || __GNU_LIBRARY__ == 1 /* GNU libc, BeOS, Haiku, Linux libc5 */
-+#if defined _IO_EOF_SEEN || __GNU_LIBRARY__ == 1 /* GNU libc, BeOS, Haiku, Linux libc5 */
-   if (fp->_IO_read_end == fp->_IO_read_ptr
-       && fp->_IO_write_ptr == fp->_IO_write_base
-       && fp->_IO_save_base == NULL)
-@@ -123,7 +123,7 @@
-           return -1;
-         }
- 
--#if defined _IO_ftrylockfile || __GNU_LIBRARY__ == 1 /* GNU libc, BeOS, Haiku, Linux libc5 */
-+#if defined _IO_EOF_SEEN || __GNU_LIBRARY__ == 1 /* GNU libc, BeOS, Haiku, Linux libc5 */
-       fp->_flags &= ~_IO_EOF_SEEN;
-       fp->_offset = pos;
- #elif defined __sferror || defined __DragonFly__ || defined __ANDROID__
---- a/lib/stdio-impl.h
-+++ b/lib/stdio-impl.h
-@@ -18,6 +18,12 @@
-    the same implementation of stdio extension API, except that some fields
-    have different naming conventions, or their access requires some casts.  */
- 
-+/* Glibc 2.28 made _IO_IN_BACKUP private.  For now, work around this
-+   problem by defining it ourselves.  FIXME: Do not rely on glibc
-+   internals.  */
-+#if !defined _IO_IN_BACKUP && defined _IO_EOF_SEEN
-+# define _IO_IN_BACKUP 0x100
-+#endif
- 
- /* BSD stdio derived implementations.  */
- 

component/m4/buildout.cfg

@@ -11,7 +11,8 @@ shared = true
 url = http://ftp.gnu.org/gnu/m4/m4-1.4.18.tar.xz
 md5sum = 730bb15d96fffe47e148d1e09235af82
 environment =
-  PATH=${xz-utils:location}/bin:${patch:location}/bin:%(PATH)s
+  PATH=${xz-utils:location}/bin:%(PATH)s
+patch-binary = ${patch:location}/bin/patch
 patch-options = -p1
 patches =
-  ${:_profile_base_location_}/01-fix-ftbfs-with-glibc-2.28.patch#058a786425e507f911649205b61ffcac
+  https://sources.debian.org/data/main/m/m4/1.4.18-5/debian/patches/01-fix-ftbfs-with-glibc-2.28.patch#058a786425e507f911649205b61ffcac

component/nginx/buildout.cfg

 [buildout]
 extends =
   ../coreutils/buildout.cfg
-  ../git/buildout.cfg
   ../libexpat/buildout.cfg
   ../openssl/buildout.cfg
-  ../patch/buildout.cfg
   ../pcre/buildout.cfg
   ../zlib/buildout.cfg
 
@@ -12,13 +10,12 @@ parts = nginx-output
 
 [nginx-common]
 recipe = slapos.recipe.cmmi
-shared = false
+shared = true
 url = https://nginx.org/download/nginx-1.19.2.tar.gz
 md5sum = 3dc55f6451ed6f819f1c796f4e5e9617
 
 [nginx]
 <= nginx-common
-shared = true
 configure-options=
   --with-http_ssl_module
   --with-http_v2_module
@@ -32,10 +29,10 @@ configure-options=
 
 [nginx-dav-ext-module]
 recipe = slapos.recipe.build:download-unpacked
+shared = true
 url = https://github.com/arut/nginx-dav-ext-module/archive/v0.0.3.tar.gz
 strip-top-level-dir = true
 md5sum = 2cb502dbda335be4ebd5fed0b3182bae
-mode = 0644
 
 [nginx-webdav]
 <= nginx-common
@@ -61,29 +58,11 @@ command = ${coreutils-output:test} -x ${:nginx} -a -f ${:mime}
 nginx = ${nginx:location}/sbin/nginx
 mime = ${nginx:location}/conf/mime.types
 
-[hexaglobe-nginx-module]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-url = http://easicloud-p.cdn.hexaglobe.net/nginx-easicloud.tar.gz
-md5sum = 57fe2ceb09740f22b5b1023f29889e0e
-strip-top-level-dir = true
-
-[nginx-enable-sub]
-# Used by Hexaglobe for watermarking
-<= nginx
-configure-options=
-  --with-ipv6
-  --with-http_ssl_module
-  --with-ld-opt="-L ${zlib:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib  -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib"
-  --with-cc-opt="-I ${pcre:location}/include -I ${openssl:location}/include -I ${zlib:location}/include"
-  --add-module=${hexaglobe-nginx-module:location}/sub_module
-# --add-module=${hexaglobe-nginx-module:location}/nginx-upstream-fair
-
 [nginx-push-stream-module]
-recipe = slapos.recipe.build:gitclone
-repository = https://github.com/wandenberg/nginx-push-stream-module.git
-revision = 3d3a204177d3a7ab8a2858e04e792a6d11bf133f
-git-executable = ${git:location}/bin/git
+recipe = slapos.recipe.build:download-unpacked
+shared = true
+url = https://github.com/wandenberg/nginx-push-stream-module/archive/0.4.0.tar.gz
+md5sum = d9cba621b8739e13bdb5e02b9425f205
 
 [nginx-push-stream]
 <= nginx-common

component/noVNC/buildout.cfg

@@ -4,6 +4,7 @@ parts =
 
 [noVNC]
 recipe = slapos.recipe.build:download-unpacked
+shared = true
 url = https://github.com/novnc/noVNC/archive/refs/tags/v1.2.0.tar.gz
 md5sum = 290dfabc4ecdd58d62ccb8c34a922962
 strip-top-level-dir = true

component/nodejs/buildout.cfg

@@ -11,8 +11,8 @@ extends =
 parts =
   nodejs
 
-[nodejs]
-<= nodejs-8.9.4
+#[nodejs]
+#<= nodejs-X.Y.Z
 
 [nodejs-14.16.0]
 <= nodejs-base
@@ -25,33 +25,25 @@ md5sum = 7dc3666f407bf4e12a01ce1be2883d31
 openssl_location = ${openssl:location}
 version = v12.18.3
 md5sum = 28bf6a4d98b238403fa58a0805f4a979
-
-[nodejs-10.19.0]
-<= nodejs-base
-openssl_location = ${openssl:location}
-version = v10.19.0
-md5sum = 9e433c753d839d2d2a6341861501674f
-
-[nodejs-10.6.0]
-<= nodejs-base
-openssl_location = ${openssl:location}
-version = v10.6.0
-md5sum = 9df233b86244ebda1ded1f91694fbe86
+PATH = ${pkgconfig:location}/bin:${python2.7:location}/bin:%(PATH)s
 
 [nodejs-8.9.4]
 <= nodejs-base
 version = v8.9.4
 md5sum = 4ddc1daff327d7e6f63da57fdfc24f55
+PATH = ${pkgconfig:location}/bin:${python2.7:location}/bin:%(PATH)s
 
 [nodejs-8.6.0]
 <= nodejs-base
 version = v8.6.0
 md5sum = 0c95e08220667d8a18b97ecec8218ac6
+PATH = ${pkgconfig:location}/bin:${python2.7:location}/bin:%(PATH)s
 
 [nodejs-8.12.0]
 <= nodejs-base
 version = v8.12.0
 md5sum = 5690333b77964edf81945fc724f6ea85
+PATH = ${pkgconfig:location}/bin:${python2.7:location}/bin:%(PATH)s
 
 [nodejs-base]
 # Server-side Javascript.
@@ -66,85 +58,15 @@ configure-options =
   --shared-openssl
   --shared-openssl-includes=${:openssl_location}/include
   --shared-openssl-libpath=${:openssl_location}/lib
+PATH = ${pkgconfig:location}/bin:%(PATH)s
 environment =
   HOME=@@LOCATION@@
-  PATH=${pkgconfig:location}/bin:%(PATH)s
+  PATH=${:PATH}
   PKG_CONFIG_PATH=${:openssl_location}/lib/pkgconfig/
   CPPFLAGS=-I${zlib:location}/include
   LDFLAGS=-Wl,-rpath=${:openssl_location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
   LD_LIBRARY_PATH=${:openssl_location}/lib
 
-[nodejs-8.6.0-output]
-# Shared binary location to ease migration
-recipe = plone.recipe.command
-stop-on-error = true
-update-command = ${:command}
-command = ${coreutils-output:test} -x ${:node} -a -x ${:npm}
-node = ${nodejs-8.6.0:location}/bin/node
-npm = ${nodejs-8.6.0:location}/bin/npm
-
-[nodejs-5]
-# Server-side Javascript.
-recipe = slapos.recipe.cmmi
-version = v5.9.1
-url = http://nodejs.org/dist/${:version}/node-${:version}.tar.gz
-md5sum = 346c9325912271dc7614fe955c75c3a6
-configure-options =
-  --shared-openssl
-  --shared-openssl-includes=${openssl:location}/include
-  --shared-openssl-libpath=${openssl:location}/lib
-environment =
-  HOME=${buildout:parts-directory}/${:_buildout_section_name_}
-  PATH=${pkgconfig:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig/
-  CPPFLAGS=-I${zlib:location}/include
-  LDFLAGS=-Wl,-rpath=${openssl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
-
-[nodejs-4]
-# Server-side Javascript.
-recipe = slapos.recipe.cmmi
-version = v4.4.1
-url = http://nodejs.org/dist/${:version}/node-${:version}.tar.gz
-md5sum = ef756c3e773f08bccada08eb37ee699c
-configure-options =
-  --shared-openssl
-  --shared-openssl-includes=${openssl:location}/include
-  --shared-openssl-libpath=${openssl:location}/lib
-environment =
-  HOME=${buildout:parts-directory}/${:_buildout_section_name_}
-  PATH=${pkgconfig:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig/
-  CPPFLAGS=-I${zlib:location}/include
-  LDFLAGS=-Wl,-rpath=${openssl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
-
-[nodejs-0.6]
-# Server-side Javascript.
-recipe = slapos.recipe.cmmi
-url = http://nodejs.org/dist/v0.6.21/node-v0.6.21.tar.gz
-md5sum = 0da985a0bf820400af92363b9f453fe4
-configure-options =
-  --openssl-includes=${openssl:location}/include
-  --openssl-libpath=${openssl:location}/lib
-environment =
-  HOME=${buildout:parts-directory}/${:_buildout_section_name_}
-  PATH=${pkgconfig:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig/
-  CPPFLAGS=-I${zlib:location}/include
-  LDFLAGS=-Wl,-rpath=${openssl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
-
-[nodejs-0.4]
-recipe = slapos.recipe.cmmi
-url = http://nodejs.org/dist/node-v0.4.12.tar.gz
-md5sum = a6375eaa43db5356bf443e25b828ae16
-configure-options =
-  --openssl-includes=${openssl:location}/include
-  --openssl-libpath=${openssl:location}/lib
-environment =
-  PATH=${pkgconfig:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig/
-  CPPFLAGS=-I${zlib:location}/include
-  LDFLAGS=-Wl,-rpath=${openssl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
-
 [npm]
 # Node.js Package Manager
 # Deprecated. Included in node >= 0.6.3.

component/open62541/buildout.cfg

+# Implementation of OPC UA (OPC Unified Architecture). https://open62541.org/
+
+[buildout]
+parts = open62541
+extends =
+  ../cmake/buildout.cfg
+
+[open62541]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://github.com/open62541/open62541/archive/refs/tags/v1.2.2.tar.gz
+md5sum = 2883bde165bc9bc3d459ccbb47acf7f4
+configure-command = ${cmake:location}/bin/cmake
+configure-options =
+  -Bbuild
+  -DCMAKE_INSTALL_PREFIX=@@LOCATION@@
+make-options = -C build

component/openssh/buildout.cfg

@@ -16,13 +16,13 @@ parts =
 [openssh]
 recipe = slapos.recipe.cmmi
 shared = true
-md5sum = 68ba883aff6958297432e5877e9a0fe2
+md5sum = 8ce5f390958baeeab635aafd0ef41453
 location = @@LOCATION@@
-url = https://ftp.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-7.7p1.tar.gz
+url = https://ftp.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-8.8p1.tar.gz
 patch-binary = ${patch:location}/bin/patch
 patch-options = -p1
 patches =
-  ${:_profile_base_location_}/no_create_privsep_path.patch#f341dc11d73df6f43c7ae1fa47b8c003
+  ${:_profile_base_location_}/no_create_privsep_path.patch#6ab983d16c9b4caf111c737dcad6ec9b
 environment = 
   CPPFLAGS=-I${zlib:location}/include -I${openssl-1.0:location}/include
   LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl-1.0:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib

component/openssh/no_create_privsep_path.patch

@@ -14,7 +14,7 @@ diff --git a/Makefile.in b/Makefile.in
 index 04e1c8e5..9bd5d01b 100644
 --- a/Makefile.in
 +++ b/Makefile.in
-@@ -329,7 +329,6 @@ install-files:
+@@ -391,7 +391,6 @@ install-files:
  	$(MKDIR_P) $(DESTDIR)$(mandir)/$(mansubdir)5
  	$(MKDIR_P) $(DESTDIR)$(mandir)/$(mansubdir)8
  	$(MKDIR_P) $(DESTDIR)$(libexecdir)

component/perl-DBD-MySQL/buildout.cfg

 [buildout]
 extends =
-  ../mysql-tritonn-5.0/buildout.cfg
   ../mariadb/buildout.cfg
   ../patch/buildout.cfg
   ../perl-Devel-CheckLib/buildout.cfg
 
 parts =
-  perl-DBD-MySQL
+  perl-DBD-mariadb
 
 [perl-DBD-MySQL-common]
 <= perl-CPAN-package
@@ -23,12 +22,6 @@ patches =
 patch-options = -p1
 extra-configure-args=--libs "-L${zlib:location}/lib -L${openssl:location}/lib $(mysql_config --libs)"
 
-[perl-DBD-MySQL]
-<= perl-DBD-MySQL-common
-environment =
-  OTHERLDFLAGS=-Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${mysql-tritonn-5.0:location}/lib/mysql -Wl,-rpath=${openssl:location}/lib
-  PATH=${mysql-tritonn-5.0:location}/bin:${patch:location}/bin:%(PATH)s
-
 [perl-DBD-mariadb]
 <= perl-DBD-MySQL-common
 environment =

component/perl-Image-ExifTool/buildout.cfg

@@ -7,5 +7,5 @@ parts =
 [perl-Image-ExifTool]
 <= perl-CPAN-package
 module = Image/Image-ExifTool
-version = 12.00
-md5sum = 8c3f99a9a8d110ba340ee3f7063c940e
+version = 12.30
+md5sum = 1f5d66d62418c8b29eb0c0b7fd272b28

component/powerdns/buildout.cfg

@@ -4,6 +4,7 @@ extends =
   ../automake/buildout.cfg
   ../boost-lib/buildout.cfg
   ../libtool/buildout.cfg
+  ../lua/buildout.cfg
   ../make/buildout.cfg
   ../openssl/buildout.cfg
   ../pkgconfig/buildout.cfg
@@ -19,17 +20,16 @@ parts =
 
 [powerdns]
 recipe = slapos.recipe.cmmi
-url = http://downloads.powerdns.com/releases/pdns-4.2.1.tar.bz2
-md5sum = b5f3998a3bc438b905c72c0473408839
+url = https://downloads.powerdns.com/releases/pdns-4.5.1.tar.bz2
+md5sum = 5f0ba98ca59bc3d84cfd09097c8b9953
 configure-options =
   --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
   --with-boost=${boost-lib:location}
   --with-libcrypto=${openssl:location}
   --with-modules="geoip"
   --with-dynmodules=""
-  --without-lua
   --disable-lua-records
-pkg_config_depends = ${yaml-cpp:location}/lib/pkgconfig
+pkg_config_depends = ${yaml-cpp:location}/lib/pkgconfig:${lua:location}/lib/pkgconfig
 environment =
   PATH=${autoconf:location}/bin:${automake:location}/bin:${libmaxminddb:location}/bin:${libtool:location}/bin:${make:location}/bin:${pkgconfig:location}/bin:%(PATH)s
   LDFLAGS=-L${boost-lib:location}/lib -Wl,-rpath=${boost-lib:location}/lib -L${libmaxminddb:location}/lib -Wl,-rpath=${libmaxminddb:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${yaml-cpp:location}/lib -Wl,-rpath=${yaml-cpp:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
@@ -39,5 +39,3 @@ environment =
   YAML_LIBS = -lyaml-cpp
 make-options =
   LIBTOOL=libtool
-make-target =
-  install

component/pysvn-python/buildout.cfg

@@ -11,10 +11,9 @@ parts =
 recipe = zc.recipe.egg:custom
 egg =
   pysvn
-find-links = http://pysvn.barrys-emacs.org/source_kits/pysvn-1.7.10.tar.gz
+find-links = https://sourceforge.net/projects/pysvn/files/pysvn/V1.9.15/pysvn-1.9.15.tar.gz/download
 patches =
-  ${:_profile_base_location_}/pysvn-1.7.10-inc_lib_dir.patch#08371129e0d0a215fb7f7811a860a89c
-  ${:_profile_base_location_}/pysvn-issue-179.patch#bd3f9629f95f0f749c5a5e93c797ee2b
+  ${:_profile_base_location_}/pysvn-1.9.15-inc_lib_dir.patch#c16eb88a862d4676c0ea2827f51b991a
 patch-options = -p1
 patch-binary = ${patch:location}/bin/patch
 include-dirs =

component/pysvn-python/pysvn-1.7.10-inc_lib_dir.patch → component/pysvn-python/pysvn-1.9.15-inc_lib_dir.patch

-diff -ur pysvn-1.7.10.orig/setup.py pysvn-1.7.10/setup.py
---- pysvn-1.7.10.orig/setup.py	2010-12-30 13:26:51.000000000 +0100
-+++ pysvn-1.7.10/setup.py	2015-08-14 10:29:25.562686564 +0200
-@@ -19,6 +19,10 @@
+diff --git a/setup.py b/setup.py
+index e5ce838..e2b95f8 100644
+--- a/setup.py
++++ b/setup.py
+@@ -40,9 +40,16 @@ import distutils.sysconfig
+ import platform
+ import shutil
+ import sys
++import subprocess
  import os
  import os.path
- import setuptools.command.bdist_egg
++import setuptools.command.bdist_egg
+ from setuptools.command.build_ext import build_ext
 +try:
 +  from ConfigParser import ConfigParser
 +except ImportError:
 +  from configparser import ConfigParser
++
  
  pysvn_version_info = {}
  f = open( 'Builder/version.info', 'r' )
-@@ -27,13 +28,26 @@
-     pysvn_version_info[ key ] = value
+@@ -77,6 +84,13 @@ class BuildExtensions(build_ext):
+             super( BuildExtensions, self ).build_extension( ext )
  
- def run(self):
-+    cfg = ConfigParser()
-+    cfg.read('setup.cfg')
-+    kw = {}
-+    for key in ('include-dirs', 'library-dirs'):
-+        if cfg.has_option('build_ext', key):
-+            kw[key] = cfg.get('build_ext', key)
+     def _build_pysvn( self, ext ):
++        cfg = ConfigParser()
++        cfg.read('setup.cfg')
++        kw = {}
++        for key in ('include-dirs', 'library-dirs'):
++            if cfg.has_option('build_ext', key):
++                kw[key] = cfg.get('build_ext', key)
 +
-     # Generate metadata first
-     self.run_command("egg_info")
-     os.chdir('Source')
--    os.system(sys.executable + ' setup.py configure')
-+    os.system(sys.executable + ' setup.py configure --include-dirs=%(include-dirs)s --library-dirs=%(library-dirs)s' % kw)
-     os.system('make clean')
-     os.system('make')
--    os.system('make egg DISTDIR="%s"' % os.path.abspath(os.path.join('..', self.dist_dir)))
-+    os.system('rm -rf dist; mkdir -p dist/EGG-INFO')
-+    os.system('cp -rvf pysvn dist')
-+    os.system('cp -rvf ../pysvn.egg-info/* dist/EGG-INFO')
-+    os.system('find dist/pysvn -type f | sed s:dist/:: > dist/EGG-INFO/SOURCES.txt')
-+    setuptools.command.bdist_egg.make_zipfile(
-+        self.egg_output, 'dist', verbose=self.verbose,
-+        dry_run=self.dry_run, mode=self.gen_header())
-     os.chdir('..')              # Go back in parent directory
-     # Add to 'Distribution.dist_files' so that the "upload" command works
-     getattr( self.distribution, 'dist_files', [] ).append(
-diff -ur pysvn-1.7.10.orig/Source/setup_configure.py pysvn-1.7.10/Source/setup_configure.py
---- pysvn-1.7.10.orig/Source/setup_configure.py	2014-11-09 11:55:47.000000000 +0100
-+++ pysvn-1.7.10/Source/setup_configure.py	2015-08-17 10:41:05.781767086 +0200
-@@ -54,6 +54,8 @@
- class Options:
+         dest_dir = os.path.join( os.path.abspath( self.build_lib ), 'pysvn' )
+ 
+         # Generate metadata first
+@@ -91,9 +105,13 @@ class BuildExtensions(build_ext):
+         # Invoke the build system. This will generate the __init__.py and
+         # .so that we'll package.
+         os.chdir( 'Source' )
+-        os.system( sys.executable + ' setup.py configure' )
+-        os.system( 'make clean' )
+-        os.system( 'make' )
++        subprocess.check_call(sys.executable + ' setup.py configure --include-dirs=%(include-dirs)s --library-dirs=%(library-dirs)s' % kw, shell=True)
++        subprocess.check_call('make clean', shell=True)
++        subprocess.check_call('make', shell=True)
++        subprocess.check_call('rm -rf dist; mkdir -p dist/EGG-INFO', shell=True)
++        subprocess.check_call('cp -rvf pysvn dist', shell=True)
++        subprocess.check_call('cp -rvf ../pysvn.egg-info/* dist/EGG-INFO', shell=True)
++        subprocess.check_call('find dist/pysvn -type f | sed s:dist/:: > dist/EGG-INFO/SOURCES.txt', shell=True)
+ 
+         # Copy the built files to the destination pysvn/ directory.
+         self.mkpath( dest_dir )
+diff --git a/Source/setup_configure.py b/Source/setup_configure.py
+index 2bf8ca1..7037f8f 100644
+--- a/Source/setup_configure.py
++++ b/Source/setup_configure.py
+@@ -55,6 +55,8 @@ class Options:
      all_options_info = {
          '--arch':               (2, '<arch>'),
+         '--distro-dir':         (2, '<dir>'),
 +        '--library-dirs':       (1, '<dir>:<dir>:...'),
 +        '--include-dirs':       (1, '<dir>:<dir>:...'),
          '--apr-inc-dir':        (1, '<dir>'),
          '--apu-inc-dir':        (1, '<dir>'),
          '--apr-lib-dir':        (1, '<dir>'),
-@@ -511,12 +513,16 @@
+@@ -542,12 +544,16 @@ class Compiler:
          raise last_exception
  
      def find_svn_bin( self ):
@@ -70,28 +80,28 @@ diff -ur pysvn-1.7.10.orig/Source/setup_configure.py pysvn-1.7.10/Source/setup_c
 +            return ''
  
      def find_svn_lib( self ):
-         folder = self.find_dir(
-@@ -589,8 +595,8 @@
+         last_exception = None
+@@ -633,8 +639,8 @@ class Compiler:
          # override the base_dir_list from the command line kw
          svn_root_dir = None
  
 -        if self.options.hasOption( kw ):
 -            base_dir_list = [self.options.getOption( kw )]
 +        if kw and self.options.hasOption( kw ):
-+            base_dir_list = self.options.getOption( kw ) + base_dir_list
++            base_dir_list = [self.options.getOption( kw )] + base_dir_list
+ 
+         debug( '__find_dir base_dir_list=%r' % (base_dir_list,) )
  
-         elif( self.options.hasOption( '--svn-root-dir' )
-         and svn_root_suffix is not None ):
-@@ -604,7 +610,7 @@
+@@ -644,7 +650,7 @@ class Compiler:
              if self.verbose:
                  print( 'Info: Checking for %s in %s' % (name, full_check_file) )
              if os.path.exists( full_check_file ):
 -                return os.path.abspath( dirname )
 +                return dirname
  
-         raise SetupError( 'cannot find %s %s - use %s' % (name, check_file, kw) )
+         raise SetupError( 'Cannot find %s %s - use %s' % (name, check_file, kw) )
  
-@@ -629,6 +635,17 @@
+@@ -669,6 +675,17 @@ class Compiler:
      def getSvnVersion( self ):
          return self.__svn_version_tuple
  
@@ -109,33 +119,42 @@ diff -ur pysvn-1.7.10.orig/Source/setup_configure.py pysvn-1.7.10/Source/setup_c
  
  class Win32CompilerMSVC90(Compiler):
      def __init__( self, setup ):
-@@ -1093,7 +1110,7 @@
-                         '%(PYCXX)s/Src',
-                         '/usr/share/python%s/CXX' % distutils.sysconfig.get_python_version() # typical Linux
+@@ -1185,6 +1202,7 @@ class UnixCompilerGCC(CompilerGCC):
+         CompilerGCC.__init__( self, setup )
+ 
+         self._find_paths_pycxx_dir = [
++                        '../Import/pycxx-7.1.5',
+                         distutils.sysconfig.get_python_inc(), # typical Linux
+                         '/usr/include'
+                         ]
+@@ -1194,7 +1212,7 @@ class UnixCompilerGCC(CompilerGCC):
+                         '/usr/src/CXX'
                          ]
+ 
 -        self._find_paths_svn_inc = [
 +        self._find_paths_svn_inc = self._include_dirs() + [
                          '/usr/include/subversion-1',            # typical Linux
                          '/usr/local/include/subversion-1',      # typical *BSD
                          '/usr/pkg/include/subversion-1',        # netbsd
-@@ -1103,14 +1120,14 @@
-                         '/usr/local/bin',                        # typical *BSD
-                         '/usr/pkg/bin',                                # netbsd
+@@ -1204,7 +1222,7 @@ class UnixCompilerGCC(CompilerGCC):
+                         '/usr/local/bin',                       # typical *BSD
+                         '/usr/pkg/bin',                         # netbsd
                          ]
 -        self._find_paths_svn_lib = [
 +        self._find_paths_svn_lib = self._library_dirs() + [
                          '/usr/lib64',                           # typical 64bit Linux
                          '/usr/lib',                             # typical Linux
                          '/usr/local/lib64',                     # typical 64bit Linux
-                         '/usr/local/lib',                       # typical *BSD
+@@ -1212,7 +1230,7 @@ class UnixCompilerGCC(CompilerGCC):
                          '/usr/pkg/lib',                         # netbsd
+                         '/usr/lib/x86_64-linux-gnu',            # debian/unbuntu
                          ]
 -        self._find_paths_apr_inc = [
 +        self._find_paths_apr_inc = self._include_dirs() + [
                          '/usr/include/apr-1',                   # typical Linux
+                         '/usr/include/apr-1.0',                 # typical Linux
                          '/usr/local/apr/include/apr-1',         # Mac OS X www.metissian.com
-                         '/usr/pkg/include/apr-1',               # netbsd
-@@ -1120,7 +1137,7 @@
+@@ -1223,7 +1241,7 @@ class UnixCompilerGCC(CompilerGCC):
                          '/usr/local/include/apache2',           # alternate *BSD
                          ]
          self._find_paths_apr_util_inc = self._find_paths_apr_inc
@@ -144,7 +163,7 @@ diff -ur pysvn-1.7.10.orig/Source/setup_configure.py pysvn-1.7.10/Source/setup_c
                          '/usr/lib64',                           # typical 64bit Linux
                          '/usr/lib',                             # typical Linux
                          '/usr/local/lib64',                     # typical 64bit Linux
-@@ -1186,6 +1203,8 @@
+@@ -1310,6 +1328,8 @@ class LinuxCompilerGCC(UnixCompilerGCC):
          if not self.setup.options.hasOption( '--norpath' ):
              py_ld_libs.extend( [
                      '-Wl,--rpath',

component/pysvn-python/pysvn-issue-179.patch

-http://pysvn.tigris.org/ds/viewMessage.do?dsForumId=1334&dsMessageId=3094617
-
-diff -ur pysvn-1.7.10.orig/Source/pysvn_svnenv.cpp pysvn-1.7.10/Source/pysvn_svnenv.cpp
---- pysvn-1.7.10.orig/Source/pysvn_svnenv.cpp	2010-11-20 13:02:37.000000000 +0100
-+++ pysvn-1.7.10/Source/pysvn_svnenv.cpp	2015-08-17 17:39:50.491209720 +0200
-@@ -292,8 +292,8 @@
-     if( accept_permanently )
-     {
-         new_cred->may_save = 1;
--        new_cred->accepted_failures = accepted_failures;
-     }
-+    new_cred->accepted_failures = accepted_failures;
- 
-     *cred = new_cred;
- 

component/python-2.7/buildout.cfg

@@ -33,17 +33,14 @@ package_version = 2.7.18
 package_version_suffix =
 md5sum = fd6cc8ec0a78c44036f825e739f36e5a
 
-# This is actually the default setting for prefix, but we can't use it in
-# other settings in this part if we don't set it explicitly here.
-prefix = @@LOCATION@@
 version = 2.7
-executable = ${:prefix}/bin/python${:version}
+executable = @@LOCATION@@/bin/python${:version}
 
 patch-options = -p1
 patches =
   ${:_profile_base_location_}/fix_compiler_module_issue_20613.patch#94443a77f903e9de880a029967fa6aa7
   ${:_profile_base_location_}/pytracemalloc_pep445.patch#9f3145817afa2b7fad801fde8447e396
-  ${:_profile_base_location_}/disabled_module_list.patch#71ad30d32bcdbc50c19cf48675b1246e
+  ${:_profile_base_location_}/disabled_module_list.patch#e038a8016475574c810cbaaf0e42f4ac
   ${:_profile_base_location_}/asyncore_poll_insteadof_select.patch#ab6991c0ee6e25aeb8951e71f280a2f1
 url =
   http://www.python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz
@@ -61,11 +58,11 @@ make-targets = make profile-opt && make install
 # recent RedHat would otherwise mangle them, either failing ('python' is
 # ambiguous and 'python2' is not supported anymore) or replacing with
 # something that's really wrong (/usr/bin/python...).
-post-install = cd '${:prefix}' &&
+post-install = cd '%(prefix)s' &&
   find -executable -type f -print0 |
   xargs -0 grep -I -m 1 '' |
   sed -n 's,:#! */usr/bin/env \+python2\?$,,p' |
-  xargs -d '\n' sed -i '1s,.*,#!${:executable},'
+  xargs -d '\n' sed -i '1s,.*,#!%(executable)s,'
 
 # the entry "-Wl,-rpath=${file:location}/lib" below is needed by python-magic,
 # which would otherwise load the system libmagic.so with ctypes

component/python-2.7/disabled_module_list.patch

@@ -5,7 +5,7 @@
  
  # This global variable is used to hold the list of modules to be disabled.
 -disabled_module_list = []
-+disabled_module_list = ['_bsddb', '_tkinter', 'bsddb185']
++disabled_module_list = ['_bsddb', '_tkinter', 'bsddb185', 'nis']
  
  def add_dir_to_list(dirlist, dir):
      """Add the directory 'dir' to the list 'dirlist' (at the front) if

component/python3/buildout.cfg

@@ -42,6 +42,7 @@ configure-options =
   --with-openssl=${openssl:location}
 # Profiled build:
   --enable-optimizations
+pre-build = printf '*disabled*\nnis\n' >> Modules/Setup.local
 pre-install = mkdir profile-opt
 
 # the entry "-Wl,-rpath=${file:location}/lib" below is needed by python-magic,

component/qemu-kvm/buildout.cfg

@@ -63,9 +63,10 @@ environment +=
 
 [debian-netinst-base]
 recipe = slapos.recipe.build:download
-filename = ${:_buildout_section_name_}
-url = https://cdimage.debian.org/cdimage/${:release}/${:arch}/iso-cd/debian-${:version}-${:arch}-netinst.iso
-release = archive/${:version}
+shared = true
+filename = debian-${:version}-${:arch}-netinst.iso
+url = https://cdimage.debian.org/cdimage/archive/${:archive}/${:arch}/iso-cd/${:filename}
+archive = ${:version}
 
 [debian-amd64-netinst-base]
 <= debian-netinst-base
@@ -86,16 +87,23 @@ md5sum = df0ce86d0b1d81e232ad08eef58754ed
 version = 9.13.0
 md5sum = 6097fdb9cbab47c96471274b9044e983
 
-[debian-amd64-netinst.iso]
-# Download the installer of Debian 10 (Buster)
-# XXX: This is not the latest version because
-#      Debian does not provide a stable URL for it.
+[debian-amd64-buster-netinst.iso]
+<= debian-amd64-netinst-base
+version = 10.11.0
+md5sum = 9d7b9cc850464d60ac174787c53e8f3f
+
+[debian-amd64-bullseye-netinst.iso]
 <= debian-amd64-netinst-base
-version = 10.10.0
-md5sum = c7d0e562e589e853b5d00563b4311720
+version = 11.1.0
+md5sum = b710c178eb434d79ce40ce703d30a5f0
+alternate-url = https://cdimage.debian.org/cdimage/release/current/${:arch}/iso-cd/${:filename}
+
+[debian-amd64-netinst.iso]
+<= debian-amd64-bullseye-netinst.iso
 
 [debian-amd64-testing-netinst.iso]
 <= debian-amd64-netinst-base
-release = bullseye_di_rc3
+alternate-url = https://cdimage.debian.org/cdimage/${archive}/${:arch}/iso-cd/${:filename}
+archive = bullseye_di_rc3
 version = bullseye-DI-rc3
 md5sum = 405917de7062c58357a3673c9901f0c4

component/rpm2cpio/buildout.cfg

@@ -5,5 +5,6 @@ parts =
 [rpm2cpio]
 # https://github.com/ruda/rpm2cpio
 recipe = slapos.recipe.build:download
+shared = true
 url = ${:_profile_base_location_}/${:_buildout_section_name_}
 md5sum = aa3a5920a1d8963592be0c2666ee05e2

component/serf/buildout.cfg

@@ -10,12 +10,22 @@ extends =
 [serf]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://archive.apache.org/dist/serf/serf-1.2.1.tar.bz2
-md5sum = 4f8e76c9c6567aee1d66aba49f76a58b
-configure-options =
-  --with-apr=${apr:location}
-  --with-apr-util=${apr-util:location}
-  --with-openssl=${openssl-1.0:location}
-environment =
-  CFLAGS=-I${zlib:location}/include -I${libuuid:location}/include
-  LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${libuuid:location}/lib -Wl,-rpath=${libuuid:location}/lib
+url = https://archive.apache.org/dist/serf/serf-1.3.9.tar.bz2
+md5sum = 370a6340ff20366ab088012cd13f2b57
+scons-command =
+  python ${scons:location}/scons.py \
+  APR="${apr:location}" \
+  APU="${apr-util:location}" \
+  OPENSSL="${openssl:location}" \
+  PREFIX=@@LOCATION@@ \
+  CFLAGS="-I${zlib:location}/include -I${libuuid:location}/include" \
+  LINKFLAGS="-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${libuuid:location}/lib -Wl,-rpath=${libuuid:location}/lib"
+configure-command = :
+make-binary = ${:scons-command}
+
+[scons]
+recipe = slapos.recipe.build:download-unpacked
+shared = true
+url = https://prdownloads.sourceforge.net/scons/scons-local-2.3.0.tar.gz
+md5sum = d262e6c6da620488d2226fabc8aabf09
+strip-top-level-dir = false

component/slapos/obs.cfg

@@ -76,8 +76,6 @@ post-install =
 [dbus-glib]
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 environment +=
-  CPPFLAGS=-I${libexpat:location}/include -I${dbus:location}/include/dbus-1.0 -I${dbus:location}/lib/dbus-1.0/include
-  LDFLAGS=-L${libexpat:location}/lib -L${gettext:location}/lib -Wl,-rpath=${zlib:location}/lib -L${dbus:location}/lib -Wl,-rpath=${dbus:location}/lib
   LD_LIBRARY_PATH=${dbus:location}/lib
   DBUS_CFLAGS=-I${dbus:location}/include/dbus-1.0 -I${dbus:location}/lib/dbus-1.0/include
 
@@ -86,9 +84,7 @@ environment +=
 pre-configure =
   sed -i 's#/opt/slapos/parts/dbus/lib/libdbus-1.la#${dbus:location}/lib/libdbus-1.la#' ${dbus-glib:location}/lib/libdbus-glib-1.la
 environment +=
-  PATH=${pkgconfig:location}/bin:${python:location}/bin:%(PATH)s
   LD_LIBRARY_PATH=${dbus:location}/lib
-  LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${dbus:location}/lib -Wl,-rpath=${dbus:location}/lib
   PYTHON=${python:location}/bin/python${python:version}
   PYTHON_INCLUDES=-I${python:location}/include/python${python:version}
   PYTHON_LIBS=-L${python:location}/lib -lpython${python:version} -lpthread -ldl -lutil -lm
@@ -114,34 +110,21 @@ make-options =
 
 [gobject-introspection]
 pre-configure =
-  ln -s ${python:location}/bin/python${python:version} ${python:location}/bin/python2.
-  sed -i 's#!/opt/slapos/parts/python2.7/bin/python2.7#!${python2.7:location}/bin/python2.7#' ${python2.7:location}/bin/python-config
-  libtoolize -c -f
-  aclocal -I${pkgconfig:location}/share/aclocal -I${gettext:location}/share/aclocal -I${libtool:location}/share/aclocal -I${glib:location}/share/aclocal
-  ./autogen.sh
+  sed -i 's#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#!${python:location}/bin/python${python:version}#' ${python:location}/bin/python${python:version}-config
 configure-options +=
   --enable-shared
 environment +=
-  PATH=${autoconf:location}/bin:${automake:location}/bin:${pkgconfig:location}/bin:${libtool:location}/bin:${intltool:location}/bin:${gettext:location}/bin:${glib:location}/bin:${flex:location}/bin:${bison-go:location}/bin:%(PATH)s
-  GIR_DIR=${buildout:parts-directory}/${:_buildout_section_name_}/share/gir-1.0
-  CPPFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include -I${python:location}/include/python${python:version}
-  LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${libffi:location}/lib -Wl,-rpath=${libffi:location}/lib -lffi -L${python:location}/lib
-  ACLOCAL_PATH=${pkgconfig:location}/share/aclocal:${gettext:location}/share/aclocal:${libtool:location}/share/aclocal:${glib:location}/share/aclocal:${intltool:location}/share/aclocal
-  M4=${m4:location}/bin/m4
   PERL5LIB=${perl:location}/lib/5.28.1/
 
 post-install = 
-  sed -i 's#!${python:location}/bin/python${python:version}#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#' ${python:location}/bin/python-config
+  sed -i 's#!${python:location}/bin/python${python:version}#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#' ${python:location}/bin/python${python:version}-config
   rm -rf ${bison-go:location}
 
 [pygobject3]
 pre-configure +=
-  sed -i 's#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#!${python:location}/bin/python${python:version}#' ${python:location}/bin/python-config
-environment +=
-  CPPFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include -I${gettext:location}/include -I${libffi:location}/include -I${python:location}/include/python${python:version}
-  LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${python:location}/lib
+  sed -i 's#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#!${python:location}/bin/python${python:version}#' ${python:location}/bin/python${python:version}-config
 post-install =
-  sed -i 's#!${python:location}/bin/python${python:version}#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#' ${python:location}/bin/python-config
+  sed -i 's#!${python:location}/bin/python${python:version}#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#' ${python:location}/bin/python${python:version}-config
 
 [ncurses]
 configure-options =
@@ -189,3 +172,9 @@ configure-options +=
   --prefix=${buildout:rootdir}/parts/${:_buildout_section_name_}
 environment +=
   DESTDIR=${buildout:destdir}
+
+[swig]
+configure-options +=
+  --prefix=${buildout:rootdir}/parts/${:_buildout_section_name_}
+environment +=
+  DESTDIR=${buildout:destdir}

component/subversion/buildout.cfg

@@ -15,26 +15,25 @@ extends =
   ../sqlite3/buildout.cfg
   ../zlib/buildout.cfg
   ../swig/buildout.cfg
+  ../lz4/buildout.cfg
 parts =
   subversion
 
 [subversion]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://apache.mirrors.tds.net/subversion/subversion-1.8.14.tar.bz2
-md5sum = fe476ba26d6835eba4393780ea907361
-# Patch available thanks to gentoo developpers
-# https://sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/dev-vcs/subversion/files/subversion-fix-parallel-build-support-for-perl-bindings.patch
-patches =
-  ${:_profile_base_location_}/subversion-fix-parallel-build-support-for-perl-bindings.patch#fd69f4c932b4882ed98c59eb102be64a
+url = https://downloads.apache.org/subversion/subversion-1.14.1.tar.bz2
+md5sum = 2eccc2c7451397e01a13682600af9563
 configure-options =
   --disable-static
   --with-apr=${apr:location}
   --with-apr-util=${apr-util:location}
   --with-serf=yes
   --without-apxs
+  --with-utf8proc=internal
   --with-zlib=${zlib:location}
   --with-sqlite=${sqlite3:location}
+  --with-lz4=${lz4:location}
   --without-berkeley-db
   --without-sasl
   --without-apr_memcache
@@ -53,9 +52,9 @@ configure-options =
 make-targets = install -j1
 environment =
   PATH=${patch:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${apache:location}/lib/pkgconfig:${sqlite3:location}/lib/pkgconfig:${openssl-1.0:location}/lib/pkgconfig:${serf:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${apache:location}/lib/pkgconfig:${sqlite3:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig:${serf:location}/lib/pkgconfig
   CPPFLAGS=-I${libexpat:location}/include -I${libuuid:location}/include
-  LDFLAGS=-L${libexpat:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${sqlite3:location}/lib -Wl,-rpath=${apache:location}/lib -L${libuuid:location}/lib -Wl,-rpath=${libuuid:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib
+  LDFLAGS=-L${libexpat:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${sqlite3:location}/lib -Wl,-rpath=${apache:location}/lib -L${libuuid:location}/lib -Wl,-rpath=${libuuid:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${serf:location}/lib -Wl,-rpath=${lz4:location}/lib
 
 [subversion-1.9]
 recipe = hexagonit.recipe.cmmi

component/subversion/subversion-fix-parallel-build-support-for-perl-bindings.patch

---- Makefile.in	2011-07-16 13:50:53.000000000 +0200
-+++ Makefile.in.new	2012-03-11 12:13:57.000000000 +0100
-@@ -732,7 +732,7 @@
- extraclean-swig-headers: clean-swig-headers
- 	$(EXTRACLEAN_SWIG_HEADERS)
- 
--$(SWIG_PL_DIR)/native/Makefile.PL: $(SWIG_SRC_DIR)/perl/native/Makefile.PL.in
-+$(SWIG_PL_DIR)/native/Makefile.PL: $(SWIG_SRC_DIR)/perl/native/Makefile.PL.in libsvn_swig_perl
- 	./config.status subversion/bindings/swig/perl/native/Makefile.PL
- 
- $(SWIG_PL_DIR)/native/Makefile: $(SWIG_PL_DIR)/native/Makefile.PL

component/tomcat/buildout.cfg

@@ -11,7 +11,7 @@ parts =
 
 [tomcat7]
 recipe = slapos.recipe.build:download-unpacked
-ignore-existing = true
+shared = true
 strip-top-level-dir = true
 url = https://archive.apache.org/dist/tomcat/tomcat-7/v${:version}/bin/apache-tomcat-${:version}.tar.gz
 version = 7.0.100
@@ -19,7 +19,7 @@ md5sum = 79be4ba5a6e770730a4be3d5cb3c7862
 
 [tomcat9]
 recipe = slapos.recipe.build:download-unpacked
-ignore-existing = true
+shared = true
 strip-top-level-dir = true
 url = https://archive.apache.org/dist/tomcat/tomcat-9/v${:version}/bin/apache-tomcat-${:version}.tar.gz
 version = 9.0.12

component/vm-img/debian.cfg

@@ -23,7 +23,7 @@ wait-ssh = 100
 [vm-debian]
 recipe = slapos.recipe.build:vm.install-debian
 environment = vm-install-environment
-dists = debian-stable
+dists = debian-buster
 size = 2Gi
 preseed.partman/early_command =
   set -e /usr/lib/apt-setup/generators/99 AllowUnauthenticated; if [ "`debconf-get mirror/http/hostname`" = archive.debian.org ]; then echo "echo 'APT::Get::$2 \"true\";' >/target/etc/apt/apt.conf.d/01$2" >$1$2; chmod +x $1$2; fi
@@ -75,6 +75,8 @@ x86_64.iso = debian-amd64-stretch-netinst.iso
 
 [debian-buster]
 <= debian-stable
+x86_64.iso = debian-amd64-buster-netinst.iso
 
 [debian-bullseye]
-<= debian-testing
+<= debian-stable
+x86_64.iso = debian-amd64-bullseye-netinst.iso

component/yarn/buildout.cfg

@@ -35,6 +35,7 @@ bin-yarn = ${:rendered}
 
 [yarn-download]
 recipe = slapos.recipe.build:download-unpacked
+shared = true
 url = https://github.com/yarnpkg/yarn/releases/download/v${:version}/yarn-v${:version}.tar.gz
 
 [yarn-download-1.22.10]

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.197'
+version = '1.0.213'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read()
 
@@ -110,7 +110,6 @@ setup(name=name,
           'generic.kumofs = slapos.recipe.generic_kumofs:Recipe',
           'generic.memcached = slapos.recipe.generic_memcached:Recipe',
           'generic.mysql.wrap_update_mysql = slapos.recipe.generic_mysql:WrapUpdateMySQL',
-          'generic.varnish = slapos.recipe.generic_varnish:Recipe',
           'gitinit = slapos.recipe.gitinit:Recipe',
           'haproxy = slapos.recipe.haproxy:Recipe',
           'ipv4toipv6 = slapos.recipe.6tunnel:FourToSix',
@@ -162,12 +161,10 @@ setup(name=name,
           'slapconfiguration.serialised = slapos.recipe.slapconfiguration:Serialised',
           'slapconfiguration.jsondump = slapos.recipe.slapconfiguration:JsonDump',
           'slapcontainer = slapos.recipe.container:Recipe',
-          'softwaretype = slapos.recipe.softwaretype:Recipe', # BBB
           'sphinx= slapos.recipe.sphinx:Recipe',
           'squid = slapos.recipe.squid:Recipe',
           'sshkeys_authority = slapos.recipe.sshkeys_authority:Recipe',
           'sshkeys_authority.request = slapos.recipe.sshkeys_authority:Request',
-          'stunnel = slapos.recipe.stunnel:Recipe',
           'switch-softwaretype = slapos.recipe.switch_softwaretype:Recipe',
           'symbolic.link = slapos.recipe.symbolic_link:Recipe',
           'tidstorage = slapos.recipe.tidstorage:Recipe',

slapos/recipe/README.generic_varnish.rst

-generic_varnish
-===============
-
-This recipe creates a varnish instance dedicated for ERP5 with a web checker[1]
-set up.
-
-How to Use generic_varnish ?
-============================
-
-On slap console, you can instantiate varnish like this::
-  
-  instance = request(
-    software_type='varnish',
-    partition_parameter_kw={
-       'backend-url':'https://[your_backend_address]:your_backend_port',
-       'web-checker-frontend-url':'http://www.example.com',
-       'web-checker-mail-address':'web-checker-result@example.com',
-       'web-checker-smtp-host':'mail.example.com',
-    }
-  )
-
-backend-url is the backend url that varnish will cache.
-
-web-checker-frontend-url is the entry-point-url that web checker will check
-the HTTP headers of all the pages in the web site.
-
-web-checker-mail-address is the email address where web checker will send
-the HTTP Cache cheking result.
-
-web-checker-smtp-host is the smtp server to be used to send the web checker
-result.
-
-[Note]
-When web-checker-* parameters are not given, web_checker will be disabled.
-
-References
-==========
-
-[1] web_checker (it is a part of erp5.util)
-http://pypi.python.org/pypi/erp5.util
-web_checker: Web site HTTP Cache header checking tool
-

slapos/recipe/generic_varnish/__init__.py

-##############################################################################
-#
-# Copyright (c) 2012 Vifib SARL and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-import os
-import re
-from slapos.recipe.librecipe import GenericSlapRecipe
-
-class Recipe(GenericSlapRecipe):
-  """
-    Instantiate varnish daemon
-
-    TODO:
-      - use varnish3.x and replace .vcl for it
-  """
-  def _install(self):
-    ip = self.options['ip']
-    backend_url = self.options.get('backend-url',
-      # BBB: Peeking in partition parameters directly. Eew.
-      self.parameter_dict.get('backend-url',
-        self.parameter_dict.get('tidstorage-url') # BBB
-      )
-    )
-    backend_server, backend_port = self._getBackendServer(backend_url)
-    path_list = []
-    if backend_url.startswith('https://'):
-      config = dict(
-        stunnel_binary=self.options['stunnel-binary'],
-        stunnel_server=ip,
-        stunnel_port=int(self.options['stunnel-port']),
-        stunnel_pid_file=self.options['stunnel-pid-file'],
-        stunnel_conf_file=self.options['stunnel-conf-file'],
-        shell_path=self.options['shell-path'],
-        backend_server=backend_server.replace('[', '').replace(']', ''),
-        backend_port=backend_port,
-      )
-      path_list.append(self.createExecutable(self.options['stunnel-wrapper'],
-        self.substituteTemplate(self.getTemplateFilename('stunnel.in'),
-          config)))
-      path_list.append(self.createFile(self.options['stunnel-conf-file'],
-        self.substituteTemplate(self.getTemplateFilename('stunnel.conf.in'),
-          config)))
-      backend_server = ip
-      backend_port = int(self.options['stunnel-port'])
-    varnishd_manager_port = int(self.options['manager-port'])
-    varnishd_server_port = int(self.options['server-port'])
-    config = dict(
-      varnishd_binary=self.options['varnishd-binary'],
-      varnish_ip=ip,
-      varnishlog_binary=self.options['varnishlog-binary'],
-      varnishd_manager_port=varnishd_manager_port,
-      varnishd_server_port=varnishd_server_port,
-      varnishd_pid_file=self.options['pid-file'],
-      varnish_instance_name=self.options['varnish-instance-name'],
-      varnish_data=self.options['varnish-data'],
-      gcc_location=self.options['gcc-location'],
-      shell_path=self.options['shell-path'],
-      vcl_file=self.options['vcl-file'],
-      backend_port=backend_port,
-      backend_server=backend_server,
-    )
-
-    path_list.append(self.createExecutable(self.options['varnishd-wrapper'],
-      self.substituteTemplate(self.getTemplateFilename('varnishd.in'),
-        config)))
-    path_list.append(self.createExecutable(self.options['varnishlog-wrapper'],
-      self.substituteTemplate(self.getTemplateFilename('varnishlog.in'),
-        config)))
-    path_list.append(self.createFile(self.options['vcl-file'],
-      self.substituteTemplate(self.getTemplateFilename('default.vcl.in'),
-        config)))
-    return path_list
-
-  def _getBackendServer(self, url):
-    r = re.compile('\/\/(\[.+\]|[\d.]+):(\d*)')
-    result = r.search(url)
-    ip = result.groups()[0]
-    port = result.groups()[1]
-    return (ip, port)

slapos/recipe/generic_varnish/template/default.vcl.in

-#This is a basic VCL configuration file for varnish.  See the vcl(7)
-#man page for details on VCL syntax and semantics.
-#
-#Default backend definition.  Set this to point to your content
-#server.
-#
-backend default {
-    .host = "%(backend_server)s";
-    .port = "%(backend_port)s";
-    .probe = {
-        .timeout = 30s;
-        .interval = 5s;
-        .window = 4;
-        .threshold = 3;
-        .request =
-            "OPTIONS /erp5/getId HTTP/1.1"
-            "Host: %(backend_server)s:%(backend_port)s"
-            "Accept-Encoding: identity"
-            "Connection: close"
-            "User-Agent: Varnish";
-    }
-}
-
-# Called at the beginning of a request, after the complete request has been received and parsed
-sub vcl_recv {
-    # Pass any requests that Varnish does not understand straight to the backend.
-    if (req.request != "GET" &&
-        req.request != "HEAD" &&
-        req.request != "PUT" &&
-        req.request != "POST" &&
-        req.request != "TRACE" &&
-        req.request != "OPTIONS" &&
-        req.request != "PURGE" &&
-        req.request != "DELETE") {
-        /* Non-RFC2616 or CONNECT which is weird. */
-        return (pipe);
-    }
-    # Pass anything other than GET and HEAD directly.
-    if (req.request != "GET" && req.request != "HEAD") {
-        return (pass);
-    }
-    if (req.http.Authorization) {
-        /* Not cacheable by default */
-        return (pass);
-    }
-
-    # No need to have cookies for static resources
-    if (req.url ~ "\.(css|gif|ico|jpg|js|png)$") {
-        unset req.http.Cookie;
-    }
-    # Remove bogus cookies
-    if (req.http.Cookie) {
-        set req.http.Cookie = regsuball(req.http.Cookie, "(^|; ) *__utm.=[^;]+;? *", "\1");
-        set req.http.Cookie = regsuball(req.http.Cookie, "(^|; ) *__ac_name=\x22\x22;? *", "\1");
-        set req.http.Cookie = regsuball(req.http.Cookie, "(^|; ) *__ac=\x22Og.3D.3D\x22;? *", "\1");
-    }
-    if (req.http.Cookie == "") {
-        unset req.http.Cookie;
-    }
-    if (req.http.Cookie && req.http.Cookie ~ "(^|; ) *__ac=") {
-        return (pass);
-    }
-
-    # We do not care about Accept-Encoding, because we don't use varnish as the front most HTTP server.
-    unset req.http.Accept-Encoding;
-
-    if (req.backend.healthy) {
-        set req.grace = 1h;
-    } else {
-        set req.grace = 1w;
-    }
-    return (lookup);
-}
-
-# Run after a pass in vcl_recv OR after a lookup that returned a hitpass
-sub vcl_pass {
-    # unset If-Modified-Since to avoid reusing anonymous's browser cache
-    # after login.
-    unset req.http.If-Modified-Since;
-    return (pass);
-}
-
-# Creates the varnish cache key by the url
-sub vcl_hash {
-    # We use url only for hash.
-    hash_data(req.url);
-    return (hash);
-}
-
-# Called after a cache lookup if the requested document was found in the cache
-sub vcl_hit {
-    return (deliver);
-}
-
-# Called after a cache lookup if the requested document was not found in the cache
-sub vcl_miss {
-    return (fetch);
-}
-
-# Called after a document has been successfully retrieved from the backend
-sub vcl_fetch {
-    # Unset Expires that is always overridden by Cache-Control.
-    unset beresp.http.Expires;
-
-    # Unset Pragma that is obsolete.
-    unset beresp.http.Pragma;
-
-    # We only cache 200 (OK) and 304 (Not Modified) responses.
-    if (beresp.status != 200 && beresp.status != 304) {
-        set beresp.ttl = 0s;
-    }
-
-    # If max-age is 0 or not set, we want no browser cache.
-    if (beresp.ttl <= 0s) {
-        set beresp.http.Cache-Control = "no-store";
-        # Mark as hit_for_pass for the next 2 minutes.
-        set beresp.ttl = 120s;
-        return (hit_for_pass);
-    }
-
-    # We don't care haproxy's cookie.
-    if (beresp.http.Set-Cookie && beresp.http.Set-Cookie !~ "^SERVERID=[^;]+; path=/$") {
-        return (hit_for_pass);
-    } else {
-        unset beresp.http.Set-Cookie;
-    }
-
-    # We set long enough grace for cachable objects.
-    set beresp.grace = 1w;
-
-    return (deliver);
-}
-
-# Called before a cached object is delivered to the client
-sub vcl_deliver {
-    if (obj.hits > 0) {
-        set resp.http.X-Cache = obj.hits;
-    } else {
-        set resp.http.X-Cache = "MISS";
-    }
-    return (deliver);
-}

slapos/recipe/generic_varnish/template/stunnel.conf.in

-foreground = yes
-pid = %(stunnel_pid_file)s
-debug = 4
-
-[remote]
-accept = %(stunnel_server)s:%(stunnel_port)s
-client = yes
-connect = %(backend_server)s:%(backend_port)s
-sni = %(stunnel_server)s

slapos/recipe/generic_varnish/template/stunnel.in

-#!%(shell_path)s
-DAEMON_OPTS="%(stunnel_conf_file)s"
-exec %(stunnel_binary)s ${DAEMON_OPTS} 2>&1

slapos/recipe/generic_varnish/template/varnishd.in

-#!%(shell_path)s
-
-DAEMON_OPTS="-F \
-             -a %(varnish_ip)s:%(varnishd_server_port)s \
-             -T %(varnish_ip)s:%(varnishd_manager_port)s \
-             -t 0 \
-             -p nuke_limit=500 \
-             -n %(varnish_instance_name)s \
-             -f %(vcl_file)s \
-             -s file,%(varnish_data)s/varnish_storage.bin,1G"
-
-PIDFILE=%(varnishd_pid_file)s
-# exporting PATH here so that we will pass the PATH variable to the subprocess
-export PATH="%(gcc_location)s:$PATH"
-exec %(varnishd_binary)s -P ${PIDFILE} ${DAEMON_OPTS} 2>&1

slapos/recipe/generic_varnish/template/varnishlog.in

-#!%(shell_path)s
-
-DAEMON_OPTS="-a -n %(varnish_instance_name)s"
-
-exec %(varnishlog_binary)s ${DAEMON_OPTS} "$@"

slapos/recipe/generic_varnish/template/varnishlogd.in

-#!%(shell_path)s
-
-DAEMON_OPTS="-F \
-             -a %(varnish_ip)s:%(varnishd_server_port)s \
-             -T %(varnish_ip)s:%(varnishd_manager_port)s \
-             -n %(varnish_instance_name)s \
-             -f %(vcl_file)s \
-             -s file,%(varnish_data)s/varnish_storage.bin,1G"
-
-PIDFILE=%(varnishd_pid_file)s
-# exporting PATH here so that it will pass the PATH variable to the subprocess
-export PATH
-
-# If unset, or set to "0" or "no", exit
-if [ -z "${VARNISHLOG_ENABLED}" ] || \
-   [ "${VARNISHLOG_ENABLED}" = "0" ] || \
-   [ "${VARNISHLOG_ENABLED}" = "no" ]; then
-  exit 0;
-fi
-
-exec %(varnishlog_binary)s ${DAEMON_OPTS} 2>&1

slapos/recipe/promise_plugin.py

@@ -24,15 +24,17 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+import ast
 import re
 import logging, os
 import zc.buildout.easy_install
 from pprint import pformat
 from slapos.recipe.librecipe import GenericBaseRecipe
+from zc.buildout import UserError
 
 script_template = '''# This script is auto generated by slapgrid, do not edit!
 import sys
-sys.path[0:0] = %(path)s
+sys.path[0:0] = %(paths)s
 
 extra_config_dict = %(config)s
 
@@ -54,6 +56,24 @@ for module in list(sys.modules):
 %(content)s
 '''
 
+execfile_template = """\
+def _(path):
+  global _
+  del _
+  with open(path) as f:
+    return compile(f.read(), path, 'exec')
+exec(_(%r))"""
+
+
+def isPythonName(name):
+  try:
+    x, = ast.parse(name).body
+    if type(x) is ast.Expr:
+      x = x.value
+      return type(x) is ast.Name and x.id == name
+  except Exception:
+    pass
+
 class Recipe(GenericBaseRecipe):
 
   _WORKING_SET_CACHE_NAME = "slapos.cookbook_pplugin_ws_cache"
@@ -64,6 +84,35 @@ class Recipe(GenericBaseRecipe):
     options['develop-eggs-directory'] = buildout_section['develop-eggs-directory']
     super(Recipe, self).__init__(buildout, name, options)
 
+    filepath = options.get('file')
+    module = options.get('module')
+    klass = options.get('class')
+
+    if klass == 'RunPromise':
+      klass = None
+    elif klass and not isPythonName(klass):
+      raise UserError("%r is not a valid class name" % klass)
+
+    if bool(module) == bool(filepath):
+      raise UserError("Either 'module' or 'file' is required but not both")
+
+    if module:
+      if not all(map(isPythonName, module.split('.'))):
+        raise UserError("%r is not a valid module name" % module)
+      if klass:
+        content = 'from %s import %s as RunPromise' % (module, klass)
+      else:
+        content = 'from %s import RunPromise' % module
+    else:
+      content = execfile_template % filepath
+      if klass:
+        content += '\n\nRunPromise = ' + klass
+    self.content = content
+
+    self.config_dict = {key[7:]: options[key]
+      for key in options
+      if key.startswith('config-')}
+
   def _get_cache_storage(self):
     """Return a mapping where to store generated working sets.
     from https://github.com/buildout/buildout/blob/master/zc.recipe.egg_/src/zc/recipe/egg/egg.py#L170
@@ -111,26 +160,10 @@ class Recipe(GenericBaseRecipe):
       else:
         working_set = set()
 
-    regex = r"^[\w_\-\.\s]+$"
-    import_path = self.options.get('import', '').strip()
-    if import_path:
-      if not re.search(regex, import_path):
-        raise ValueError("Import path %r is not a valid" % import_path)
-      content_string = "from %s import RunPromise" % import_path
-    else:
-      # old parameter for compatibility
-      content_string = self.options['content'].strip()
-      if not re.search(regex, content_string):
-        raise ValueError("Promise content %r is not valid" % content_string)
-
-    config_dict = {key[7:]: self.options[key]
-      for key in self.options
-      if key.startswith('config-')}
-
     return self.createFile(self.options['output'], script_template % {
-      'path': pformat([dist.location for dist in working_set], indent=2),
-      'content': content_string,
-      'config': pformat(config_dict, indent=2),
+      'paths': pformat(tuple(dist.location for dist in working_set), indent=2),
+      'config': pformat(self.config_dict, indent=2),
+      'content': self.content,
       }, int(self.options.get('mode', '0644'), 8)),
 
   update = install

slapos/recipe/readline.py

@@ -43,13 +43,14 @@ class Recipe(object):
 
   def __init__(self, buildout, name, options):
     storage_path = options['storage-path']
+    when_absent = options.get('when-absent', None)
     try:
       with open(storage_path) as f:
         readline = f.readline()
-    except IOError, e:
+    except IOError as e:
       if e.errno != errno.ENOENT:
         raise
-      readline = None
+      readline = when_absent
 
     self.readline = readline
     options['readline'] = readline

slapos/recipe/softwaretype.py

-##############################################################################
-#
-# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-
-import os
-import sys
-import copy
-from six.moves.configparser import ConfigParser
-import json
-import subprocess
-import slapos.slap
-import netaddr
-import logging
-import errno
-import re
-
-import zc.buildout
-import six
-
-class SlapConfigParser(ConfigParser, object):
-  """ 
-      This class overrite ConfigParser.write method to fix parse problem when
-      configuration like: 
-      foo += bar is included in buildout file. softwaretype recipe will generate
-      buildout file with foo + = bar because ConfigParser doesn't reconize += 
-      delimiter and read key as "foo +", value as "bar".
-      Then ConfigParser.write method generate
-      
-      [section]
-      foo + = bar
-      ...
-      
-      This is invalid with buildout version 2.
-  """
-
-  def write(self, fp):
-    """Write an .ini-format representation of the configuration state."""
-    regex = re.compile(r'^(.*)\s+([+-]{1})$')
-    if self._defaults:
-      fp.write("[%s]\n" % DEFAULTSECT)
-      for (key, value) in self._defaults.items():
-        op = ""
-        result = regex.match(key)
-        if result is not None:
-          key, op = result.groups()
-
-        line = "%s %s= %s\n" % (key, op, str(value).replace('\n', '\n\t'))
-        fp.write(line)
-      fp.write("\n")
-    for section in self._sections:
-      fp.write("[%s]\n" % section)
-      for (key, value) in self._sections[section].items():
-        if key == "__name__":
-          continue
-        if (value is not None) or (self._optcre == self.OPTCRE):
-          op = ""
-          result = regex.match(key)
-          if result is not None:
-            key, op = result.groups()
-          key = "%s %s= %s" % (key, op, str(value).replace('\n', '\n\t'))
-
-        fp.write("%s\n" % key)
-      fp.write("\n")
-
-class Recipe:
-
-  def __init__(self, buildout, name, options):
-    self.buildout = buildout
-    self.options = options
-    self.name = name
-    self.logger = logging.getLogger(self.name)
-
-  def _getIpAddress(self, test_method):
-    """Internal helper method to fetch ip address"""
-    if not 'ip_list' in self.parameter_dict:
-      raise AttributeError
-    for name, ip in self.parameter_dict['ip_list']:
-      if test_method(ip):
-        return ip
-    raise AttributeError
-
-  def _getTapIpAddressList(self, test_method):
-    """Internal helper method to fetch full ip address assigned for tap"""
-    if not 'full_ip_list' in self.parameter_dict:
-      return ()
-    for item in self.parameter_dict['full_ip_list']:
-      if len(item) == 5:
-        tap, ip, gw, mask, net = item
-        if tap.startswith('route_') and test_method(ip) and \
-                          test_method(gw) and test_method(mask):
-          return (ip, gw, mask, net)
-    return ()
-
-  def getLocalIPv4Address(self):
-    """Returns local IPv4 address available on partition"""
-    # XXX: Lack checking for locality of address
-    return self._getIpAddress(netaddr.valid_ipv4)
-
-  def getGlobalIPv6Address(self):
-    """Returns global IPv6 address available on partition"""
-    # XXX: Lack checking for globality of address
-    return self._getIpAddress(netaddr.valid_ipv6)
-  
-  def getLocalTapIPv4AddressList(self):
-    """Returns global IPv6 address available for tap interface"""
-    # XXX: Lack checking for locality of address
-    return self._getTapIpAddressList(netaddr.valid_ipv4)
-
-  def getNetworkInterface(self):
-    """Returns the network interface available on partition"""
-    if not 'ip_list' in self.parameter_dict:
-      raise AttributeError
-    for name, ip in self.parameter_dict['ip_list']:
-      if name:
-        return name
-    raise AttributeError("Not network interface found")
-  
-  def mkdir_p(self, path, mode=0o700):
-    """
-    Creates a directory and its parents, if needed.
-    NB: If the directory already exists, it does not change its permission.
-    """
-
-    try:
-        os.makedirs(path, mode)
-    except OSError as exc:
-        if exc.errno == errno.EEXIST and os.path.isdir(path):
-            pass
-        else:
-            raise
-
-  def install(self):
-    slap = slapos.slap.slap()
-    slap_connection = self.buildout['slap_connection']
-    computer_id = slap_connection['computer_id']
-    computer_partition_id = slap_connection['partition_id']
-    server_url = slap_connection['server_url']
-    key_file = slap_connection.get('key_file')
-    cert_file = slap_connection.get('cert_file')
-    instance_root = self.buildout['buildout']['directory']
-    storage_configuration_dict = self.buildout.get('storage-configuration')
-    network_dict = self.buildout.get('network-information')
-    storage_home = ''
-    global_ipv4_network = ''
-    if storage_configuration_dict:
-      storage_home = storage_configuration_dict.get('storage-home')
-    if network_dict:
-      global_ipv4_network = network_dict.get('global-ipv4-network')
-    slap.initializeConnection(server_url, key_file, cert_file)
-    self.computer_partition = slap.registerComputerPartition(
-      computer_id,
-      computer_partition_id)
-    self.parameter_dict = self.computer_partition.getInstanceParameterDict()
-    software_type = self.parameter_dict['slap_software_type']
-
-    # Raise if request software_type does not exist ...
-    if software_type not in self.options:
-      # ... Except for backward compatibility. Then use "default".
-      if software_type in ['RootSoftwareInstance']:
-        software_type = 'default'
-      else:
-        raise zc.buildout.UserError("This software type (%s) isn't mapped." % \
-            software_type)
-
-    instance_file_path = self.options[software_type]
-
-    if not os.path.exists(instance_file_path):
-      raise zc.buildout.UserError("The specified buildout config file %r does "
-                                  "not exist." % instance_file_path)
-
-    if six.PY3:
-      buildout = SlapConfigParser(strict=False)
-    else:
-      buildout = SlapConfigParser()
-    with open(instance_file_path) as instance_path:
-      buildout.readfp(instance_path)
-
-    buildout.set('buildout', 'installed', '.installed-%s.cfg' % self.name)
-
-    if not buildout.has_section('slap-parameter'):
-      buildout.add_section('slap-parameter')
-    for parameter, value in self.parameter_dict.items():
-      # All parameters evaluating to False are... False, and shouldn't
-      # convey any information.
-      # Here, all those parameters are simply ignored.
-      if value:
-        if isinstance(value, str):
-          buildout.set('slap-parameter', parameter, value)
-        else:
-          buildout.set('slap-parameter', parameter, json.dumps(value))
-
-    buildout.add_section('slap-network-information')
-    buildout.set('slap-network-information', 'local-ipv4',
-                 self.getLocalIPv4Address())
-    buildout.set('slap-network-information', 'global-ipv6',
-                 self.getGlobalIPv6Address())
-    buildout.set('slap-network-information', 'network-interface',
-                 self.getNetworkInterface())
-    tap_ip_list = self.getLocalTapIPv4AddressList()
-    tap_ipv4 = tap_gateway = tap_netmask = tap_network = ''
-    if tap_ip_list:
-      tap_ipv4, tap_gateway, tap_netmask, tap_network= tap_ip_list
-    buildout.set('slap-network-information', 'tap-ipv4', tap_ipv4)
-    buildout.set('slap-network-information', 'tap-gateway', tap_gateway)
-    buildout.set('slap-network-information', 'tap-netmask', tap_netmask)
-    buildout.set('slap-network-information', 'tap-network', tap_network)
-    buildout.set('slap-network-information', 'global-ipv4-network',
-                                                          global_ipv4_network)
-
-    # Copy/paste slap_connection
-    buildout.add_section('slap-connection')
-    for key, value in six.iteritems(self.buildout['slap_connection']):
-      # XXX: Waiting for SlapBaseRecipe to use dash instead of underscores
-      buildout.set('slap-connection', key.replace('_', '-'), value)
-    # XXX: Needed for lxc. Use non standard API
-    buildout.set('slap-connection', 'requested', self.computer_partition._requested_state)
-
-    # setup storage directory
-    buildout.add_section('storage-configuration')
-    buildout.set('storage-configuration', 'storage-home', storage_home)
-    if storage_home and os.path.exists(storage_home) and \
-                                  os.path.isdir(storage_home):
-      # Create folder instance_root/DATA/ if not exist
-      data_home = os.path.join(instance_root, 'DATA')
-      self.mkdir_p(data_home)
-      for filename in os.listdir(storage_home):
-        storage_path = os.path.join(storage_home, filename, computer_partition_id)
-        if os.path.exists(storage_path) and os.path.isdir(storage_path):
-          storage_link = os.path.join(data_home, filename)
-          if os.path.lexists(storage_link):
-            if not os.path.islink(storage_link):
-              raise zc.buildout.UserError(
-                  'Target %r already exists but is not a link' % storage_link)
-              #os.unlink(storage_link)
-          else:
-            os.symlink(storage_path, storage_link)
-          buildout.set('storage-configuration', filename, storage_link)
-
-    work_directory = os.path.abspath(self.buildout['buildout'][
-      'directory'])
-    buildout_filename = os.path.join(work_directory,
-                                     'buildout-%s.cfg' % self.name)
-    with open(buildout_filename, 'w') as buildout_file:
-      buildout.write(buildout_file)
-
-    # XXX-Antoine: We gotta find a better way to do this. I tried to check
-    # out how slapgrid-cp was running buildout. But it is worse than that.
-    command_line_args = copy.copy(sys.argv) + ['-c', buildout_filename]
-
-    self.logger.info("Invoking commandline : '%s'",
-                     ' '.join(command_line_args))
-
-    subprocess.check_call(command_line_args, cwd=work_directory,
-                          env=os.environ.copy())
-    return []
-  update = install

slapos/recipe/stunnel/__init__.py

-##############################################################################
-#
-# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-import os
-import signal
-import errno
-
-from slapos.recipe.librecipe import GenericBaseRecipe
-
-def kill(pid_file, sig=signal.SIGUSR1):
-  if os.path.exists(pid_file):
-    with open(pid_file) as f:
-      pid = int(f.read().strip())
-    try:
-      os.kill(pid, sig)
-    except OSError as e:
-      if e.errno != errno.ESRCH: # No such process
-        raise e
-      os.unlink(pid_file)
-
-class Recipe(GenericBaseRecipe):
-
-  def install(self):
-    path_list = []
-
-    self.isClient = self.optionIsTrue('client', default=False)
-    if self.isClient:
-      self.logger.info("Client mode")
-    else:
-      self.logger.info("Server mode")
-
-    conf = {}
-
-    for type_ in ['remote', 'local']:
-      for data in ['host', 'port']:
-        confkey, opt = ['%s%s%s' % (type_, i, data) for i in ['_', '-']]
-        conf[confkey] = self.options[opt]
-
-    pid_file = self.options['pid-file']
-    conf.update(pid_file=pid_file)
-
-    log_file = self.options['log-file']
-    conf.update(log=log_file)
-
-    if self.isClient:
-      template = self.getTemplateFilename('client.conf.in')
-
-    else:
-      template = self.getTemplateFilename('server.conf.in')
-      key = self.options['key-file']
-      cert = self.options['cert-file']
-      conf.update(key=key, cert=cert)
-
-    conf_file = self.createFile(
-      self.options['config-file'],
-      self.substituteTemplate(template, conf))
-    path_list.append(conf_file)
-
-    wrapper = self.createWrapper(
-      self.options['wrapper'],
-      (self.options['stunnel-binary'], conf_file),
-    )
-    path_list.append(wrapper)
-
-    # Reload configuration
-    kill(pid_file, signal.SIGHUP)
-
-    if 'post-rotate-script' in self.options:
-      path_list.append(self.createPythonScript(
-        self.options['post-rotate-script'],
-        __name__ + '.kill', (pid_file,)))
-
-    return path_list

slapos/recipe/stunnel/template/client.conf.in

-foreground = yes
-output = %(log)s
-pid = %(pid_file)s
-syslog = no
-
-[service]
-client = yes
-accept = %(local_host)s:%(local_port)s
-connect = %(remote_host)s:%(remote_port)s

slapos/recipe/stunnel/template/server.conf.in

-foreground = yes
-output = %(log)s
-pid = %(pid_file)s
-syslog = no
-key = %(key)s
-cert = %(cert)s
-
-[service]
-accept = %(remote_host)s:%(remote_port)s
-connect = %(local_host)s:%(local_port)s

slapos/test/recipe/test_plugin.py

 import os, shutil, tempfile, unittest
 from slapos.recipe import promise_plugin
 from slapos.test.utils import makeRecipe
+from zc.buildout import UserError
 from pprint import pformat
 import stat, json
 import six
@@ -10,7 +11,7 @@ class TestPromisePlugin(unittest.TestCase):
   def setUp(self):
     self.tmp = tempfile.mkdtemp()
     self.output = os.path.join(self.tmp, 'output.py')
-    self.options = options = {
+    self.options = {
       'output': self.output,
       'eggs': 'slapos.cookbook'
     }
@@ -18,9 +19,16 @@ class TestPromisePlugin(unittest.TestCase):
   def tearDown(self):
     shutil.rmtree(self.tmp)
 
-  def test_parameters(self):
-    self.options['mode'] = '0644'
-    self.options['import'] = 'slapos.promise.plugin.check_site_available'
+  def makeRecipe(self):
+    return makeRecipe(
+            promise_plugin.Recipe,
+            options=self.options,
+            name='plugin')
+
+  def installRecipe(self):
+    self.makeRecipe().install()
+
+  def setConfig(self):
     self.options['config-param1'] = "YY^@12"
     self.options['config-param2'] = "23'91'"
     self.options['config-param3'] = None
@@ -28,88 +36,102 @@ class TestPromisePlugin(unittest.TestCase):
 in multi line
 123444
 """
-    recipe = makeRecipe(
-            promise_plugin.Recipe,
-            options=self.options,
-            name='plugin')
-    recipe.install()
 
+  def assertOutput(self, *expect):
     self.assertTrue(os.path.exists(self.output))
     with open(self.output, 'r') as f:
       content = f.read()
-    self.assertIn("from slapos.promise.plugin.check_site_available import RunPromise", content)
-    self.assertEqual(stat.S_IMODE(os.stat(self.output).st_mode), int('644', 8))
-
-    expected_dict = dict(
-      param1=self.options['config-param1'],
-      param2=self.options['config-param2'],
-      param3=self.options['config-param3'],
-      param4=self.options['config-param4'],
-    )
-    self.assertIn('extra_config_dict = %s' % pformat(expected_dict, indent=2), content)
-
-  def test_no_module_set(self):
-    recipe = makeRecipe(
-            promise_plugin.Recipe,
-            options=self.options,
-            name='plugin')
-    with self.assertRaises(KeyError):
-      recipe.install()
-
-  def test_default(self):
-    self.options['import'] = 'slapos.promise.plugin.check_site_available'
-    recipe = makeRecipe(
-            promise_plugin.Recipe,
-            options=self.options,
-            name='plugin')
-    recipe.install()
-
-    self.assertTrue(os.path.exists(self.output))
+    for s in expect:
+      self.assertIn(s, content)
+
+  def assertConfig(self):
+    items = self.options.items()
+    expect = {k[7:] : v  for k, v in items if k.startswith('config-')}
+    self.assertOutput("extra_config_dict = %s" % pformat(expect, indent=2))
+
+  def assertEmptyConfig(self):
+    self.assertOutput("extra_config_dict = %s" % ('{}' if six.PY3 else '{ }'))
+
+  def test_module(self):
+    self.options['module'] = 'slapos.promise.plugin.check_site_available'
+    self.installRecipe()
+    self.assertOutput("from %s import RunPromise" % self.options['module'])
+    self.assertEmptyConfig()
+
+  def test_file(self):
+    self.options['file'] = __file__
+    self.installRecipe()
+    self.assertOutput("exec(_(%r))" % self.options['file'])
+    self.assertEmptyConfig()
+
+  def test_module_and_parameters(self):
+    self.options['module'] = 'slapos.promise.plugin.check_site_available'
+    self.setConfig()
+    self.installRecipe()
+    self.assertOutput("from %s import RunPromise" % self.options['module'])
+    self.assertConfig()
+
+  def test_file_and_parameters(self):
+    self.options['file'] = __file__
+    self.setConfig()
+    self.installRecipe()
+    self.assertOutput("exec(_(%r))" % self.options['file'])
+    self.assertConfig()
+
+  def test_mode(self):
+    self.options['mode'] = '0644'
+    self.options['module'] = 'slapos.promise.plugin.check_site_available'
+    self.installRecipe()
     self.assertEqual(stat.S_IMODE(os.stat(self.output).st_mode), int('644', 8))
-    with open(self.output) as f:
-      content = f.read()
-    self.assertIn("from slapos.promise.plugin.check_site_available import RunPromise", content)
-    self.assertIn('extra_config_dict = %s' % ('{}' if six.PY3 else '{ }'), content)
 
+  def test_module_and_class(self):
+    self.options['module'] = m = 'slapos.promise.plugin.check_site_available'
+    self.options['class']  = 'MyPromise'
+    self.installRecipe()
+    self.assertOutput("from %s import MyPromise as RunPromise" % m)
+
+  def test_file_and_class(self):
+    self.options['file'] = __file__
+    self.options['class'] = 'MyPromise'
+    self.installRecipe()
+    self.assertOutput("exec(_(%r))\n\nRunPromise = MyPromise" % __file__)
+
+  def test_no_module_or_file(self):
+    with self.assertRaises(UserError) as p:
+      self.makeRecipe()
+    msg = str(p.exception)
+    self.assertEqual(msg, "Either 'module' or 'file' is required but not both")
+
+  def test_module_and_file(self):
+    self.options['module'] = 'slapos.promise.plugin.check_site_available'
+    self.options['file'] = __file__
+    self.test_no_module_or_file()
 
   def test_bad_parameters(self):
-    self.options['import'] = 'slapos.promise.plugin.check_site_available'
-    self.options['config-param1; print "toto"'] = """#xxxx"\nimport os; os.stat(f)"""
-    self.options['config-param2\n@domething'] = '"#$$*PPP\n\n p = 2*5; print "result is %s" % p'
-    recipe = makeRecipe(
-            promise_plugin.Recipe,
-            options=self.options,
-            name='plugin')
-    recipe.install()
-
-    self.assertTrue(os.path.exists(self.output))
-    with open(self.output) as f:
-      content = f.read()
-
-    expected_param1 = r"""'param1; print "toto"': '#xxxx"\nimport os; os.stat(f)',"""
-    expected_param2 = r"""'param2\n@domething': '"#$$*PPP\n\n p = 2*5; print "result is %s" % p'"""
-    self.assertIn(expected_param1, content)
-    self.assertIn(expected_param2, content)
-
-  def test_bad_module_path(self):
-    self.options['import'] = 'slapos.promise.plugin.check_site_available; print "toto"'
-    recipe = makeRecipe(
-            promise_plugin.Recipe,
-            options=self.options,
-            name='plugin')
-    with self.assertRaises(ValueError) as p:
-      recipe.install()
-
-    self.assertEqual(str(p.exception), "Import path %r is not a valid" % self.options['import'])
-
-  def test_bad_content(self):
-    self.options['content'] = 'from slapos.plugin.check_site_available import toto; print "toto"'
-    recipe = makeRecipe(
-            promise_plugin.Recipe,
-            options=self.options,
-            name='plugin')
-    with self.assertRaises(ValueError) as p:
-      recipe.install()
-
-    self.assertEqual(str(p.exception), "Promise content %r is not valid" % self.options['content'])
+    self.options['module'] = 'slapos.promise.plugin.check_site_available'
+    self.options.update((
+      ('config-param1; print "toto"', '#xxxx"\nimport os; os.stat(f)'),
+      ('config-param2\n@domething', '"#$$*PPP\np = 2*5; print "result=%s" % p')
+    ))
+    self.installRecipe()
+    self.assertOutput(
+      r"""'param1; print "toto"': '#xxxx"\nimport os; os.stat(f)',""",
+      r"""'param2\n@domething': '"#$$*PPP\np = 2*5; print "result=%s" % p'"""
+    )
 
+  def test_bad_module(self):
+    self.options['module'] = 'slapos.promise.plugin.check_site_available; print "toto"'
+    with self.assertRaises(UserError) as p:
+      self.makeRecipe()
+    self.assertEqual(str(p.exception), "%r is not a valid module name" % self.options['module'])
+
+  def test_bad_file(self):
+    self.options['file'] = 'print "toto"'
+    self.installRecipe()
+    self.assertOutput(r"""exec(_('print "toto"'))""")
+
+  def test_bad_class(self):
+    self.options['class'] = 'MyPromise; print "toto"'
+    with self.assertRaises(UserError) as p:
+      self.makeRecipe()
+    self.assertEqual(str(p.exception), "%r is not a valid class name" % self.options['class'])

software/backupserver/buildout.hash.cfg

@@ -46,7 +46,7 @@ md5sum = ae4a0043414336a521b524d9c95f1c68
 
 [template-pullrdiffbackup]
 filename = instance-pullrdiffbackup.cfg.in
-md5sum = f2e6f30a0e8228cbfb93eaaae10fe884
+md5sum = 45a4faa217ea5b83ecf271791e1632dd
 
 [template]
 filename = instance.cfg.in

software/backupserver/instance-pullrdiffbackup.cfg.in

@@ -107,8 +107,7 @@ recipe = slapos.cookbook:promise.plugin
 eggs =
   slapos.toolbox
 output = $${directory:plugin}/{{ slave_reference }}_check_backup.py
-content =
-  from slapos.promise.plugin.backupserver_check_backup import RunPromise
+module = slapos.promise.plugin.backupserver_check_backup
 config-status_dirbasename = $${variables:status_dirbasename}
 config-status_name = {{ '$${' ~ slave_reference }}-backup-script:status_name}
 config-status_fullpath = {{ '$${' ~ slave_reference }}-backup-script:status_log}
@@ -117,7 +116,6 @@ config-cron_frequency = {{ frequency }}
 config-monitor_url = $${monitor-publish:monitor-base-url}
 config-statistic_dirbasename = $${variables:statistic_dirbasename}
 config-statistic_name = {{ '$${' ~ slave_reference }}-backup-script:statistic_name}
-mode = 600
 {%   do part_list.append("%s-promise-check-backup" % slave_reference) -%}
 
 [{{ slave_reference }}-backup-script]
@@ -194,7 +192,7 @@ virtual-depends =
 
 [nginx-listen-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = nginx_listen.py
 config-host = $${nginx-configuration:ip}
 config-port = $${nginx-configuration:port}

software/caddy-frontend/buildout.hash.cfg

@@ -22,15 +22,15 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
 
 [profile-caddy-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = 51087ac7615bd7cc01e60eb23701f625
+md5sum = 0950e09ad1f03f0789308f5f7a7eb1b8
 
 [profile-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = b6fc5a004a1235ffad3af0b4cb0e661f
+md5sum = 7c2e52b76c42bed95702763c344e41dd
 
 [profile-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = 9bb51f663f69d66b5b3708bf892dd3e6
+md5sum = 313671d343ceccfca5af1baa642132c5
 
 [profile-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
@@ -46,7 +46,7 @@ md5sum = 88af61e7abbf30dc99a1a2526161128d
 
 [template-default-slave-virtualhost]
 _update_hash_filename_ = templates/default-virtualhost.conf.in
-md5sum = 13cd08d630cc51666a9f7e469fb6ea52
+md5sum = 37475d79f28c5f126bc1947fdb938fdb
 
 [template-backend-haproxy-configuration]
 _update_hash_filename_ = templates/backend-haproxy.cfg.in
@@ -102,7 +102,7 @@ md5sum = 38792c2dceae38ab411592ec36fff6a8
 
 [profile-kedifa]
 filename = instance-kedifa.cfg.in
-md5sum = eab5ae579471ca86b40bd2da3b53fefa
+md5sum = dfb4dabd1e4094de1276d171f998ef47
 
 [template-backend-haproxy-rsyslogd-conf]
 _update_hash_filename_ = templates/backend-haproxy-rsyslogd.conf.in

software/caddy-frontend/instance-apache-frontend.cfg.in

@@ -58,6 +58,7 @@ parts =
   promise-slave-introspection-https
   promise-slave-introspection-configuration
   logrotate-entry-slave-introspection
+  backend-haproxy-statistic-frontend-promise
 
 [caddyprofiledeps]
 recipe = caddyprofiledeps
@@ -538,7 +539,7 @@ context =
 
 [trafficserver-promise-listen-port]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = trafficserver-port-listening.py
 config-host = ${trafficserver-variable:local-ip}
 config-port = ${trafficserver-variable:input-port}
@@ -551,7 +552,7 @@ environment = TS_ROOT=${buildout:directory}
 
 [trafficserver-promise-cache-availability]
 <= monitor-promise-base
-module = trafficserver_cache_availability
+promise = trafficserver_cache_availability
 name = trafficserver-cache-availability.py
 config-wrapper-path = ${trafficserver-ctl:wrapper-path}
 
@@ -652,48 +653,48 @@ context =
 
 [promise-frontend-caddy-configuration]
 <= monitor-promise-base
-module = validate_frontend_configuration
+promise = validate_frontend_configuration
 name = frontend-caddy-configuration-promise.py
 config-verification-script = ${promise-helper-last-configuration-state:rendered}
 
 [promise-caddy-frontend-v4-https]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = caddy_frontend_ipv4_https.py
 config-host = {{ instance_parameter_dict['ipv4-random'] }}
 config-port = ${configuration:port}
 
 [promise-caddy-frontend-v4-http]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = caddy_frontend_ipv4_http.py
 config-host = {{ instance_parameter_dict['ipv4-random'] }}
 config-port = ${configuration:plain_http_port}
 
 [promise-caddy-frontend-v6-https]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = caddy_frontend_ipv6_https.py
 config-host = {{ instance_parameter_dict['ipv6-random'] }}
 config-port = ${configuration:port}
 
 [promise-caddy-frontend-v6-http]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = caddy_frontend_ipv6_http.py
 config-host = {{ instance_parameter_dict['ipv6-random'] }}
 config-port = ${configuration:plain_http_port}
 
 [promise-backend-haproxy-http]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = backend_haproxy_http.py
 config-host = {{ instance_parameter_dict['ipv4-random'] }}
 config-port = ${backend-haproxy-configuration:http-port}
 
 [promise-backend-haproxy-https]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = backend_haproxy_https.py
 config-host = {{ instance_parameter_dict['ipv4-random'] }}
 config-port = ${backend-haproxy-configuration:https-port}
@@ -797,7 +798,7 @@ extra-context =
 
 [promise-backend-haproxy-configuration]
 <= monitor-promise-base
-module = validate_frontend_configuration
+promise = validate_frontend_configuration
 name = backend-haproxy-configuration.py
 config-verification-script = ${promise-backend-haproxy-configuration-helper:rendered}
 
@@ -887,7 +888,7 @@ context =
 
 [monitor-verify-re6st-connectivity]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = re6st-connectivity.py
 config-url = ${configuration:re6st-verification-url}
 
@@ -921,6 +922,14 @@ config-url = https://[${slap-configuration:ipv6-random}]:{{ instance_parameter_d
 config-https-only = true
 return = domain secure_access
 
+[backend-haproxy-statistic-frontend-promise]
+<= monitor-promise-base
+promise = check_url_available
+name = backend-haproxy-statistic-frontend.py
+config-http-code = 401
+config-url =
+  ${backend-haproxy-statistic-frontend:connection-secure_access}
+
 [slave-introspection-configuration-state]
 <= jinja2-template-base
 template = {{ software_parameter_dict['template_configuration_state_script'] }}
@@ -967,7 +976,7 @@ extra-context =
 
 [promise-slave-introspection-configuration]
 <= monitor-promise-base
-module = validate_frontend_configuration
+promise = validate_frontend_configuration
 name = slave-introspection-configuration.py
 config-verification-script = ${promise-slave-introspection-configuration-helper:rendered}
 
@@ -984,7 +993,7 @@ context =
 
 [promise-slave-introspection-https]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = slave_introspection_https.py
 config-host = {{ instance_parameter_dict['ipv6-random'] }}
 config-port = ${frontend-configuration:slave-introspection-https-port}
@@ -999,7 +1008,7 @@ delaycompress =
 
 [promise-logrotate-setup]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = ${:_buildout_section_name_}.py
 config-command =
   ${logrotate:wrapper-path} -d

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -103,10 +103,12 @@ context =
 {% set authorized_slave_string_list = [] %}
 {% set authorized_slave_list = [] %}
 {% set rejected_slave_dict = {} %}
+{% set critical_rejected_slave_dict = {} %}
 {% set warning_slave_dict = {} %}
 {% set used_host_list = [] %}
 {% for slave in sorted(instance_parameter_dict['slave-instance-list']) %}
 {%   set slave_error_list = [] %}
+{%   set slave_critical_error_list = [] %}
 {%   set slave_warning_list = [] %}
 {%   set slave_server_alias_unclashed = [] %}
 {%   set slave_type = slave.get('type') %}
@@ -165,7 +167,9 @@ context =
 {%     endif %}
 {%   set custom_domain = slave.get('custom_domain') %}
 {%   if custom_domain and custom_domain in used_host_list %}
-{%      do slave_error_list.append('custom_domain %r clashes' % (custom_domain,)) %}
+{%      set message = 'custom_domain %r clashes' % (custom_domain,) %}
+{%      do slave_error_list.append(message) %}
+{%      do slave_critical_error_list.append(message) %}
 {%   else %}
 {%      do used_host_list.append(custom_domain) %}
 {%   endif %}
@@ -182,7 +186,9 @@ context =
 {%         if slave_alias in slave_server_alias_unclashed or slave_alias == custom_domain %}
 {#           optionally do something about reporting back that server-alias has been unclashed #}
 {%         elif slave_alias in used_host_list %}
-{%           do slave_error_list.append('server-alias \'%s\' clashes' % (slave_alias,)) %}
+{%           set message = 'server-alias \'%s\' clashes' % (slave_alias,) %}
+{%           do slave_error_list.append(message) %}
+{%           do slave_critical_error_list.append(message) %}
 {%         else %}
 {%           do slave_server_alias_unclashed.append(slave_alias) %}
 {%           do used_host_list.append(slave_alias) %}
@@ -251,6 +257,9 @@ context =
 {%   else %}
 {%     do rejected_slave_dict.__setitem__(slave.get('slave_reference'), sorted(slave_error_list)) %}
 {%   endif %}
+{%   if len(slave_critical_error_list) > 0 %}
+{%     do critical_rejected_slave_dict.__setitem__(slave.get('slave_reference'), sorted(slave_critical_error_list)) %}
+{%   endif %}
 {%   if len(slave_warning_list) > 0 %}
 {%     do warning_slave_dict.__setitem__(slave.get('slave_reference'), sorted(slave_warning_list)) %}
 {%   endif %}
@@ -373,7 +382,7 @@ kedifa-csr_id-certificate = ${request-kedifa:connection-csr_id-certificate}
 {%   set section_part = '${request-' + frontend %}
 [{{ part_name }}]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = check-backend-haproxy-statistic-url-{{ frontend }}.py
 config-url =
   {{ section_part }}:connection-backend-haproxy-statistic-url}
@@ -752,9 +761,9 @@ filename = rejected-slave.json
 directory = ${directory:promise-output}
 rendered = ${:directory}/${:filename}
 template = {{ software_parameter_dict['template_empty'] }}
-{% if rejected_slave_dict %}
+{% if critical_rejected_slave_dict %}
 {# sort_keys are important in order to avoid shuffling parameters on each run #}
-content = {{ dumps(json_module.dumps(rejected_slave_dict, indent=2, sort_keys=True)) }}
+content = {{ dumps(json_module.dumps(critical_rejected_slave_dict, indent=2, sort_keys=True)) }}
 {% else %}
 content =
 {% endif %}
@@ -856,15 +865,15 @@ rendered = ${directory:etc}/nginx-rejected-slave.conf
 
 [promise-rejected-slave-publish-ip-port]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = rejected-slave-publish-ip-port-listening.py
 config-host = ${rejected-slave-publish-configuration:ip}
 config-port = ${rejected-slave-publish-configuration:port}
 
 [rejected-slave-promise]
 <= monitor-promise-base
-module = check_socket_listening
-module = check_file_state
+promise = check_socket_listening
+promise = check_file_state
 name = rejected-slave.py
 config-filename = ${rejected-slave-json:rendered}
 config-state = empty

software/caddy-frontend/instance-kedifa.cfg.in

@@ -193,7 +193,7 @@ template = inline:
 
 [promise-expose-csr_id-ip-port]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = expose-csr_id-ip-port-listening.py
 config-host = ${expose-csr_id-configuration:ip}
 config-port = ${expose-csr_id-configuration:port}
@@ -250,7 +250,7 @@ extra-context =
 
 [promise-kedifa-http-reply]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = kedifa-http-reply.py
 # Kedifa replies 400 on /, so use it to be sure that Kedifa replied
 config-http-code = 400
@@ -331,7 +331,7 @@ monitor-base-url = ${monitor-instance-parameter:monitor-base-url}
 
 [promise-logrotate-setup]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = ${:_buildout_section_name_}.py
 config-command =
   ${logrotate:wrapper-path} -d

software/caddy-frontend/software.cfg

@@ -20,19 +20,8 @@ parts +=
   caucase-eggs
   template
   caddyprofiledeps
-  kedifa-develop
   kedifa
 
-[kedifa-repository]
-recipe = slapos.recipe.build:gitclone
-repository = https://lab.nexedi.com/nexedi/kedifa.git
-git-executable = ${git:location}/bin/git
-revision = 3fccc2ec945c59b644a12fa40225795abd61e0b0
-
-[kedifa-develop]
-recipe = zc.recipe.egg:develop
-setup = ${kedifa-repository:location}
-
 [kedifa]
 recipe = zc.recipe.egg
 eggs =
@@ -222,6 +211,7 @@ mode = 0644
 <=download-template
 
 [versions]
+kedifa = 0.0.6
 # Modern KeDiFa requires zc.lockfile
 zc.lockfile = 1.4
 

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -353,7 +353,7 @@ filename = {{ '%s.conf' % slave_reference }}
 {%-     do part_list.append(monitor_ipv6_section_title) %}
 [{{ monitor_ipv6_section_title }}]
 <= monitor-promise-base
-module = check_icmp_packet_lost
+promise = check_icmp_packet_lost
 name = {{ monitor_ipv6_section_title }}.py
 config-address = {{ dumps(monitor_ipv6_test) }}
 # promise frequency in minutes (2 times/day)
@@ -365,7 +365,7 @@ config-frequency = 720
 {%-     do part_list.append(monitor_ipv4_section_title) %}
 [{{ monitor_ipv4_section_title }}]
 <= monitor-promise-base
-module = check_icmp_packet_lost
+promise = check_icmp_packet_lost
 name = {{ monitor_ipv4_section_title }}.py
 config-address = {{ dumps(monitor_ipv4_test) }}
 config-ipv4 = true
@@ -632,7 +632,7 @@ template = inline:
 
 [promise-expose-csr_id-ip-port]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = expose-csr_id-ip-port-listening.py
 config-host = ${expose-csr_id-configuration:ip}
 config-port = ${expose-csr_id-configuration:port}
@@ -655,7 +655,7 @@ commands =
 
 [promise-logrotate-setup]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = ${:_buildout_section_name_}.py
 config-command =
   ${logrotate:wrapper-path} -d
\ No newline at end of file

software/caddy-frontend/templates/default-virtualhost.conf.in

@@ -2,6 +2,11 @@
 {%- if slave_parameter['prefer-gzip-encoding-to-backend'] %}
 {%- do proxy_append_list.append(('prefer-gzip', 'Proxy which always overrides Accept-Encoding to gzip if such is found')) %}
 {%- endif %} {#- if slave_parameter['prefer-gzip-encoding-to-backend'] #}
+{%- if slave_parameter['path'].strip().strip('/') %}
+{%-   set zope_path = slave_parameter['path'].strip().strip('/') ~ '/' %}
+{%- else %}
+{%-   set zope_path = '' %}
+{%- endif %}
 {%- set http_host_list = [] %}
 {%- set https_host_list = [] %}
 {%- for host in slave_parameter['host_list'] %}
@@ -129,27 +134,27 @@
     regexp (.*)
     if {>Accept-Encoding} match "(^gzip,.*|.*, gzip,.*|.*, gzip$|^gzip$)"
 {%- if tls %}
-    to /prefer-gzip/VirtualHostBase/{scheme}%2F%2F{hostonly}:{{ slave_parameter['virtualhostroot-https-port'] }}%2F{{ slave_parameter['path'] }}%2FVirtualHostRoot/{1}
+    to /prefer-gzip/VirtualHostBase/{scheme}%2F{hostonly}:{{ slave_parameter['virtualhostroot-https-port'] }}%2F{{ zope_path }}VirtualHostRoot/{1}
 {%- else %}
-    to /prefer-gzip/VirtualHostBase/{scheme}%2F%2F{hostonly}:{{ slave_parameter['virtualhostroot-http-port'] }}%2F{{ slave_parameter['path'] }}%2FVirtualHostRoot/{1}
+    to /prefer-gzip/VirtualHostBase/{scheme}%2F{hostonly}:{{ slave_parameter['virtualhostroot-http-port'] }}%2F{{ zope_path }}VirtualHostRoot/{1}
 {%- endif %}
   }
   rewrite {
     regexp (.*)
     if {>Accept-Encoding} not_match "(^gzip,.*|.*, gzip,.*|.*, gzip$|^gzip$)"
 {%- if tls %}
-    to /VirtualHostBase/{scheme}%2F%2F{hostonly}:{{ slave_parameter['virtualhostroot-https-port'] }}%2F{{ slave_parameter['path'] }}%2FVirtualHostRoot/{1}
+    to /VirtualHostBase/{scheme}%2F{hostonly}:{{ slave_parameter['virtualhostroot-https-port'] }}%2F{{ zope_path }}VirtualHostRoot/{1}
 {%- else %}
-    to /VirtualHostBase/{scheme}%2F%2F{hostonly}:{{ slave_parameter['virtualhostroot-http-port'] }}%2F{{ slave_parameter['path'] }}%2FVirtualHostRoot/{1}
+    to /VirtualHostBase/{scheme}%2F{hostonly}:{{ slave_parameter['virtualhostroot-http-port'] }}%2F{{ zope_path }}VirtualHostRoot/{1}
 {%- endif %}
   }
 {%- else %}
   rewrite {
     regexp (.*)
 {%- if tls %}
-    to /VirtualHostBase/{scheme}%2F%2F{hostonly}:{{ slave_parameter['virtualhostroot-https-port'] }}%2F{{ slave_parameter['path'] }}%2FVirtualHostRoot/{1}
+    to /VirtualHostBase/{scheme}%2F{hostonly}:{{ slave_parameter['virtualhostroot-https-port'] }}%2F{{ zope_path }}VirtualHostRoot/{1}
 {%- else %}
-    to /VirtualHostBase/{scheme}%2F%2F{hostonly}:{{ slave_parameter['virtualhostroot-http-port'] }}%2F{{ slave_parameter['path'] }}%2FVirtualHostRoot/{1}
+    to /VirtualHostBase/{scheme}%2F{hostonly}:{{ slave_parameter['virtualhostroot-http-port'] }}%2F{{ zope_path }}VirtualHostRoot/{1}
 {%- endif %}
   } {# rewrite #}
 {%- endif %} {#- if slave_parameter['prefer-gzip-encoding-to-backend'] #}

software/caddy-frontend/test/test.py

@@ -795,7 +795,7 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     with cls.slap.instance_supervisor_rpc as instance_supervisor:
       return getattr(instance_supervisor, method)(*args, **kwargs)
 
-  def assertRejectedSlavePromiseWithPop(self, parameter_dict):
+  def assertRejectedSlavePromiseEmptyWithPop(self, parameter_dict):
     rejected_slave_promise_url = parameter_dict.pop(
       'rejected-slave-promise-url')
 
@@ -806,7 +806,7 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
       else:
         result_json = result.json()
       self.assertEqual(
-        parameter_dict['rejected-slave-dict'],
+        {},
         result_json
       )
     except AssertionError:
@@ -971,12 +971,11 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     return parsed_parameter_dict
 
   def getMasterPartitionPath(self):
-    # partition w/o etc/trafficserver, but with buildout.cfg
+    # partition with etc/nginx-rejected-slave.conf
     return [
       q for q in glob.glob(os.path.join(self.instance_path, '*',))
-      if not os.path.exists(
-        os.path.join(q, 'etc', 'trafficserver')) and os.path.exists(
-          os.path.join(q, 'buildout.cfg'))][0]
+      if os.path.exists(
+        os.path.join(q, 'etc', 'nginx-rejected-slave.conf'))][0]
 
   def parseConnectionParameterDict(self):
     return self.parseParameterDict(
@@ -1238,7 +1237,7 @@ class TestMasterRequestDomain(HttpFrontendTestCase, TestDataMixin):
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)
     self.assertBackendHaproxyStatisticUrl(parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict, 'master-')
-    self.assertRejectedSlavePromiseWithPop(parameter_dict)
+    self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
 
     self.assertEqual(
       {
@@ -1269,7 +1268,7 @@ class TestMasterRequest(HttpFrontendTestCase, TestDataMixin):
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)
     self.assertBackendHaproxyStatisticUrl(parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict, 'master-')
-    self.assertRejectedSlavePromiseWithPop(parameter_dict)
+    self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
     self.assertEqual(
       {
         'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
@@ -1707,7 +1706,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)
     self.assertBackendHaproxyStatisticUrl(parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict, 'master-')
-    self.assertRejectedSlavePromiseWithPop(parameter_dict)
+    self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
 
     expected_parameter_dict = {
       'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
@@ -2231,7 +2230,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
       result,
       'Path',
       '/VirtualHostBase/'
-      'https//typezopepath.example.com:443/path/to/some/resource'
+      'https/typezopepath.example.com:443/path/to/some/resource'
       '/VirtualHostRoot/'
       'test-path/deeper'
     )
@@ -2793,7 +2792,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertEqualResultJson(
       result,
       'Path',
-      '/VirtualHostBase/https//typezope.example.com:443/'
+      '/VirtualHostBase/https/typezope.example.com:443'
       '/VirtualHostRoot/test-path/deeper'
     )
 
@@ -2833,8 +2832,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertEqualResultJson(
       result,
       'Path',
-      '/VirtualHostBase/https//'
-      'typezopeprefergzipencodingtobackendhttpsonly.example.com:443/'
+      '/VirtualHostBase/https/'
+      'typezopeprefergzipencodingtobackendhttpsonly.example.com:443'
       '/VirtualHostRoot/test-path/deeper'
     )
 
@@ -2845,8 +2844,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertEqualResultJson(
       result,
       'Path',
-      '/VirtualHostBase/http//'
-      'typezopeprefergzipencodingtobackendhttpsonly.example.com:80/'
+      '/VirtualHostBase/http/'
+      'typezopeprefergzipencodingtobackendhttpsonly.example.com:80'
       '/VirtualHostRoot/test-path/deeper'
     )
 
@@ -2868,8 +2867,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertEqualResultJson(
       result,
       'Path',
-      '/VirtualHostBase/https//'
-      'typezopeprefergzipencodingtobackendhttpsonly.example.com:443/'
+      '/VirtualHostBase/https/'
+      'typezopeprefergzipencodingtobackendhttpsonly.example.com:443'
       '/VirtualHostRoot/test-path/deeper'
     )
     self.assertEqual(
@@ -2883,8 +2882,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertEqualResultJson(
       result,
       'Path',
-      '/VirtualHostBase/http//'
-      'typezopeprefergzipencodingtobackendhttpsonly.example.com:80/'
+      '/VirtualHostBase/http/'
+      'typezopeprefergzipencodingtobackendhttpsonly.example.com:80'
       '/VirtualHostRoot/test-path/deeper'
     )
     self.assertEqual(
@@ -2911,8 +2910,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertEqualResultJson(
       result,
       'Path',
-      '/VirtualHostBase/https//'
-      'typezopeprefergzipencodingtobackend.example.com:443/'
+      '/VirtualHostBase/https/'
+      'typezopeprefergzipencodingtobackend.example.com:443'
       '/VirtualHostRoot/test-path/deeper'
     )
 
@@ -2949,8 +2948,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertEqualResultJson(
       result,
       'Path',
-      '/VirtualHostBase/https//'
-      'typezopeprefergzipencodingtobackend.example.com:443/'
+      '/VirtualHostBase/https/'
+      'typezopeprefergzipencodingtobackend.example.com:443'
       '/VirtualHostRoot/test-path/deeper'
     )
     self.assertEqual(
@@ -2982,8 +2981,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertEqualResultJson(
       result,
       'Path',
-      '/VirtualHostBase/http//typezopevirtualhostroothttpport'
-      '.example.com:12345//VirtualHostRoot/test-path'
+      '/VirtualHostBase/http/typezopevirtualhostroothttpport'
+      '.example.com:12345/VirtualHostRoot/test-path'
     )
 
   def test_type_zope_virtualhostroot_https_port(self):
@@ -3000,8 +2999,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertEqualResultJson(
       result,
       'Path',
-      '/VirtualHostBase/https//typezopevirtualhostroothttpsport'
-      '.example.com:12345//VirtualHostRoot/test-path'
+      '/VirtualHostBase/https/typezopevirtualhostroothttpsport'
+      '.example.com:12345/VirtualHostRoot/test-path'
     )
 
   def test_type_notebook(self):
@@ -5286,7 +5285,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)
     self.assertBackendHaproxyStatisticUrl(parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict, 'master-')
-    self.assertRejectedSlavePromiseWithPop(parameter_dict)
+    self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
 
     expected_parameter_dict = {
       'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
@@ -5296,10 +5295,6 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
       'rejected-slave-amount': '0',
       'slave-amount': '12',
       'rejected-slave-dict': {
-        # u"_ssl_ca_crt_only":
-        # [u"ssl_ca_crt is present, so ssl_crt and ssl_key are required"],
-        # u"_ssl_key-ssl_crt-unsafe":
-        # [u"slave ssl_key and ssl_crt does not match"]
       },
       'warning-list': [
         u'apache-certificate is obsolete, please use master-key-upload-url',
@@ -5948,7 +5943,7 @@ class TestSlaveSlapOSMasterCertificateCompatibilityUpdate(
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)
     self.assertBackendHaproxyStatisticUrl(parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict, 'master-')
-    self.assertRejectedSlavePromiseWithPop(parameter_dict)
+    self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
 
     expected_parameter_dict = {
       'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
@@ -6053,7 +6048,7 @@ class TestSlaveCiphers(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)
     self.assertBackendHaproxyStatisticUrl(parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict, 'master-')
-    self.assertRejectedSlavePromiseWithPop(parameter_dict)
+    self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
 
     expected_parameter_dict = {
       'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
@@ -6293,6 +6288,29 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
       }
     }
 
+  def assertRejectedSlavePromiseWithPop(self, parameter_dict):
+    rejected_slave_promise_url = parameter_dict.pop(
+      'rejected-slave-promise-url')
+
+    try:
+      result = requests.get(rejected_slave_promise_url, verify=False)
+      if result.text == '':
+        result_json = {}
+      else:
+        result_json = result.json()
+      self.assertEqual(
+        {
+          u'_SITE_4': [u"custom_domain 'duplicate.example.com' clashes"],
+          u'_SITE_2': [u"custom_domain 'duplicate.example.com' clashes"],
+          u'_SITE_3': [u"server-alias 'duplicate.example.com' clashes"]
+        },
+        result_json
+      )
+    except AssertionError:
+      raise
+    except Exception as e:
+      self.fail(e)
+
   def test_master_partition_state(self):
     parameter_dict = self.parseConnectionParameterDict()
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_file_list_plugin-CADDY.txt

@@ -23,6 +23,7 @@ T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend-haproxy-statistic-frontend.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py
 T-2/etc/plugin/buildout-T-2-status.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_file_list_plugin-CADDY.txt

@@ -23,6 +23,7 @@ T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend-haproxy-statistic-frontend.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py
 T-2/etc/plugin/buildout-T-2-status.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_file_list_plugin-CADDY.txt

@@ -23,6 +23,7 @@ T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend-haproxy-statistic-frontend.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py
 T-2/etc/plugin/buildout-T-2-status.py

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_file_list_plugin-CADDY.txt

@@ -23,6 +23,7 @@ T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend-haproxy-statistic-frontend.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py
 T-2/etc/plugin/buildout-T-2-status.py

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_file_list_plugin-CADDY.txt

@@ -23,6 +23,7 @@ T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend-haproxy-statistic-frontend.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py
 T-2/etc/plugin/buildout-T-2-status.py

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_file_list_plugin-CADDY.txt

@@ -23,6 +23,7 @@ T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend-haproxy-statistic-frontend.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py
 T-2/etc/plugin/buildout-T-2-status.py

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlDefaultSlave.test_file_list_plugin-CADDY.txt

@@ -23,6 +23,7 @@ T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend-haproxy-statistic-frontend.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py
 T-2/etc/plugin/buildout-T-2-status.py

software/caddy-frontend/test/test_data/test.TestRe6stVerificationUrlSlave.test_file_list_plugin-CADDY.txt

@@ -23,6 +23,7 @@ T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend-haproxy-statistic-frontend.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py
 T-2/etc/plugin/buildout-T-2-status.py

software/caddy-frontend/test/test_data/test.TestReplicateSlave.test_file_list_plugin-CADDY.txt

@@ -24,6 +24,7 @@ T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend-haproxy-statistic-frontend.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py
 T-2/etc/plugin/buildout-T-2-status.py
@@ -47,6 +48,7 @@ T-2/etc/plugin/trafficserver-port-listening.py
 T-3/etc/plugin/__init__.py
 T-3/etc/plugin/backend-client-caucase-updater.py
 T-3/etc/plugin/backend-haproxy-configuration.py
+T-3/etc/plugin/backend-haproxy-statistic-frontend.py
 T-3/etc/plugin/backend_haproxy_http.py
 T-3/etc/plugin/backend_haproxy_https.py
 T-3/etc/plugin/buildout-T-3-status.py

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_plugin-CADDY.txt

@@ -23,6 +23,7 @@ T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend-haproxy-statistic-frontend.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py
 T-2/etc/plugin/buildout-T-2-status.py

software/caddy-frontend/test/test_data/test.TestSlaveCiphers.test_file_list_plugin-CADDY.txt

@@ -23,6 +23,7 @@ T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend-haproxy-statistic-frontend.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py
 T-2/etc/plugin/buildout-T-2-status.py

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_plugin-CADDY.txt

@@ -23,6 +23,7 @@ T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend-haproxy-statistic-frontend.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py
 T-2/etc/plugin/buildout-T-2-status.py

software/caddy-frontend/test/test_data/test.TestSlaveHealthCheck.test_file_list_plugin-CADDY.txt

@@ -23,6 +23,7 @@ T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend-haproxy-statistic-frontend.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py
 T-2/etc/plugin/buildout-T-2-status.py

software/caddy-frontend/test/test_data/test.TestSlaveHostHaproxyClash.test_file_list_plugin-CADDY.txt

@@ -23,6 +23,7 @@ T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend-haproxy-statistic-frontend.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py
 T-2/etc/plugin/buildout-T-2-status.py

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibility.test_file_list_plugin-CADDY.txt

@@ -23,6 +23,7 @@ T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend-haproxy-statistic-frontend.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py
 T-2/etc/plugin/buildout-T-2-status.py

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster.test_file_list_plugin-CADDY.txt

@@ -23,6 +23,7 @@ T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend-haproxy-statistic-frontend.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py
 T-2/etc/plugin/buildout-T-2-status.py

software/caddy-frontend/test/test_data/test.TestSlaveSlapOSMasterCertificateCompatibilityUpdate.test_file_list_plugin-CADDY.txt

@@ -23,6 +23,7 @@ T-1/etc/plugin/promise-logrotate-setup.py
 T-2/etc/plugin/__init__.py
 T-2/etc/plugin/backend-client-caucase-updater.py
 T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend-haproxy-statistic-frontend.py
 T-2/etc/plugin/backend_haproxy_http.py
 T-2/etc/plugin/backend_haproxy_https.py
 T-2/etc/plugin/buildout-T-2-status.py

software/cloudooo/buildout.hash.cfg

@@ -18,4 +18,4 @@ md5sum = e986de01a57161b32425f1cd3ccac924
 
 [template-cloudooo-instance]
 filename = instance-cloudooo.cfg.in
-md5sum = 6e4bdb1df02aed5c96ccf7b9c3c71b89
+md5sum = 3e6830c21c206b3ae1140375e5e63b46

software/cloudooo/instance-cloudooo.cfg.in

@@ -105,7 +105,7 @@ ssl-session-cache = ${directory:log}/apache-ssl-session-cache
 [apache-promise]
 # Check any apache port in ipv4, expect other ports and ipv6 to behave consistently
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = apache.py
 config-url = https://{{ ipv4 }}:{{ apache_dict.values()[0][0] }}
 # XXX cloudooo replies "400 Bad Request" for GET on / but what we want to check

software/dream/buildout.hash.cfg

 [instance]
 filename = instance.cfg
-md5sum = 646e50cfa93681e8bd85767621c7a39d
+md5sum = ddd17fab15afa5a27cdc0761fbc8f34c

software/dream/instance.cfg

@@ -41,7 +41,7 @@ wrapper-path = $${directory:service}/dream_platform
 
 [dream-platform-url-available]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = $${:_buildout_section_name_}.py
 config-url=  $${dream_platform_parameter:url}
 

software/dream/software.cfg

@@ -29,10 +29,16 @@ script = dream_testrunner
 initialization =
   ${manpy:initialization}
 
+[nodejs]
+<= nodejs-8.9.4
+
 [npm_install]
 recipe = plone.recipe.command
 stop-on-error = true
-command = cd ${dream-repository.git:location} && PATH=${git:location}/bin/:${nodejs:location}/bin/:$PATH ${nodejs:location}/bin/npm install .
+command =
+  cd ${dream-repository.git:location}
+  PATH=${git:location}/bin:${nodejs:location}/bin:$PATH
+  npm install .
 update_command = ${:command}
 
 [versions]

software/erp5/test/setup.py

@@ -24,6 +24,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from __future__ import absolute_import
 from setuptools import setup, find_packages
 
 version = '0.0.1.dev0'
@@ -52,7 +53,6 @@ setup(name=name,
         'cryptography',
         'pexpect',
         'pyOpenSSL',
-        'typing; python_version<"3"',
       ],
       test_suite='test',
     )

software/erp5/test/test/__init__.py

@@ -25,6 +25,7 @@
 #
 ##############################################################################
 
+from __future__ import absolute_import
 import json
 import os
 

software/erp5/test/test/test_balancer.py

+from __future__ import absolute_import
 import glob
 import hashlib
 import json
@@ -8,10 +9,8 @@ import shutil
 import subprocess
 import tempfile
 import time
-import urllib
-import urlparse
-from BaseHTTPServer import BaseHTTPRequestHandler
-from typing import Dict
+import six.moves.urllib.request, six.moves.urllib.parse, six.moves.urllib.error
+from six.moves.BaseHTTPServer import BaseHTTPRequestHandler
 
 import mock
 import OpenSSL.SSL
@@ -28,6 +27,7 @@ from slapos.testing.utils import (CrontabMixin, ManagedHTTPServer,
                                   findFreeTCPPort)
 
 from . import ERP5InstanceTestCase, setUpModule
+from six.moves import range
 
 setUpModule  # pyflakes
 
@@ -44,10 +44,10 @@ class EchoHTTPServer(ManagedHTTPServer):
       response = json.dumps(
           {
               'Path': self.path,
-              'Incoming Headers': self.headers.dict
+              'Incoming Headers': dict(self.headers.items()),
           },
           indent=2,
-      )
+      ).encode('utf-8')
       self.end_headers()
       self.wfile.write(response)
 
@@ -67,11 +67,11 @@ class EchoHTTP11Server(ManagedHTTPServer):
       response = json.dumps(
           {
               'Path': self.path,
-              'Incoming Headers': self.headers.dict
+              'Incoming Headers': dict(self.headers.items()),
           },
           indent=2,
-      )
-      self.send_header("Content-Length", len(response))
+      ).encode('utf-8')
+      self.send_header("Content-Length", str(len(response)))
       self.end_headers()
       self.wfile.write(response)
 
@@ -139,7 +139,7 @@ class BalancerTestCase(ERP5InstanceTestCase):
 
   @classmethod
   def _getInstanceParameterDict(cls):
-    # type: () -> Dict
+    # type: () -> dict
     return {
         'tcpv4-port': 8000,
         'computer-memory-percent-threshold': 100,
@@ -174,10 +174,11 @@ class BalancerTestCase(ERP5InstanceTestCase):
 
   @classmethod
   def getInstanceParameterDict(cls):
-    # type: () -> Dict
+    # type: () -> dict
     return {'_': json.dumps(cls._getInstanceParameterDict())}
 
   def setUp(self):
+    # type: () -> None
     self.default_balancer_url = json.loads(
         self.computer_partition.getConnectionParameterDict()['_'])['default']
 
@@ -192,7 +193,7 @@ class SlowHTTPServer(ManagedHTTPServer):
       self.send_header("Content-Type", "text/plain")
       time.sleep(2)
       self.end_headers()
-      self.wfile.write("OK\n")
+      self.wfile.write(b"OK\n")
 
     log_message = logging.getLogger(__name__ + '.SlowHandler').info
 
@@ -203,7 +204,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
   __partition_reference__ = 'l'
   @classmethod
   def _getInstanceParameterDict(cls):
-    # type: () -> Dict
+    # type: () -> dict
     parameter_dict = super(TestLog, cls)._getInstanceParameterDict()
     # use a slow server instead
     parameter_dict['dummy_http_server'] = [[cls.getManagedResource("slow_web_server", SlowHTTPServer).netloc, 1, False]]
@@ -212,7 +213,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
   def test_access_log_format(self):
     # type: () -> None
     requests.get(
-        urlparse.urljoin(self.default_balancer_url, '/url_path'),
+        six.moves.urllib.parse.urljoin(self.default_balancer_url, '/url_path'),
         verify=False,
     )
     time.sleep(.5) # wait a bit more until access is logged
@@ -285,6 +286,7 @@ class TestLog(BalancerTestCase, CrontabMixin):
     self.assertFalse(os.path.exists(rotated_log_file))
 
   def test_error_log(self):
+    # type: () -> None
     # stop backend server
     backend_server = self.getManagedResource("slow_web_server", SlowHTTPServer)
     self.addCleanup(backend_server.open)
@@ -326,7 +328,7 @@ class BalancerCookieHTTPServer(ManagedHTTPServer):
           # The name of this cookie is SERVERID
           assert self.headers['X-Balancer-Current-Cookie'] == 'SERVERID'
         self.end_headers()
-        self.wfile.write(server._name)
+        self.wfile.write(server._name.encode('utf-8'))
       log_message = logging.getLogger(__name__ + '.BalancerCookieHTTPServer').info
 
     return RequestHandler
@@ -338,7 +340,7 @@ class TestBalancer(BalancerTestCase):
   __partition_reference__ = 'b'
   @classmethod
   def _getInstanceParameterDict(cls):
-    # type: () -> Dict
+    # type: () -> dict
     parameter_dict = super(TestBalancer, cls)._getInstanceParameterDict()
 
     # use two backend servers
@@ -349,6 +351,7 @@ class TestBalancer(BalancerTestCase):
     return parameter_dict
 
   def test_balancer_round_robin(self):
+    # type: () -> None
     # requests are by default balanced to both servers
     self.assertEqual(
         {requests.get(self.default_balancer_url, verify=False).text for _ in range(10)},
@@ -356,6 +359,7 @@ class TestBalancer(BalancerTestCase):
     )
 
   def test_balancer_server_down(self):
+    # type: () -> None
     # if one backend is down, it is excluded from balancer
     self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).close()
     self.addCleanup(self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).open)
@@ -365,14 +369,16 @@ class TestBalancer(BalancerTestCase):
     )
 
   def test_balancer_set_cookie(self):
+    # type: () -> None
     # if backend provides a "SERVERID" cookie, balancer will overwrite it with the
     # backend selected by balancing algorithm
     self.assertIn(
-        requests.get(urlparse.urljoin(self.default_balancer_url, '/set_cookie'), verify=False).cookies['SERVERID'],
+        requests.get(six.moves.urllib.parse.urljoin(self.default_balancer_url, '/set_cookie'), verify=False).cookies['SERVERID'],
         ('default-0', 'default-1'),
     )
 
   def test_balancer_respects_sticky_cookie(self):
+    # type: () -> None
     # if request is made with the sticky cookie, the client stick on one balancer
     cookies = dict(SERVERID='default-1')
     self.assertEqual(
@@ -388,6 +394,7 @@ class TestBalancer(BalancerTestCase):
         'backend_web_server1')
 
   def test_balancer_stats_socket(self):
+    # type: () -> None
     # real time statistics can be obtained by using the stats socket and there
     # is a wrapper which makes this a bit easier.
     socat_process = subprocess.Popen(
@@ -397,14 +404,14 @@ class TestBalancer(BalancerTestCase):
         stderr=subprocess.STDOUT
     )
     try:
-      output, _ = socat_process.communicate("show stat\n")
+      output, _ = socat_process.communicate(b"show stat\n")
     except:
       socat_process.kill()
       socat_process.wait()
       raise
     self.assertEqual(socat_process.poll(), 0)
     # output is a csv
-    self.assertIn('family_default,FRONTEND,', output)
+    self.assertIn(b'family_default,FRONTEND,', output)
 
 
 class TestTestRunnerEntryPoints(BalancerTestCase):
@@ -413,7 +420,7 @@ class TestTestRunnerEntryPoints(BalancerTestCase):
   __partition_reference__ = 't'
   @classmethod
   def _getInstanceParameterDict(cls):
-    # type: () -> Dict
+    # type: () -> dict
     parameter_dict = super(
         TestTestRunnerEntryPoints,
         cls,
@@ -436,23 +443,24 @@ class TestTestRunnerEntryPoints(BalancerTestCase):
     return parameter_dict
 
   def test_use_proper_backend(self):
+    # type: () -> None
     # requests are directed to proper backend based on URL path
     test_runner_url_list = self.getRootPartitionConnectionParameterDict(
     )['default-test-runner-url-list']
     url_0, url_1, url_2 = test_runner_url_list
     self.assertEqual(
-        urlparse.urlparse(url_0).netloc,
-        urlparse.urlparse(url_1).netloc)
+        six.moves.urllib.parse.urlparse(url_0).netloc,
+        six.moves.urllib.parse.urlparse(url_1).netloc)
     self.assertEqual(
-        urlparse.urlparse(url_0).netloc,
-        urlparse.urlparse(url_2).netloc)
+        six.moves.urllib.parse.urlparse(url_0).netloc,
+        six.moves.urllib.parse.urlparse(url_2).netloc)
 
     path_0 = '/VirtualHostBase/https/{netloc}/VirtualHostRoot/_vh_unit_test_0/something'.format(
-        netloc=urlparse.urlparse(url_0).netloc)
+        netloc=six.moves.urllib.parse.urlparse(url_0).netloc)
     path_1 = '/VirtualHostBase/https/{netloc}/VirtualHostRoot/_vh_unit_test_1/something'.format(
-        netloc=urlparse.urlparse(url_0).netloc)
+        netloc=six.moves.urllib.parse.urlparse(url_0).netloc)
     path_2 = '/VirtualHostBase/https/{netloc}/VirtualHostRoot/_vh_unit_test_2/something'.format(
-        netloc=urlparse.urlparse(url_0).netloc)
+        netloc=six.moves.urllib.parse.urlparse(url_0).netloc)
 
     self.assertEqual(
         {
@@ -489,7 +497,7 @@ class TestHTTP(BalancerTestCase):
   """
   @classmethod
   def _getInstanceParameterDict(cls):
-    # type: () -> Dict
+    # type: () -> dict
     parameter_dict = super(TestHTTP, cls)._getInstanceParameterDict()
     # use a HTTP/1.1 server instead
     parameter_dict['dummy_http_server'] = [[cls.getManagedResource("HTTP/1.1 Server", EchoHTTP11Server).netloc, 1, False]]
@@ -511,7 +519,7 @@ class TestHTTP(BalancerTestCase):
             '%{http_version}',
             self.default_balancer_url,
         ]),
-        '2',
+        b'2',
     )
 
   def test_keep_alive(self):
@@ -530,7 +538,7 @@ class TestHTTP(BalancerTestCase):
       session.get(self.default_balancer_url).raise_for_status()
     new_conn.assert_not_called()
 
-    parsed_url = urlparse.urlparse(self.default_balancer_url)
+    parsed_url = six.moves.urllib.parse.urlparse(self.default_balancer_url)
     # check that we have an open file for the ip connection
     self.assertTrue([
         c for c in psutil.Process(os.getpid()).connections()
@@ -553,12 +561,12 @@ class ContentTypeHTTPServer(ManagedHTTPServer):
       # type: () -> None
       self.send_response(200)
       if self.path == '/':
-        self.send_header("Content-Length", 0)
+        self.send_header("Content-Length", '0')
         return self.end_headers()
       content_type = self.path[1:]
-      body = "OK"
+      body = b"OK"
       self.send_header("Content-Type", content_type)
-      self.send_header("Content-Length", len(body))
+      self.send_header("Content-Length", str(len(body)))
       self.end_headers()
       self.wfile.write(body)
 
@@ -571,7 +579,7 @@ class TestContentEncoding(BalancerTestCase):
   __partition_reference__ = 'ce'
   @classmethod
   def _getInstanceParameterDict(cls):
-    # type: () -> Dict
+    # type: () -> dict
     parameter_dict = super(TestContentEncoding, cls)._getInstanceParameterDict()
     parameter_dict['dummy_http_server'] = [
         [cls.getManagedResource("content_type_server", ContentTypeHTTPServer).netloc, 1, False],
@@ -599,7 +607,7 @@ class TestContentEncoding(BalancerTestCase):
         'application/font-woff2',
         'application/x-font-opentype',
         'application/wasm',):
-      resp = requests.get(urlparse.urljoin(self.default_balancer_url, content_type), verify=False)
+      resp = requests.get(six.moves.urllib.parse.urljoin(self.default_balancer_url, content_type), verify=False)
       self.assertEqual(resp.headers['Content-Type'], content_type)
       self.assertEqual(
           resp.headers.get('Content-Encoding'),
@@ -609,7 +617,7 @@ class TestContentEncoding(BalancerTestCase):
 
   def test_no_gzip_encoding(self):
     # type: () -> None
-    resp = requests.get(urlparse.urljoin(self.default_balancer_url, '/image/png'), verify=False)
+    resp = requests.get(six.moves.urllib.parse.urljoin(self.default_balancer_url, '/image/png'), verify=False)
     self.assertNotIn('Content-Encoding', resp.headers)
     self.assertEqual(resp.text, 'OK')
 
@@ -692,7 +700,7 @@ class CaucaseCertificate(ManagedResource):
       cas_args + [
           '--send-csr', self.csr_file,
       ],
-    ).split()[0]
+    ).split()[0].decode()
     assert csr_id
 
     for _ in range(30):
@@ -708,11 +716,11 @@ class CaucaseCertificate(ManagedResource):
         time.sleep(1)
     else:
       raise RuntimeError('getting service certificate failed.')
-    with open(self.cert_file) as f:
-      assert 'BEGIN CERTIFICATE' in f.read()
+    with open(self.cert_file) as cert_file:
+      assert 'BEGIN CERTIFICATE' in cert_file.read()
 
   def revoke(self, caucase):
-    # type: (str, CaucaseService) -> None
+    # type: (CaucaseService) -> None
     """Revoke the client certificate on this caucase instance.
     """
     subprocess.check_call([
@@ -729,7 +737,7 @@ class TestFrontendXForwardedFor(BalancerTestCase):
 
   @classmethod
   def _getInstanceParameterDict(cls):
-    # type: () -> Dict
+    # type: () -> dict
     frontend_caucase = cls.getManagedResource('frontend_caucase', CaucaseService)
     certificate = cls.getManagedResource('client_certificate', CaucaseCertificate)
     certificate.request(u'shared frontend', frontend_caucase)
@@ -784,10 +792,10 @@ class TestServerTLSProvidedCertificate(BalancerTestCase):
 
   @classmethod
   def _getInstanceParameterDict(cls):
-    # type: () -> Dict
+    # type: () -> dict
     server_caucase = cls.getManagedResource('server_caucase', CaucaseService)
     server_certificate = cls.getManagedResource('server_certificate', CaucaseCertificate)
-    server_certificate.request(cls._ipv4_address.decode(), server_caucase)
+    server_certificate.request(six.ensure_text(cls._ipv4_address), server_caucase)
     parameter_dict = super(TestServerTLSProvidedCertificate, cls)._getInstanceParameterDict()
     with open(server_certificate.cert_file) as f:
       parameter_dict['ssl']['cert'] = f.read()
@@ -806,7 +814,7 @@ class TestClientTLS(BalancerTestCase):
 
   @classmethod
   def _getInstanceParameterDict(cls):
-    # type: () -> Dict
+    # type: () -> dict
     frontend_caucase1 = cls.getManagedResource('frontend_caucase1', CaucaseService)
     certificate1 = cls.getManagedResource('client_certificate1', CaucaseCertificate)
     certificate1.request(u'client_certificate1', frontend_caucase1)
@@ -827,7 +835,6 @@ class TestClientTLS(BalancerTestCase):
 
   def test_refresh_crl(self):
     # type: () -> None
-
     logger = self.logger
 
     class DebugLogFile:
@@ -846,6 +853,7 @@ class TestClientTLS(BalancerTestCase):
       # when client certificate can be authenticated, backend receive the CN of
       # the client certificate in "remote-user" header
       def _make_request():
+        # type: () -> dict
         return requests.get(
             self.default_balancer_url,
             cert=(client_certificate.cert_file, client_certificate.key_file),
@@ -897,6 +905,7 @@ class TestClientTLS(BalancerTestCase):
       with self.assertRaisesRegexp(Exception, 'certificate revoked'):
         _make_request()
 
+
 class TestPathBasedRouting(BalancerTestCase):
   """Check path-based routing rewrites URLs as expected.
   """
@@ -904,7 +913,7 @@ class TestPathBasedRouting(BalancerTestCase):
 
   @classmethod
   def _getInstanceParameterDict(cls):
-    # type: () -> Dict
+    # type: () -> dict
     parameter_dict = super(
       TestPathBasedRouting,
       cls,
@@ -936,14 +945,15 @@ class TestPathBasedRouting(BalancerTestCase):
     published_dict = json.loads(self.computer_partition.getConnectionParameterDict()['_'])
     scheme = 'scheme'
     netloc = 'example.com:8080'
-    prefix = '/VirtualHostBase/' + scheme + '//' + urllib.quote(
+    prefix = '/VirtualHostBase/' + scheme + '//' + six.moves.urllib.parse.quote(
       netloc,
       safe='',
     )
-    # For easier reading of test data, visualy separating the virtual host
+    # For easier reading of test data, visually separating the virtual host
     # base from the virtual host root
     vhr = '/VirtualHostRoot'
     def assertRoutingEqual(family, path, expected_path):
+      # type: (str, str, str) -> None
       # sanity check: unlike the rules, this test is sensitive to outermost
       # slashes, and paths must be absolute-ish for code simplicity.
       assert path.startswith('/')
@@ -959,7 +969,7 @@ class TestPathBasedRouting(BalancerTestCase):
       # test will need to be updated accordingly.
       self.assertEqual(
         requests.get(
-          urlparse.urljoin(published_dict[family], prefix + vhr + path),
+          six.moves.urllib.parse.urljoin(published_dict[family], prefix + vhr + path),
           verify=False,
         ).json()['Path'],
         expected_path,
@@ -978,7 +988,7 @@ class TestPathBasedRouting(BalancerTestCase):
     # Rule precedence: family rules applied before general rules.
     assertRoutingEqual('default', '/next',          prefix + '/erp5/web_site_module/another_next_website' + vhr + '/_vh_next')
     # Fallback on general rules when no family-specific rule matches
-    # Note: the root is special in that there is aways a trailing slash in the
+    # Note: the root is special in that there is always a trailing slash in the
     # produced URL.
     assertRoutingEqual('default', '/',              prefix + '/erp5/web_site_module/123' + vhr + '/')
     # Rule-less family reach general rules.

software/erp5/test/test/test_erp5.py

@@ -25,10 +25,11 @@
 #
 ##############################################################################
 
+from __future__ import absolute_import
 import os
 import json
 import glob
-import urlparse
+import six.moves.urllib.parse
 import socket
 import time
 
@@ -37,6 +38,9 @@ import requests
 
 from . import ERP5InstanceTestCase
 from . import setUpModule
+import six
+from six.moves import map
+from six.moves import range
 setUpModule # pyflakes
 
 
@@ -48,7 +52,7 @@ class TestPublishedURLIsReachableMixin(object):
     # We access ERP5 trough a "virtual host", which should make
     # ERP5 produce URLs using https://virtual-host-name:1234/virtual_host_root
     # as base.
-    virtual_host_url = urlparse.urljoin(
+    virtual_host_url = six.moves.urllib.parse.urljoin(
         base_url,
         '/VirtualHostBase/https/virtual-host-name:1234/{}/VirtualHostRoot/_vh_virtual_host_root/'
         .format(site_id))
@@ -76,7 +80,7 @@ class TestPublishedURLIsReachableMixin(object):
 
     # login page can be rendered and contain the text "ERP5"
     r = session.get(
-        urlparse.urljoin(base_url, '{}/login_form'.format(site_id)),
+        six.moves.urllib.parse.urljoin(base_url, '{}/login_form'.format(site_id)),
         verify=verify,
         allow_redirects=False,
     )
@@ -119,6 +123,7 @@ class TestMedusa(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   def getInstanceParameterDict(cls):
     return {'_': json.dumps({'wsgi': False})}
 
+
 class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   """Test ERP5 Jupyter notebook
   """
@@ -143,6 +148,7 @@ class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
       [result.status_code, result.is_redirect, result.headers['Location']]
     )
 
+
 class TestBalancerPorts(ERP5InstanceTestCase):
   """Instantiate with two zope families, this should create for each family:
    - a balancer entry point with corresponding haproxy
@@ -169,7 +175,7 @@ class TestBalancerPorts(ERP5InstanceTestCase):
     }
 
   def checkValidHTTPSURL(self, url):
-    parsed = urlparse.urlparse(url)
+    parsed = six.moves.urllib.parse.urlparse(url)
     self.assertEqual(parsed.scheme, 'https')
     self.assertTrue(parsed.hostname)
     self.assertTrue(parsed.port)
@@ -254,7 +260,7 @@ class TestSeleniumTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMi
     with open(config_file.strip()) as f:
       self.assertEqual(
           f.read(),
-          json.dumps(json.loads(self.getInstanceParameterDict()['_'])['test-runner']))
+          json.dumps(json.loads(self.getInstanceParameterDict()['_'])['test-runner'], sort_keys=True))
 
 
 class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
@@ -270,8 +276,8 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
     """
     # self.computer_partition_root_path is the path of root partition.
     # we want to assert that no scripts exist in any partition.
-    bin_programs = map(os.path.basename,
-      glob.glob(self.computer_partition_root_path + "/../*/bin/*"))
+    bin_programs = list(map(os.path.basename,
+      glob.glob(self.computer_partition_root_path + "/../*/bin/*")))
 
     self.assertTrue(bin_programs) # just to check the glob was correct.
     self.assertNotIn('runUnitTest', bin_programs)
@@ -352,7 +358,7 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
       storage["storage"] = "root"
       storage["server"] = zeo_addr
       with open('%s/etc/zope-%s.conf' % (partition, zope)) as f:
-        conf = map(str.strip, f.readlines())
+        conf = list(map(str.strip, f.readlines()))
       i = conf.index("<zodb_db root>") + 1
       conf = iter(conf[i:conf.index("</zodb_db>", i)])
       for line in conf:
@@ -361,23 +367,23 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
             if line == '</zeoclient>':
               break
             checkParameter(line, storage)
-          for k, v in storage.iteritems():
+          for k, v in six.iteritems(storage):
             self.assertIsNone(v, k)
           del storage
         else:
           checkParameter(line, zodb)
-      for k, v in zodb.iteritems():
+      for k, v in six.iteritems(zodb):
         self.assertIsNone(v, k)
 
     partition = self.getComputerPartitionPath('zope-a')
-    for zope in xrange(3):
+    for zope in range(3):
       checkConf({
           "cache-size-bytes": "20MB",
         }, {
           "cache-size": "50MB",
         })
     partition = self.getComputerPartitionPath('zope-bb')
-    for zope in xrange(5):
+    for zope in range(5):
       checkConf({
           "cache-size-bytes": "500MB" if zope else 1<<20,
         }, {

software/erp5/test/test/test_mariadb.py

@@ -26,10 +26,11 @@
 #
 ##############################################################################
 
+from __future__ import absolute_import
 import os
 import json
 import glob
-import urlparse
+import six.moves.urllib.parse
 import socket
 import sys
 import time
@@ -60,6 +61,7 @@ class MariaDBTestCase(ERP5InstanceTestCase):
 
   @classmethod
   def _getInstanceParameterDict(cls):
+    # type: () -> dict
     return {
         'tcpv4-port': 3306,
         'max-connection-count': 5,
@@ -75,12 +77,14 @@ class MariaDBTestCase(ERP5InstanceTestCase):
 
   @classmethod
   def getInstanceParameterDict(cls):
+    # type: () -> dict
     return {'_': json.dumps(cls._getInstanceParameterDict())}
 
   def getDatabaseConnection(self):
+    # type: () -> MySQLdb.connections.Connection
     connection_parameter_dict = json.loads(
         self.computer_partition.getConnectionParameterDict()['_'])
-    db_url = urlparse.urlparse(connection_parameter_dict['database-list'][0])
+    db_url = six.moves.urllib.parse.urlparse(connection_parameter_dict['database-list'][0])
     self.assertEqual('mysql', db_url.scheme)
 
     self.assertTrue(db_url.path.startswith('/'))
@@ -91,12 +95,15 @@ class MariaDBTestCase(ERP5InstanceTestCase):
         host=db_url.hostname,
         port=db_url.port,
         db=database_name,
+        use_unicode=True,
+        charset='utf8mb4'
     )
 
 
 class TestCrontabs(MariaDBTestCase, CrontabMixin):
 
   def test_full_backup(self):
+    # type: () -> None
     self._executeCrontabAtDate('mariadb-backup', '2050-01-01')
     with gzip.open(
         os.path.join(
@@ -106,10 +113,11 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
             'mariadb-full',
             '20500101000000.sql.gz',
         ),
-        'r') as dump:
+        'rt') as dump:
       self.assertIn('CREATE TABLE', dump.read())
 
   def test_logrotate_and_slow_query_digest(self):
+    # type: () -> None
     # slow query digest needs to run after logrotate, since it operates on the rotated
     # file, so this tests both logrotate and slow query digest.
 
@@ -148,7 +156,7 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
         'slowquery_digest',
         'slowquery_digest.txt-2050-01-01.xz',
     )
-    with lzma.open(slow_query_report, 'r') as f:
+    with lzma.open(slow_query_report, 'rt') as f:
       # this is the hash for our "select sleep(n)" slow query
       self.assertIn("ID 0xF9A57DD5A41825CA", f.read())
 
@@ -170,7 +178,7 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
       subprocess.check_output('faketime 2050-01-01 %s' % check_slow_query_promise_plugin['command'], shell=True)
     self.assertEqual(
         error_context.exception.output,
-"""\
+b"""\
 Threshold is lower than expected: 
 Expected total queries : 1.0 and current is: 2
 Expected slowest query : 0.1 and current is: 3
@@ -179,6 +187,7 @@ Expected slowest query : 0.1 and current is: 3
 
 class TestMariaDB(MariaDBTestCase):
   def test_utf8_collation(self):
+    # type: () -> None
     cnx = self.getDatabaseConnection()
     with contextlib.closing(cnx):
       cnx.query(
@@ -199,11 +208,12 @@ class TestMariaDB(MariaDBTestCase):
           """
           select * from test_utf8_collation where col1 = "a"
           """)
-      self.assertEqual((('à',),), cnx.store_result().fetch_row(maxrows=2))
+      self.assertEqual(((u'à',),), cnx.store_result().fetch_row(maxrows=2))
 
 
 class TestMroonga(MariaDBTestCase):
   def test_mroonga_plugin_loaded(self):
+    # type: () -> None
     cnx = self.getDatabaseConnection()
     with contextlib.closing(cnx):
       cnx.query("show plugins")
@@ -213,6 +223,7 @@ class TestMroonga(MariaDBTestCase):
           plugins)
 
   def test_mroonga_normalize_udf(self):
+    # type: () -> None
     # example from https://mroonga.org/docs/reference/udf/mroonga_normalize.html#usage
     cnx = self.getDatabaseConnection()
     with contextlib.closing(cnx):
@@ -220,7 +231,8 @@ class TestMroonga(MariaDBTestCase):
           """
           SELECT mroonga_normalize("ABCDあぃうぇ㍑")
           """)
-      self.assertEqual((('abcdあぃうぇリットル',),),
+      # XXX this is returned as bytes by mroonga/mariadb (this might be a bug)
+      self.assertEqual(((u'abcdあぃうぇリットル'.encode('utf-8'),),),
                        cnx.store_result().fetch_row(maxrows=2))
 
       if 0:
@@ -233,10 +245,11 @@ class TestMroonga(MariaDBTestCase):
             """
             SELECT mroonga_normalize("aBｃＤあぃウェ㍑", "NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark")
             """)
-        self.assertEqual((('ABCDあぃうぇ㍑',),),
+        self.assertEqual(((u'ABCDあぃうぇ㍑'.encode('utf-8'),),),
                          cnx.store_result().fetch_row(maxrows=2))
 
   def test_mroonga_full_text_normalizer(self):
+    # type: () -> None
     # example from https://mroonga.org//docs/tutorial/storage.html#how-to-specify-the-normalizer
     cnx = self.getDatabaseConnection()
     with contextlib.closing(cnx):
@@ -269,11 +282,12 @@ class TestMroonga(MariaDBTestCase):
            WHERE MATCH (content) AGAINST ("+ﾌﾞﾗｯｸ" IN BOOLEAN MODE)
           """)
       self.assertEqual(
-          ((datetime.date(2013, 4, 23), 'ブラックコーヒーを飲んだ。'),),
+          ((datetime.date(2013, 4, 23), u'ブラックコーヒーを飲んだ。'),),
           cnx.store_result().fetch_row(maxrows=2),
       )
 
   def test_mroonga_full_text_normalizer_TokenBigramSplitSymbolAlphaDigit(self):
+    # type: () -> None
     # Similar to as ERP5's testI18NSearch with erp5_full_text_mroonga_catalog
     cnx = self.getDatabaseConnection()
     with contextlib.closing(cnx):
@@ -317,11 +331,12 @@ class TestMroonga(MariaDBTestCase):
       self.assertEqual(((1,),), cnx.store_result().fetch_row(maxrows=2))
 
   def test_mroonga_full_text_stem(self):
+    # type: () -> None
     # example from https://mroonga.org//docs/tutorial/storage.html#how-to-specify-the-token-filters
     cnx = self.getDatabaseConnection()
     with contextlib.closing(cnx):
       cnx.query("SELECT mroonga_command('register token_filters/stem')")
-      self.assertEqual((('true',),), cnx.store_result().fetch_row(maxrows=2))
+      self.assertEqual(((b'true',),), cnx.store_result().fetch_row(maxrows=2))
       cnx.query(
           """
           CREATE TABLE memos (

software/erp5/test/test/test_wcfs.py

@@ -16,6 +16,7 @@
 # See COPYING file for full licensing terms.
 # See https://www.nexedi.com/licensing for rationale and options.
 
+from __future__ import absolute_import
 import json
 import os.path
 import unittest

software/erp5testnode/buildout.hash.cfg

@@ -14,8 +14,8 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = d9c7c3acb975d44c6c66ca32c550ac58
+md5sum = 6dcbe21f99aa6675e8e3b74bc9cbb0e6
 
 [template-default]
 filename = instance-default.cfg.jinja.in
-md5sum = ffb6b74b55e7ca01666254353ae1cebe
+md5sum = 536a28ff250c691ca374f75aa5f0aa76

software/erp5testnode/instance-default.cfg.jinja.in

@@ -261,7 +261,7 @@ instance-promises =
 
 [shellinabox-frontend-listen-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = $${:_buildout_section_name_}.py
 config-host = $${shellinabox-frontend:hostname}
 config-port = $${shellinabox-frontend:port}
@@ -278,6 +278,6 @@ config-port = $${shellinabox-frontend:port}
 
 [testnode-log-frontend-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = $${:_buildout_section_name_}.py
 config-url = $${testnode-log-frontend:connection-secure_access}

software/erp5testnode/instance.cfg

@@ -34,6 +34,6 @@ default-parameters =
     "node-quantity": 1,
     "test-suite-master-url": "",
     "instance-dict": "",
-    "software-path-list": ["https://lab.nexedi.com/nexedi/slapos/raw/1.0.181/software/seleniumrunner/software.cfg"],
+    "software-path-list": ["https://lab.nexedi.com/nexedi/slapos/raw/1.0.210/software/seleniumrunner/software.cfg"],
     "keep-log-days": 15
   }

software/erp5testnode/software.cfg

@@ -49,7 +49,7 @@ WebTest = 2.0.33
 soupsieve = 1.8
 #gitdb2 = 2.0.4
 #smmap2 = 2.0.4
-waitress = 1.2.1
+waitress = 1.4.4
 z3c.etestbrowser = 2.0.1
 zope.testbrowser = 5.3.2
 WSGIProxy2 = 0.4.6

software/fluentd/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+[instance-profile]
+filename = instance.cfg
+md5sum = c265bf7ec199190ba9f77960cac5da38
+
+[template-fluentd]
+filename = instance-fluentd.cfg
+md5sum = 35f9d95f6a75e28bfeafc3568ca16f05

software/fluentd/instance-fluentd.cfg

+[service-fluentd]
+recipe = slapos.cookbook:wrapper
+wrapper-path = {{ directory['service'] }}/fluentd-service
+command-line = ${fluentd:location}/bin/fluentd -v -c {{ fluentd_agent_conf }}
+environment =
+  GEM_PATH=${fluentd:location}/lib/ruby/gems/
+
+{% set part_list = [] -%}
+
+{% for port in port_list -%}
+{%   set promise_section_title = 'fluentd-port-' ~ port ~ '-listening' -%}
+{%   do part_list.append(promise_section_title) -%}
+[{{ promise_section_title }}]
+<= monitor-promise-base
+promise = check_socket_listening
+name = {{ promise_section_title }}.py
+config-host = $${slap-configuration:ipv6-random}
+config-port = {{ port }}
+{% endfor %}
+
+[buildout]
+parts =
+  service-fluentd
+{%- for part in part_list %}
+  {{ part }}
+{%- endfor %}
+
+extends = ${monitor-template:output}

software/fluentd/instance.cfg

+[buildout]
+parts =
+  switch-softwaretype
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[switch-softwaretype]
+recipe = slapos.cookbook:switch-softwaretype
+default = dynamic-template-fluentd:rendered
+RootSoftwareInstance = $${:default}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = $${buildout:directory}
+etc = $${:home}/etc
+var = $${:home}/var
+service = $${:etc}/service
+bin = $${:home}/bin
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap_connection:computer_id}
+partition = $${slap_connection:partition_id}
+url = $${slap_connection:server_url}
+key = $${slap_connection:key_file}
+cert = $${slap_connection:cert_file}
+
+[dynamic-template-fluentd]
+recipe = slapos.recipe.template:jinja2
+template = ${template-fluentd:output}
+rendered = $${buildout:directory}/instance-fluentd.cfg
+extensions = jinja2.ext.do
+context =
+  key fluentd_agent_conf fluentd-agent-conf:rendered
+  key port_list fluentd-conf:port-list
+  section directory directory
+
+[fluentd-conf]
+recipe = slapos.recipe.build
+slapparameter-dict = $${slap-configuration:configuration}
+init =
+  import re
+  options['text'] = options['slapparameter-dict'].get('conf_text') or ''
+  options['port-list'] = re.findall(r'<source>.*port (\d+).*<\/source>', options['text'], re.DOTALL)
+
+[fluentd-agent-conf]
+recipe  = slapos.recipe.template:jinja2
+template = inline:{{ conf }}
+rendered = $${directory:etc}/fluentd-agent.conf
+mode = 0644
+context =
+  key conf fluentd-conf:text

software/fluentd/instance.cfg.in

-[buildout]
-parts =
-  service-fluentd
-
-eggs-directory = {{ buildout['eggs-directory'] }}
-develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
-offline = true
-
-[instance-parameter]
-recipe = slapos.cookbook:slapconfiguration.serialised
-computer = ${slap_connection:computer_id}
-partition = ${slap_connection:partition_id}
-url = ${slap_connection:server_url}
-key = ${slap_connection:key_file}
-cert = ${slap_connection:cert_file}
-
-[directory]
-recipe = slapos.cookbook:mkdirectory
-home = ${buildout:directory}
-etc = ${:home}/etc
-var = ${:home}/var
-script = ${:etc}/run/
-service = ${:etc}/service
-promise = ${:etc}/promise/
-log = ${:var}/log
-bin = ${:home}/bin
-
-[service-fluentd]
-recipe  = slapos.cookbook:wrapper
-wrapper-path    = ${directory:service}/fluentd-service
-command-line    = {{ fluentd_location }}/bin/fluentd
-    -v
-    -c ${fluentd-agent-conf:rendered}
-environment =
-  GEM_PATH={{ fluentd_location }}/lib/ruby/gems/1.8/
-
-[fluentd-agent-conf]
-recipe  = slapos.recipe.template:jinja2
-template = inline:{% raw -%}
-  {{ slapparameter_dict.get('conf_text', '') }}
-  {%- endraw %}
-rendered = ${directory:etc}/fluentd-agent.conf
-mode = 0644
-context =
-  key slapparameter_dict instance-parameter:configuration

software/fluentd/software.cfg

 [buildout]
 extends =
+  buildout.hash.cfg
   ../../component/fluentd/buildout.cfg
   ../../stack/slapos.cfg
+  ../../stack/monitor/buildout.cfg
 
 parts =
   instance-profile
@@ -10,17 +12,20 @@ parts =
 [python]
 part = python3
 
-[instance-profile]
-recipe = slapos.recipe.template:jinja2
-template = ${:_profile_base_location_}/instance.cfg.in
-rendered = ${buildout:directory}/instance.cfg
+[template-base]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
 mode = 0644
-extensions = jinja2.ext.do
-context =
-  section buildout  buildout
-  key fluentd_location fluentd:location
+
+[instance-profile]
+< = template-base
+output = ${buildout:directory}/template.cfg
+
+[template-fluentd]
+< = template-base
+output = ${buildout:directory}/template-fluentd.cfg
 
 [fluentd]
 gems +=
-  fluent-plugin-wendelin==0.4
-  fluent-plugin-bin==0.2
+  fluent-plugin-wendelin==0.5
+  fluent-plugin-bin==0.3

software/fluentd/test/setup.py

@@ -41,6 +41,7 @@ setup(name=name,
       url="https://lab.nexedi.com/nexedi/slapos",
       packages=find_packages(),
       install_requires=[
+          'msgpack',
           'slapos.core',
           'slapos.libnetworkcache',
           'erp5.util',

software/fluentd/test/test.py

@@ -25,10 +25,25 @@
 #
 ##############################################################################
 
+import msgpack
 import os
+import random
+import shutil
+import socket
+import struct
+import subprocess
+import tempfile
+import time
 import six
 
+from six.moves.SimpleHTTPServer import SimpleHTTPRequestHandler
+from six.moves.socketserver import StreamRequestHandler, TCPServer
+
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+from slapos.testing.utils import findFreeTCPPort
+
+FLUENTD_PORT = 24224
+FLUSH_INTERVAL = 1
 
 
 setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
@@ -50,3 +65,218 @@ class FluentdTestCase(SlapOSInstanceTestCase):
 
     for expected_process_name in expected_process_name_list:
       self.assertIn(expected_process_name, process_names)
+
+
+class OneRequestServer(TCPServer):
+
+  address_family = socket.AF_INET6
+  timeout = 1
+
+  def get_first_data(self, flush_interval=1):
+    start = time.time()
+    while(not self.RequestHandlerClass.received_data
+          and time.time() - start < 10*flush_interval):
+      self.handle_request()
+    return self.RequestHandlerClass.received_data
+
+
+class WendelinTutorialTestCase(FluentdTestCase):
+
+  @classmethod
+  def get_configuration(cls):
+    return ''
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'conf_text': cls._conf,}
+
+  @classmethod
+  def measureDict(cls):
+    return {k: v.encode() for k, v in
+      zip((b'pressure', b'humidity', b'temperature'), cls._measurementList)}
+
+  @classmethod
+  def setUpClass(cls):
+    fluentd_dir = os.path.join(cls.computer_partition_root_path,
+                               'software_release', 'parts', 'fluentd')
+    cls._fluentd_bin = os.path.join(fluentd_dir, 'bin', 'fluentd')
+    cls._gem_path = os.path.join(fluentd_dir, 'lib', 'ruby', 'gems')
+    cls._tmp_dir = tempfile.mkdtemp()
+    cls._measurementList = cls.sensor_value_list()
+    cls._conf = cls.get_configuration()
+
+    super(FluentdTestCase, cls).setUpClass()
+
+  @classmethod
+  def sensor_value_list(cls):
+    return [str(value) for value in (round(random.uniform(870, 1084), 2),
+                                     round(random.uniform(0, 100), 2),
+                                     round(random.uniform(-20, 50), 3))]
+
+  def serve(self, port, request_handler_class):
+    server_address = (self._ipv6_address, port)
+    server = OneRequestServer(server_address, request_handler_class)
+
+    data = server.get_first_data(FLUSH_INTERVAL)
+    server.server_close()
+    return data
+
+  @classmethod
+  def tearDownClass(cls):
+    shutil.rmtree(cls._tmp_dir)
+    super(FluentdTestCase, cls).tearDownClass()
+
+  def read_fluentd_conf(self, configuration):
+    conf_path = os.path.join(self._tmp_dir, 'fluentd.conf')
+    with open(conf_path, "w") as conf_file:
+      conf_file.write(configuration)
+    return subprocess.check_output(
+      [self._fluentd_bin, '-c', conf_path, '--dry-run'],
+      env={'GEM_PATH': self._gem_path},
+      universal_newlines=True,
+    )
+
+  def _test_configuration(self, expected_str):
+    self.assertRegexpMatches(
+      self.read_fluentd_conf(self._conf),
+      expected_str,
+    )
+
+
+class FluentdHTTPRequestHandler(StreamRequestHandler):
+
+  received_data = b''
+
+  def handle(self):
+    data = self.rfile.readline().strip()
+    # ignore heartbeats (https://docs.fluentd.org/output/forward#heartbeat_type)
+    if len(data) > 0:
+      FluentdHTTPRequestHandler.received_data = data
+
+
+# see https://wendelin.nexedi.com/wendelin-Learning.Track/wendelin-Tutorial.Setup.Fluentd.on.Sensor
+class SensorConfTestCase(WendelinTutorialTestCase):
+
+  @classmethod
+  def get_configuration(cls):
+    script_path = os.path.join(cls._tmp_dir, "custom_read_bme280.py")
+    with open(script_path, "w") as script:
+      script.write(cls.sensor_script(cls._measurementList))
+    return cls.sensor_conf(script_path)
+
+  @classmethod
+  def sensor_conf(cls, script_path):
+    return '''\
+<source>
+  @type exec
+  tag tag.name
+  command python %s
+  run_interval %ss
+  <parse>
+    keys pressure, humidity, temperature
+  </parse>
+</source>
+<match tag.name>
+  @type forward
+  <server>
+    name myserver1
+    host %s
+  </server>
+  <buffer>
+    flush_mode immediate
+  </buffer>
+</match>''' % (script_path, FLUSH_INTERVAL, cls._ipv6_address)
+
+  @classmethod
+  def sensor_script(cls, measurementList):
+    return '''\
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+print("%s")''' % "\t".join(measurementList)
+
+  def test_configuration(self):
+    self._test_configuration(
+      r'adding forwarding server \'myserver1\' host="%s" port=%s weight=60'
+      % (self._ipv6_address, FLUENTD_PORT)
+    )
+
+  def test_send_data(self):
+    tag, data, header = msgpack.unpackb(
+      self.serve(FLUENTD_PORT, FluentdHTTPRequestHandler),
+      raw=True,
+    )
+    self.assertEqual(b'tag.name', tag)
+    self.assertEqual(self.measureDict(), msgpack.unpackb(data)[-1])
+    self.assertEqual({b'compressed': b'text', b'size': 1}, header)
+
+
+class WendelinHTTPRequestHandler(SimpleHTTPRequestHandler):
+
+  received_data = b''
+
+  def do_POST(self):
+    WendelinHTTPRequestHandler.received_data = self.rfile.read(
+      int(self.headers['Content-Length']))
+    self.send_response(200)
+    self.end_headers()
+
+
+# see https://wendelin.nexedi.com/wendelin-Learning.Track/wendelin-Tutorial.Setup.Fluentd.on.IOTGateway
+class GatewayConfTestCase(WendelinTutorialTestCase):
+
+  @classmethod
+  def gateway_conf(cls, fluentd_port, wendelin_port):
+    return '''\
+<source>
+  @type forward
+  port %s
+  bind %s
+</source>
+<match tag.name>
+  @type wendelin
+  streamtool_uri http://[%s]:%s/erp5/portal_ingestion_policies/default
+  user      foo
+  password  bar
+  <buffer>
+    flush_mode interval
+    @type file
+    path fluentd-buffer-file/
+    flush_interval %ss
+  </buffer>
+</match>''' % (fluentd_port, cls._ipv6_address, cls._ipv6_address,
+               wendelin_port, FLUSH_INTERVAL)
+
+  @classmethod
+  def get_configuration(cls):
+    fluentd_port = findFreeTCPPort(cls._ipv6_address)
+    cls._fluentd_port = fluentd_port
+    wendelin_port = findFreeTCPPort(cls._ipv6_address)
+    cls._wendelin_port = wendelin_port
+    return cls.gateway_conf(fluentd_port, wendelin_port)
+
+  def test_configuration_file(self):
+    self._test_configuration('starting fluentd')
+
+  def test_wendelin_data_forwarding(self):
+    sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+    sock.connect((self._ipv6_address, self._fluentd_port))
+
+    data = [
+      msgpack.ExtType(0, struct.pack('!Q', int(time.time()) << 32)),
+      self.measureDict(),
+    ]
+    sock.sendall(
+      msgpack.packb([
+        b'tag.name',
+        msgpack.packb(data),
+        {b'size': 1, b'compressed': b'text'},
+      ], use_bin_type=False),
+    )
+    sock.close()
+
+    self.assertEqual(
+      data,
+      msgpack.unpackb(
+        self.serve(self._wendelin_port, WendelinHTTPRequestHandler)),
+    )
\ No newline at end of file

software/galene/software.cfg

@@ -10,6 +10,9 @@ parts =
   eggs
   instance-cfg
 
+[python]
+part = python3
+
 # eggs for instance.cfg
 [eggs]
 recipe  = zc.recipe.egg
@@ -22,7 +25,7 @@ eggs    =
 <= go-git-package
 go.importpath = lab.nexedi.com/nexedi/galene
 repository = https://lab.nexedi.com/nexedi/galene.git
-revision = galene-0.3.3
+revision = galene-0.4
 
 [gowork]
 install =

software/gitlab/buildout.hash.cfg

@@ -54,7 +54,7 @@ md5sum = 0f1ec4077dab586cc003ae13f689eda2
 
 [instance-gitlab.cfg.in]
 _update_hash_filename_ = instance-gitlab.cfg.in
-md5sum = 6b34d4b96ae0067977fa509046d71231
+md5sum = f099d01baefe41c8f0944c2437b30881
 
 [instance-gitlab-export.cfg.in]
 _update_hash_filename_ = instance-gitlab-export.cfg.in

software/gitlab/instance-gitlab.cfg.in

@@ -430,7 +430,7 @@ tune-command =
 # [promise-<something>] to check <something> by url
 [promise-byurl]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name   = ${:_buildout_section_name_}.py
 config-http-code   = 200
 
@@ -472,7 +472,7 @@ depend  =
 
 [promise-postgresql]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = promise-postgresql.py
 config-command =
     {{ postgresql_location }}/bin/psql \
@@ -515,7 +515,7 @@ depend  =
 
 [promise-redis]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = promise-redis.py
 config-command = ${service-redis:promise_wrapper}
 
@@ -769,7 +769,7 @@ depend  =
 # XXX this depends on gitlab-workhorse being up
 #     (nginx is configured to proxy all requests to gitlab-workhorse)
 config-url     = ${backend-info:url}/users/sign_in
-module = check_url_available
+promise = check_url_available
 
 [logrotate-entry-nginx]
 <= logrotate-entry-base

software/headless-chromium/buildout.hash.cfg

@@ -4,7 +4,7 @@ md5sum = 5dfeeb5eca125dcaa5f9e537f941dd41
 
 [instance-headless-chromium]
 _update_hash_filename_ = instance-headless-chromium.cfg.in
-md5sum = fad685238b26ca20537c12ce7432e7e7
+md5sum = 7392d20e48dbc599eb8e9d02e8095bbf
 
 [template-nginx-conf]
 _update_hash_filename_ = templates/nginx.conf.in

software/headless-chromium/instance-headless-chromium.cfg.in

@@ -174,7 +174,7 @@ monitor-httpd-port = {{ parameter_dict['monitor-httpd-port'] }}
 # queried with the correct credentials.
 [frontend-ok-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = headless-chromium-frontend-ok.py
 url = ${remote-debugging-frontend:connection-secure_access}
 config-url = ${:url}
@@ -185,7 +185,7 @@ config-password = ${frontend-instance-password:passwd}
 # when queried with no credentials.
 [frontend-secure-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = headless-chromium-frontend-secure.py
 url = ${remote-debugging-frontend:connection-secure_access}
 config-url = ${:url}

software/helloworld/buildout.hash.cfg

@@ -14,4 +14,4 @@
 # not need these here).
 [instance-profile]
 filename = instance.cfg.in
-md5sum = c771dee1ef9aedad7c6ebf9418afe08e
+md5sum = 483b76d8e6bf72d72a38a3f7bf66fe08

software/helloworld/instance.cfg.in

@@ -87,7 +87,7 @@ wrapper-path = ${directory:service}/helloweb-${:kind}
 # promise, that checks that helloweb service is alive
 [helloweb-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = helloweb-${:kind}.py
 
 {# macro to instantiate service of `kind` to listen on `port` #}

software/html5as/buildout.hash.cfg

@@ -21,7 +21,7 @@ md5sum = 9e486efe4ab1aba8cb72b04f6c6da8ad
 
 [instance_html5as]
 _update_hash_filename_ = instance_html5as.cfg.in
-md5sum = 283440057c659bde2ae7fcc2c4c5b781
+md5sum = f86b2f37c0acd21ca1f41d90c5477d75
 
 [template_nginx_conf]
 _update_hash_filename_ = templates/nginx_conf.in

software/html5as/instance_html5as.cfg.in

@@ -185,7 +185,7 @@ context =
 # Port Listening checking promise
 [port-listening-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = nginx-port-listening.py
 config-host = ${html5as:ip}
 config-port = ${html5as:port}
@@ -235,7 +235,7 @@ return = domain secure_access
 # Add a promise to make sure the cdn is properly configured
 [html5as-frontend-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = html5as-http-frontend.py
 url = ${html5as-frontend:connection-secure_access}
 config-url = ${:url}

software/htmlvalidatorserver/buildout.hash.cfg

@@ -22,7 +22,7 @@ md5sum = 87781e6bcb523bb8434888d5f984f36c
 
 [template-validator]
 filename = instance-validator.cfg.in
-md5sum = 9d12472bb2e337d3cc18f2cc6f235425
+md5sum = dc8b8d03b0af9cd32398d1fe86267bb7
 
 [template]
 filename = instance.cfg.in

software/htmlvalidatorserver/instance-validator.cfg.in

@@ -55,7 +55,7 @@ scheme = https
 
 [tomcat-listen-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = tomcat_listen.py
 config-host = $${tomcat-configuration:ip}
 config-port = $${tomcat-configuration:port}

software/hugo/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [template-cfg]
 filename = instance.cfg.in
-md5sum = 9653104b2217dc26b23f9c1b997124ca
+md5sum = 6f5acc546a7e9ad502d5fe586a3c3072
 
 [template_nginx_conf]
 _update_hash_filename_ = templates/nginx_conf.in

software/hugo/instance.cfg.in

@@ -113,6 +113,7 @@ template =
   . ${hugo:go-environment}
   cd ${basedirectory:data}/${slap-configuration:configuration.site}
   if [ -d "public" ]; then rm -Rf public; fi
+  export TMPDIR=${hugo:path-tmp}
   hugo && hugo server --bind=${hugo:ip} --port=${hugo:hugo-port} --baseURL=${hugo-frontend:connection-secure_access} --appendPort=false  
 
 [hugo-server-service]
@@ -124,7 +125,7 @@ hash-files =
 
 [hugo-port-listening-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = hugo-port-listening.py
 config-host = ${hugo:ip}
 config-port = ${hugo:hugo-port}
@@ -144,7 +145,7 @@ return = domain secure_access
 
 [hugo-frontend-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = hugo-http-frontend.py
 url = ${hugo-frontend:connection-secure_access}
 config-url = ${:url}
@@ -212,7 +213,7 @@ template =
 
 [nginx-port-listening-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = nginx-port-listening.py
 config-host = ${hugo:ip}
 config-port = ${hugo:nginx-port}
@@ -238,7 +239,7 @@ return = domain secure_access
 
 [nginx-frontend-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = nginx-http-frontend.py
 url = ${nginx-frontend:connection-secure_access}
 config-url = ${:url}

software/jscrawler/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = 6c17361a49cfc47564063b867aab6e8c
 
 [template-jscrawler]
 filename = instance-jscrawler.cfg.jinja2.in
-md5sum = f61e0507717447e47c76a2b2712f17f4
+md5sum = 33bfddbc23fa794ab97770ef7776b390
 
 [template-jscrawler-builder]
 filename = template-jscrawler.builder.sh.in

software/jscrawler/instance-jscrawler.cfg.jinja2.in

@@ -47,7 +47,7 @@ return = secure_access domain
 
 [jscrawler-frontend-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = jscrawler_frontend.py
 config-url = ${request-jscrawler-frontend:connection-secure_access}
 
@@ -59,7 +59,7 @@ log = ${httpd-wrapper:log-file}
 
 [httpd-listen-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = httpd-listen.py
 config-host = ${httpd-wrapper:host}
 config-port = ${httpd-wrapper:port}

software/jstestnode/buildout.hash.cfg

@@ -27,4 +27,4 @@ md5sum = 98faa5ad8cfb23a11d97a459078a1d05
 
 [template-runTestSuite]
 filename = runTestSuite.in
-md5sum = 3e2fbde4b6a1cc202b5fd2a0f14413fd
+md5sum = 9d82f8afa0de8ec2b9c4f32e058fb2bc

software/jstestnode/runTestSuite.in

@@ -69,7 +69,7 @@ def main():
     target = test_runner.get('target', 'firefox')
     if target == 'node':
       # Execute NodeJS tests
-      result_string = check_output(['${nodejs-output:node}', '${jio-repository.git:location}/test/node.js'],
+      result_string = check_output(['${nodejs:location}/bin/node', '${jio-repository.git:location}/test/node.js'],
                                    cwd='${jio-repository.git:location}',
                                    env={'CI': 'true'})
       result_dict = json.loads(result_string)

software/jstestnode/software.cfg

@@ -30,10 +30,7 @@ parts =
   instance
 
 [nodejs]
-<= nodejs-8.6.0
-
-[nodejs-output]
-<= nodejs-8.6.0-output
+<= nodejs-14.16.0
 
 [eggs]
 recipe = zc.recipe.egg
@@ -54,7 +51,11 @@ develop = true
 [renderjs-install]
 recipe = plone.recipe.command
 stop-on-error = true
-command = cd ${renderjs-repository.git:location} && PATH=${git:location}/bin/:${nodejs:location}/bin/:${bzip2:location}/bin/:$PATH ${nodejs-output:npm} install . && PATH=${git:location}/bin/:${nodejs:location}/bin/:${bzip2:location}/bin/:$PATH ./node_modules/grunt-cli/bin/grunt
+command = set -e
+  cd ${renderjs-repository.git:location}
+  PATH=${git:location}/bin:${nodejs:location}/bin:${bzip2:location}/bin:$PATH
+  npm install .
+  ./node_modules/grunt-cli/bin/grunt
 update-command = ${:command}
 
 [jio-repository.git]
@@ -67,7 +68,11 @@ develop = true
 [jio-install]
 recipe = plone.recipe.command
 stop-on-error = true
-command = cd ${jio-repository.git:location} && PATH=${git:location}/bin/:${nodejs:location}/bin/:$PATH ${nodejs:location}/bin/npm install jslint@0.9.2 jison@0.4.16 git://github.com/qunitjs/node-qunit.git#v0.9.3 sinon@1.7.3 && PATH=${curl:location}/bin/:${nodejs:location}/bin/:$PATH make
+command = set -e
+  cd ${jio-repository.git:location}
+  PATH=${git:location}/bin:${nodejs:location}/bin:$PATH
+  ${nodejs:location}/bin/npm install jslint@0.9.2 jison@0.4.16 git://github.com/qunitjs/node-qunit.git#v0.9.3 sinon@1.7.3
+  make
 update-command = ${:command}
 
 [rsvp-repository.git]
@@ -80,7 +85,10 @@ develop = true
 [rsvp-install]
 recipe = plone.recipe.command
 stop-on-error = true
-command = cd ${rsvp-repository.git:location} && PATH=${git:location}/bin/:${nodejs:location}/bin/:$PATH ${nodejs:location}/bin/npm install .
+command = set -e
+  cd ${rsvp-repository.git:location}
+  PATH=${git:location}/bin:${nodejs:location}/bin:$PATH
+  npm install .
 update-command = ${:command}
 
 [uritemplate-repository.git]
@@ -93,7 +101,10 @@ develop = true
 [uritemplate-install]
 recipe = plone.recipe.command
 stop-on-error = true
-command = cd ${uritemplate-repository.git:location} && PATH=${git:location}/bin/:${nodejs:location}/bin/:$PATH ${nodejs:location}/bin/npm install .
+command = set -e
+  cd ${uritemplate-repository.git:location}
+  PATH=${git:location}/bin:${nodejs:location}/bin:$PATH
+  npm install .
 update-command = ${:command}
 
 [macro-template]

software/jupyter/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = de37ec3d4adb0be4c67bcc7397f27c91
 
 [instance-jupyter]
 filename = instance-jupyter.cfg.in
-md5sum = cbc90e517ae3680ab8bef04c6f503af5
+md5sum = 95e3da48abdd257fb9d5dbdf14ea87b9
 
 [jupyter-notebook-config]
 filename = jupyter_notebook_config.py.jinja

software/jupyter/instance-jupyter.cfg.in

@@ -146,7 +146,7 @@ sla-instance_guid = ${slap-parameter:frontend-instance-guid}
 
 [frontend-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = frontend_promise.py
 config-url = ${publish-connection-parameter:url}
 
@@ -160,7 +160,7 @@ sla-instance_guid = ${slap-parameter:frontend-additional-instance-guid}
 
 [frontend-additional-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = frontend_additional_promise.py
 config-url = ${publish-connection-parameter:url-additional}
 {% endif %}

software/kvm/README.rst

@@ -43,12 +43,12 @@ KVM instance parameters:
 - nbd2-port (default: 1024)
 - nbd2-host
 
-- ram-size (default: 1024)
-- disk-size = (default: 10)
+- ram-size (default: 4096)
+- disk-size = (default: 40)
 - disk-type (default: virtio)
       Disk size and Disk type are used if no virtual hard drive is specified.
 
-- cpu-count (default: 1)
+- cpu-count (default: 2)
 - cpu-options
     cpu-option is a string: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]
 - numa

software/kvm/buildout.hash.cfg

@@ -15,23 +15,23 @@
 
 [template]
 filename = instance.cfg.in
-md5sum = 0d34ff81779115bf899f7bc752877b70
+md5sum = f2b0f1ed27148504f220e06eaceff935
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = d0f96be4e80b96e6ac33f6d474767b13
+md5sum = 17c58f74d1ed4cb7dce11bf9af71dd33
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in
-md5sum = 59b92e1300aad4e9b116c532caf7d042
+md5sum = fcb35c32ef985e3d69a7914711675dcc
 
 [template-kvm-resilient]
 filename = instance-kvm-resilient.cfg.jinja2
-md5sum = 7de5756f59ef7d823cd8ed33e6d15230
+md5sum = a0fd4911401cbbda74323e8d1c7b18ad
 
 [template-kvm-import]
 filename = instance-kvm-import.cfg.jinja2.in
-md5sum = 7b15fdc6f19b1f44ff5a56586102ffe2
+md5sum = 0415353c961ece593dd5d6457dab5200
 
 [template-kvm-import-script]
 filename = template/kvm-import.sh.jinja2
@@ -47,7 +47,7 @@ md5sum = b617d64de73de1eed518185f310bbc82
 
 [template-nbd]
 filename = instance-nbd.cfg.jinja2
-md5sum = 259e06f289f68297e0609e4ab1af8e86
+md5sum = 4bcb07c1a9223e2d956651aa25d23654
 
 [template-ansible-promise]
 filename = template/ansible-promise.in
@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
 
 [template-kvm-run]
 filename = template/template-kvm-run.in
-md5sum = a502782244d1be536b732ebb40725f47
+md5sum = f840e8b7af83982525f66d7ec12b7085
 
 [template-kvm-controller]
 filename = template/kvm-controller-run.in
@@ -75,7 +75,7 @@ md5sum = fb330a796fadb6cd5c85217f80a42af3
 
 [template-httpd]
 filename = instance-kvm-http.cfg.in
-md5sum = f4bcde62e008c2da9c65617ba7f73f08
+md5sum = d57764bb7135037b4d21543b2f56ce1d
 
 [image-download-controller]
 _update_hash_filename_ = template/image-download-controller.py

software/kvm/instance-kvm-cluster-input-schema.json

@@ -225,7 +225,7 @@
               "title": "RAM size",
               "description": "RAM size, in MB.",
               "type": "integer",
-              "default": 1024,
+              "default": 4096,
               "minimum": 1024,
               "multipleOf": 512
             },
@@ -233,6 +233,7 @@
               "title": "Maximum RAM size, in MB",
               "description": "Define the maximum size of the memory. The size is in MB and should be a multiple of 512. Defaults to ram-size + 512",
               "type": "integer",
+              "default": 4608,
               "minimum": 1024,
               "multipleOf": 512
             },
@@ -254,7 +255,7 @@
               "title": "Disk size",
               "description": "Disk size, in GB.",
               "type": "integer",
-              "default": 10,
+              "default": 40,
               "minimum": 1
             },
             "disk-format": {
@@ -313,6 +314,7 @@
               "title": "CPU count",
               "description": "Number of CPU cores.",
               "type": "integer",
+              "default": 2,
               "minimum": 1
             },
             "cpu-max-count": {
@@ -474,9 +476,15 @@
               "oneOf": [
                 {
                   "const": [
-                    "https://shacache.nxdcdn.com/0a6aee1d9aafc1ed095105c052f9fdd65ed00ea9274188c9cd0072c8e6838ab40e246d45a1e6956d74ef1b04a1fc042151762f25412e9ff0cbf49418eef7992e#a3ebc76aec372808ad80000108a2593a"
+                    "https://shacache.nxdcdn.com/02257c3ec27e45d9f022c181a69b59da67e5c72871cdb4f9a69db323a1fad58093f2e69702d29aa98f5f65e920e0b970d816475a5a936e1f3bf33832257b7e92#b710c178eb434d79ce40ce703d30a5f0"
+                  ],
+                  "title": "Debian Bullseye 11.1 netinst x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/d82b0510fd919c2a851ee93ea0f7ad6779bfa597297a5c7463b63746799f001321ec4c9b8ba6cfe20248dd2da28100ad3b78e74489a8c0c573238f226a509a9d#9d7b9cc850464d60ac174787c53e8f3f"
                   ],
-                  "title": "Debian Buster 10.5 netinst x86_64"
+                  "title": "Debian Buster 10.11 netinst x86_64"
                 },
                 {
                   "const": [

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -77,14 +77,14 @@ config-authorized-key = {{ dumps(slapparameter_dict.get('authorized-keys') | joi
 {% endif -%}
 config-nbd-port = {{ dumps(kvm_parameter_dict.get('nbd-port', 1024)) }}
 config-nbd2-port = {{ dumps(kvm_parameter_dict.get('nbd-port2', 1024)) }}
-config-ram-size = {{ dumps(kvm_parameter_dict.get('ram-size', 1024)) }}
-config-ram-max-size = {{ dumps(kvm_parameter_dict.get('ram-max-size', int(kvm_parameter_dict.get('ram-size', 1024)) + 512)) }}
+config-ram-size = {{ dumps(kvm_parameter_dict.get('ram-size', 4096)) }}
+config-ram-max-size = {{ dumps(kvm_parameter_dict.get('ram-max-size', int(kvm_parameter_dict.get('ram-size', 4096)) + 512)) }}
 config-enable-device-hotplug = {{ dumps(kvm_parameter_dict.get('enable-device-hotplug', False)) }}
 config-ram-hotplug-slot-size = {{ dumps(kvm_parameter_dict.get('ram-hotplug-slot-size', 512)) }}
-config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 10)) }}
+config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 40)) }}
 config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }}
 config-disk-format = {{ dumps(kvm_parameter_dict.get('disk-format', 'qcow2')) }}
-config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 1)) }}
+config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 2)) }}
 config-cpu-max-count = {{ dumps(kvm_parameter_dict.get('cpu-max-count', int(kvm_parameter_dict.get('cpu-count', 1)) + 1)) }}
 {{ setconfig('numa', kvm_parameter_dict.get('numa', '')) }}
 {{ setconfig('machine-options', kvm_parameter_dict.get('machine-options', '')) }}

software/kvm/instance-kvm-http.cfg.in

@@ -65,7 +65,7 @@ stop-on-error = true
 
 [httpd-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = apache-httpd.py
 config-host = ${apache-conf:ip}
 config-port = ${apache-conf:port}

software/kvm/instance-kvm-import.cfg.jinja2.in

@@ -91,6 +91,6 @@ mode = 700
 [kvm-disk-image-corruption-promise]
 # Check that disk image is not corrupted
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = kvm-disk-image-corruption.py
 config-command = ${kvm-disk-image-corruption-bin:output}

software/kvm/instance-kvm-input-schema.json

@@ -13,7 +13,7 @@
       "title": "RAM size",
       "description": "RAM size, in MB.",
       "type": "integer",
-      "default": 1024,
+      "default": 4096,
       "minimum": 1024,
       "multipleOf": 512
     },
@@ -21,6 +21,7 @@
       "title": "Maximum RAM size, in MB",
       "description": "Define the maximum size of the memory. The size is in MB and should be a multiple of 512. Defaults to ram-size + 512",
       "type": "integer",
+      "default": 4608,
       "minimum": 1024,
       "multipleOf": 512
     },
@@ -42,7 +43,7 @@
       "title": "Disk size",
       "description": "Disk size, in GB.",
       "type": "integer",
-      "default": 10,
+      "default": 40,
       "minimum": 1
     },
     "disk-type": {
@@ -101,7 +102,8 @@
       "title": "CPU count",
       "description": "Number of CPU cores.",
       "type": "integer",
-      "minimum": 1
+      "minimum": 1,
+      "default": 2
     },
     "cpu-max-count": {
       "title": "Maximum CPU amount",

software/kvm/instance-kvm-resilient.cfg.jinja2

@@ -102,13 +102,13 @@ mode = 700
 [kvm-frontend-url-promise]
 # Check that url parameter is complete
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = kvm-frontend-url.py
 config-command = ${kvm-frontend-url-bin:output}
 
 [kvm-backend-url-promise]
 # Check that backend url is reachable
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = frontend_promise.py
 config-url = ${publish-connection-information:url}

software/kvm/instance-kvm.cfg.jinja2

@@ -22,8 +22,8 @@
 {% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %}
 {% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %}
 {% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url') -%}
-{% set cpu_max_count = dumps(slapparameter_dict.get('cpu-max-count', int(slapparameter_dict.get('cpu-count', 1)) + 1)) %}
-{% set ram_max_size = dumps(slapparameter_dict.get('ram-max-size', int(slapparameter_dict.get('ram-size', 1024)) + 512)) %}
+{% set cpu_max_count = dumps(slapparameter_dict.get('cpu-max-count', int(slapparameter_dict.get('cpu-count', 2)) + 1)) %}
+{% set ram_max_size = dumps(slapparameter_dict.get('ram-max-size', int(slapparameter_dict.get('ram-size', 4096)) + 512)) %}
 {% set extends_list = [] -%}
 {% set part_list = [] -%}
 
@@ -91,7 +91,7 @@ bytes = 8
 ## boot-image-url-select support BEGIN
 [empty-file-state-base-select-promise]
 <= monitor-promise-base
-module = check_file_state
+promise = check_file_state
 name = ${:_buildout_section_name_}.py
 config-state = empty
 # It's very hard to put the username and password correctly, after schema://
@@ -190,7 +190,7 @@ config-filename = ${boot-image-url-select-download-wrapper:error-state-file}
 ## boot-image-url-list support BEGIN
 [empty-file-state-base-list-promise]
 <= monitor-promise-base
-module = check_file_state
+promise = check_file_state
 name = ${:_buildout_section_name_}.py
 config-state = empty
 # It's very hard to put the username and password correctly, after schema://
@@ -289,7 +289,7 @@ config-filename = ${boot-image-url-list-download-wrapper:error-state-file}
 ## virtual-hard-drive-url support BEGIN
 [empty-file-state-base-virtual-promise]
 <= monitor-promise-base
-module = check_file_state
+promise = check_file_state
 name = ${:_buildout_section_name_}.py
 config-state = empty
 # It's very hard to put the username and password correctly, after schema://
@@ -441,8 +441,8 @@ socket-path = ${kvm-controller-parameter-dict:socket-path}
 smp-max-count = {{ cpu_max_count }}
 ram-max-size = {{ ram_max_size }}
 {%- if enable_device_hotplug %}
-init-ram-size = 1024
-init-smp-count = 1
+init-ram-size = 4096
+init-smp-count = 2
 {%- else %}
 init-ram-size = ${kvm-controller-parameter-dict:ram-size}
 init-smp-count = ${kvm-controller-parameter-dict:cpu-count}
@@ -586,7 +586,7 @@ command-line = ${kvm-controller:rendered}
 
 [kvm-vnc-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = vnc_promise.py
 config-host = ${kvm-parameter-dict:vnc-ip}
 config-port = ${kvm-parameter-dict:vnc-port}
@@ -614,7 +614,7 @@ mode = 700
 [kvm-disk-image-corruption-promise]
 # Check that disk image is not corrupted
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = kvm-disk-image-corruption.py
 config-command = ${kvm-disk-image-corruption-bin:output}
 
@@ -643,7 +643,7 @@ context =
 
 [kvm-started-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = qemu-virtual-machine-is-ready.py
 config-command = ${kvm-started-bin:rendered}
 
@@ -708,7 +708,7 @@ wrapper = ${directory:bin}/websockify
 
 [novnc-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = novnc_promise.py
 config-host = ${novnc-instance:ip}
 config-port = ${novnc-instance:port}
@@ -764,7 +764,7 @@ sla-instance_guid = ${slap-parameter:frontend-instance-guid}
 
 [frontend-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = frontend_promise.py
 config-url = ${publish-connection-information:url}
 
@@ -778,7 +778,7 @@ sla-instance_guid = ${slap-parameter:frontend-additional-instance-guid}
 
 [frontend-additional-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = frontend_additional_promise.py
 config-url = ${publish-connection-information:url-additional}
 {% endif %}
@@ -801,7 +801,7 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [httpd-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = httpd.py
 config-host = ${httpd:host}
 config-port = ${httpd:port}
@@ -1029,7 +1029,7 @@ context =
 
 [ansible-vm-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = ansible_{{ name }}.py
 config-command = ${ansible-vm-bin:rendered}
 
@@ -1068,14 +1068,14 @@ nbd2-host =
 boot-image-url-list =
 
 enable-device-hotplug = False
-ram-size = 1024
+ram-size = 4096
 ram-hotplug-slot-size = 512
-disk-size = 10
+disk-size = 40
 disk-type = virtio
 disk-format = qcow2
 disk-device-path =
 
-cpu-count = 1
+cpu-count = 2
 disk-cache =
 disk-aio =
 auto-ballooning = True

software/kvm/instance-nbd.cfg.jinja2

@@ -43,7 +43,7 @@ rendered = ${rootdirectory:bin}/check-nbd-running.sh
 
 [nbd-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = nbd_promise.py
 config-command = ${nbd-checker-bin:rendered}
 
@@ -65,7 +65,7 @@ key = ${gen-passwd:passwd}
 
 [onetimeupload-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = onetimeupload_promise.py
 config-host = ${onetimeupload-instance:ip}
 config-port = ${onetimeupload-instance:port}

software/kvm/instance.cfg.in

@@ -82,7 +82,7 @@ extra-context =
   raw dash_executable_location ${dash:location}/bin/dash
   raw dnsresolver_executable ${buildout:bin-directory}/dnsresolver
   raw dcron_executable_location ${dcron:location}/sbin/crond
-  raw debian_amd64_netinst_location ${debian-amd64-netinst.iso:location}/${debian-amd64-netinst.iso:filename}
+  raw debian_amd64_netinst_location ${debian-amd64-bullseye-netinst.iso:target}
   raw whitelist_domains_default ${whitelist-domains-default:location}/${whitelist-domains-default:filename}
   raw whitelist_firewall_download_controller ${whitelist-firewall-download-controller:target}
   raw image_download_controller ${image-download-controller:target}

software/kvm/software.cfg

@@ -13,7 +13,6 @@ extends =
   ../../component/numpy/buildout.cfg
   ../../component/gzip/buildout.cfg
   ../../stack/slapos.cfg
-  ../../component/nodejs/buildout.cfg
   ../../stack/resilient/buildout.cfg
   buildout.hash.cfg
 
@@ -57,36 +56,6 @@ eggs =
 scripts =
   websockify
 
-[http-proxy]
-# https://github.com/nodejitsu/node-http-proxy
-recipe = slapos.recipe.build:download-unpacked
-#XXX-Cedric : use upstream when merged
-url = https://nodeload.github.com/desaintmartin/node-http-proxy/zipball/20120621
-md5sum = 20204d0b29c2cef26e1c91e99eedca6b
-
-[proxy-by-url]
-# https://github.com/dominictarr/proxy-by-url
-recipe = slapos.recipe.build:download-unpacked
-#XXX-Cedric : use upstream when merged
-url = https://nodeload.github.com/desaintmartin/proxy-by-url/zipball/20120621
-md5sum = c2609948aa708581f93b981b23880314
-
-[npm-modules]
-recipe = plone.recipe.command
-destination = ${buildout:parts-directory}/${:_buildout_section_name_}
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-command =
-  export HOME=${:location};
-  rm -fr ${:destination} &&
-  mkdir -p ${:destination} &&
-  cd ${:destination} &&
-  ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install colors@0.6.0-1 &&
-  ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install socket.io@0.8.7 &&
-  ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install socket.io-client@0.8.7 &&
-  ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install optimist@0.3.1 &&
-  ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install pkginfo@0.2.3
-
-
 # Create all templates that will be used to deploy instances
 [download-base]
 recipe = hexagonit.recipe.download

software/kvm/template/template-kvm-run.in

@@ -277,6 +277,8 @@ kvm_argument_list = [qemu_path,
   '-pidfile', pid_file_path, '-msg', 'timestamp=on',
   '-D', logfile,
   '-nodefaults',
+  # switch to tablet mode for the mouse to have it synced with a client, see https://wiki.gentoo.org/wiki/QEMU/Options#USB
+  '-usbdevice', 'tablet',
 ]
 for disk_info in disk_info_list:
   kvm_argument_list += (

software/kvm/test/test.py

@@ -31,10 +31,12 @@ import os
 import glob
 import hashlib
 import psutil
+import re
 import requests
 import six
 import slapos.util
 import sqlite3
+import stat
 from six.moves.urllib.parse import parse_qs, urlparse
 import unittest
 import subprocess
@@ -228,7 +230,7 @@ class TestMemoryManagement(InstanceTestCase, KvmMixin):
   def test(self):
     kvm_pid_1, info_list = self.getKvmProcessInfo(['-smp', '-m'])
     self.assertEqual(
-      ['1,maxcpus=2', '1024M,slots=128,maxmem=1536M'],
+      ['2,maxcpus=3', '4096M,slots=128,maxmem=4608M'],
       info_list
     )
     self.rerequestInstance({
@@ -262,7 +264,7 @@ class TestMemoryManagement(InstanceTestCase, KvmMixin):
 
     kvm_pid_1, info_list = self.getKvmProcessInfo(['-smp', '-m'])
     self.assertEqual(
-      ['1,maxcpus=2', '1024M,slots=128,maxmem=1536M'],
+      ['2,maxcpus=3', '4096M,slots=128,maxmem=4608M'],
       info_list
     )
     self.assertEqual(
@@ -273,15 +275,15 @@ class TestMemoryManagement(InstanceTestCase, KvmMixin):
     parameter_dict = {
       'enable-device-hotplug': 'true',
       # to avoid restarts the max RAM and CPU has to be static
-      'ram-max-size': '2048',
-      'cpu-max-count': '4',
+      'ram-max-size': '8192',
+      'cpu-max-count': '6',
     }
     self.rerequestInstance(parameter_dict)
     self.slap.waitForInstance(max_retry=2)
     kvm_pid_2, info_list = self.getKvmProcessInfo(['-smp', '-m'])
 
     self.assertEqual(
-      ['1,maxcpus=4', '1024M,slots=128,maxmem=2048M'],
+      ['2,maxcpus=6', '4096M,slots=128,maxmem=8192M'],
       info_list
     )
     self.assertEqual(
@@ -290,21 +292,21 @@ class TestMemoryManagement(InstanceTestCase, KvmMixin):
     )
     self.assertNotEqual(kvm_pid_1, kvm_pid_2, "Unexpected: KVM not restarted")
     parameter_dict.update(**{
-      'ram-size': '1536',
-      'cpu-count': '2'
+      'ram-size': '5120',
+      'cpu-count': '4'
     })
     self.rerequestInstance(parameter_dict)
     self.slap.waitForInstance(max_retry=10)
     kvm_pid_3, info_list = self.getKvmProcessInfo(['-smp', '-m'])
 
     self.assertEqual(
-      ['1,maxcpus=4', '1024M,slots=128,maxmem=2048M'],
+      ['2,maxcpus=6', '4096M,slots=128,maxmem=8192M'],
       info_list
     )
     self.assertEqual(kvm_pid_2, kvm_pid_3, "Unexpected: KVM restarted")
     self.assertEqual(
       getHotpluggedCpuRamValue(),
-      {'cpu_count': 1, 'ram_mb': 512}
+      {'cpu_count': 2, 'ram_mb': 1024}
     )
 
 
@@ -812,6 +814,29 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
     self.stopImageHttpServer()
     super(InstanceTestCase, self).tearDown()
 
+  def getRunningImageList(self, kvm_instance_partition,
+      _match_cdrom=re.compile('file=(.+),media=cdrom$').match,
+      _sub_iso=re.compile(r'(/debian)(-[^-/]+)(-[^/]+-netinst\.iso)$').sub,
+      ):
+    with self.slap.instance_supervisor_rpc as instance_supervisor:
+      kvm_pid = next(q for q in instance_supervisor.getAllProcessInfo()
+                       if 'kvm-' in q['name'])['pid']
+    sub_shared = re.compile(r'^%s/[^/]+/[0-9a-f]{32}/'
+                            % re.escape(self.slap.shared_directory)).sub
+    image_list = []
+    for entry in psutil.Process(kvm_pid).cmdline():
+      m = _match_cdrom(entry)
+      if m:
+        path = m.group(1)
+        st = os.stat(path)
+        if stat.S_ISREG(st.st_mode) and st.st_size:
+          image_list.append(
+            _sub_iso(r'\1-${ver}\3',
+            sub_shared(r'${shared}/',
+            path.replace(kvm_instance_partition, '${inst}')
+          )))
+    return image_list
+
   def test(self):
     partition_parameter_kw = {
       self.key: self.test_input % (
@@ -843,23 +868,6 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
     self.assertTrue(os.path.islink(image2_link))
     self.assertEqual(os.readlink(image2_link), image2)
 
-    def getRunningImageList():
-      running_image_list = []
-      with self.slap.instance_supervisor_rpc as instance_supervisor:
-        kvm_pid = [q for q in instance_supervisor.getAllProcessInfo()
-                   if 'kvm-' in q['name']][0]['pid']
-        kvm_process = psutil.Process(kvm_pid)
-        software_root = '/'.join([
-          self.slap.software_directory,
-          hashlib.md5(self.getSoftwareURL().encode('utf-8')).hexdigest()])
-        for entry in kvm_process.cmdline():
-          if entry.startswith('file') and 'media=cdrom' in entry:
-            # do cleanups
-            entry = entry.replace(software_root, '')
-            entry = entry.replace(kvm_instance_partition, '')
-            running_image_list.append(entry)
-      return running_image_list
-
     # mimic the requirement: restart the instance by requesting it stopped and
     # then started started, like user have to do it
     self.rerequestInstance(partition_parameter_kw, state='stopped')
@@ -869,12 +877,11 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
 
     self.assertEqual(
       [
-        'file=/srv/%s/image_001,media=cdrom' % (self.image_directory,),
-        'file=/srv/%s/image_002,media=cdrom' % (self.image_directory,),
-        'file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
-        'media=cdrom'
+        '${inst}/srv/%s/image_001' % self.image_directory,
+        '${inst}/srv/%s/image_002' % self.image_directory,
+        '${shared}/debian-${ver}-amd64-netinst.iso',
       ],
-      getRunningImageList()
+      self.getRunningImageList(kvm_instance_partition)
     )
 
     # cleanup of images works, also asserts that configuration changes are
@@ -896,9 +903,8 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
 
     # again only default image is available in the running process
     self.assertEqual(
-      ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
-       'media=cdrom'],
-      getRunningImageList()
+      ['${shared}/debian-${ver}-amd64-netinst.iso'],
+      self.getRunningImageList(kvm_instance_partition)
     )
 
   def assertPromiseFails(self, promise):
@@ -1032,23 +1038,6 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
     kvm_instance_partition = os.path.join(
       self.slap.instance_directory, self.kvm_instance_partition_reference)
 
-    def getRunningImageList():
-      running_image_list = []
-      with self.slap.instance_supervisor_rpc as instance_supervisor:
-        kvm_pid = [q for q in instance_supervisor.getAllProcessInfo()
-                   if 'kvm-' in q['name']][0]['pid']
-        kvm_process = psutil.Process(kvm_pid)
-        software_root = '/'.join([
-          self.slap.software_directory,
-          hashlib.md5(self.getSoftwareURL().encode('utf-8')).hexdigest()])
-        for entry in kvm_process.cmdline():
-          if entry.startswith('file') and 'media=cdrom' in entry:
-            # do cleanups
-            entry = entry.replace(software_root, '')
-            entry = entry.replace(kvm_instance_partition, '')
-            running_image_list.append(entry)
-      return running_image_list
-
     # mimic the requirement: restart the instance by requesting it stopped and
     # then started started, like user have to do it
     self.rerequestInstance(partition_parameter_kw, state='stopped')
@@ -1058,12 +1047,11 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
 
     self.assertEqual(
       [
-        'file=/srv/boot-image-url-select-repository/image_001,media=cdrom',
-        'file=/srv/boot-image-url-list-repository/image_001,media=cdrom',
-        'file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
-        'media=cdrom'
+        '${inst}/srv/boot-image-url-select-repository/image_001',
+        '${inst}/srv/boot-image-url-list-repository/image_001',
+        '${shared}/debian-${ver}-amd64-netinst.iso',
       ],
-      getRunningImageList()
+      self.getRunningImageList(kvm_instance_partition)
     )
 
     # cleanup of images works, also asserts that configuration changes are
@@ -1100,9 +1088,8 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
 
     # again only default image is available in the running process
     self.assertEqual(
-      ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
-       'media=cdrom'],
-      getRunningImageList()
+      ['${shared}/debian-${ver}-amd64-netinst.iso'],
+      self.getRunningImageList(kvm_instance_partition)
     )
 
 

software/metabase/buildout.hash.cfg

 [instance-profile]
 filename = instance.cfg.in
-md5sum = 8e48fa7c66a59b3d5faf0216922a574f
+md5sum = 143f46b125389f39905226ec9482ce2a

software/metabase/instance.cfg.in

@@ -45,7 +45,7 @@ promises =
 
 [metabase-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = $${:_buildout_section_name_}.py
 config-url= $${metabase-instance:url}/api/session/properties
 
@@ -91,7 +91,7 @@ command-line =
 
 [postgresql-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = promise-postgresql.py
 config-command = $${postgresql-psql:wrapper-path} -c '\q'
 

software/monitor/buildout.hash.cfg

@@ -14,23 +14,27 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = 2114ae8c1e92bd33ef1347f36f567c74
+md5sum = b6c2df0d4a62473d6dae26b10c0a4adc
 
 [template-monitor]
 _update_hash_filename_ = instance-monitor.cfg.jinja2
-md5sum = 165a15672fc85981f68b9af2d6253254
+md5sum = f23c007d6d6aed137cfd54aaa7ba52ab
 
 [json-test-template]
 _update_hash_filename_ = json-test-template.json.in.jinja2
 md5sum = 2eb5596544d9c341acf653d4f7ce2680
 
+[template-monitor-edgetest-basic]
+_update_hash_filename_ = instance-monitor-edgetest-basic.cfg.jinja2
+md5sum = 05c00ac393b50cfdef5d3bc5af93fe98
+
 [template-monitor-edgetest]
 _update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2
 md5sum = 3c8ab4e78f66c974eb95afc595a13514
 
 [template-monitor-edgebot]
 _update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
-md5sum = 365a6cc6831267a73fa5ebd56ad394ee
+md5sum = 436bb5251c8f1cd1e64bd5d3987d699c
 
 [network-bench-cfg]
 filename = network_bench.cfg.in
@@ -42,4 +46,4 @@ md5sum = d3cfa1f6760e3fa64ccd64acf213bdfb
 
 [template-surykatka-ini]
 _update_hash_filename_ = surykatka.ini.jinja2
-md5sum = 609c6cca763b73a80fa05ee56475eb20
+md5sum = 7e9b874c20faaa8190d2bf2b74caa727

software/monitor/instance-edgetest-basic-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object",
+  "properties": {
+    "nameserver-list": {
+      "default": [],
+      "title": "Nameservers",
+      "description": "List of nameservers IPs (like 1.2.3.4) to use.",
+      "type": "array"
+    },
+    "check-frontend-ip-list": {
+      "default": [],
+      "title": "Default Frontend IPs to check",
+      "description": "List of default frontend IPs (like 1.2.3.4) to check, if empty no constraint is used.",
+      "type": "array"
+    },
+    "check-dict": {
+      "title": "Configuration of checks",
+      "description": "Configure checks.",
+      "patternProperties": {
+        ".*": {
+          "properties": {
+            "url-list": {
+              "default": [],
+              "title": "URLs to check",
+              "description": "List of URLs to check, can be domain (example.com) or URL (https://example.com/, http://example.com/path)",
+              "type": "array"
+            },
+            "check-status-code": {
+              "title": "HTTP Code Check",
+              "description": "Expected response HTTP Code. Defaults to global configuration.",
+              "type": "number",
+              "minimum": 100,
+              "maximum": 599
+            },
+            "check-frontend-ip-list": {
+              "title": "Frontend IPs to check (optional)",
+              "description": "List of frontend IPs (like 1.2.3.4) to check. Defaults to global configuration.",
+              "type": "array"
+            },
+            "check-certificate-expiration-days": {
+              "title": "Certificate Expiration Check (days)",
+              "description": "Amount of days to consider certificate as being to-be-expired. Defaults to global configuration.",
+              "type": "number",
+              "minimum": 1
+            },
+            "check-maximum-elapsed-time": {
+              "title": "Maximum Elapsed Check (seconds)",
+              "description": "Maximum elapsed time for a site to reply to be considered good. Defaults to global configuration.",
+              "type": "number",
+              "minimum": 1
+            },
+            "failure-amount": {
+              "title": "Failure Amount",
+              "description": "Amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages. Defaults to global configuration.",
+              "type": "number",
+              "minimum": 1
+            },
+            "check-http-header-dict": {
+              "title": "[UI UNSUPPORTED] HTTP Header Check",
+              "description": "[UI UNSUPPORTED] JSON object of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}. Note: Shall be expressed directly as object, without any additional qouting.",
+              "type": "object",
+              "default": {}
+            }
+          },
+          "type": "object"
+        }
+      },
+      "type": "object",
+      "default": {}
+    },
+    "check-status-code": {
+      "title": "Default HTTP Code Check",
+      "description": "Default expected response HTTP Code.",
+      "type": "number",
+      "default": 200,
+      "minimum": 100,
+      "maximum": 599
+    },
+    "check-certificate-expiration-days": {
+      "title": "Default Certificate Expiration Check (days)",
+      "description": "Default amount of days to consider certificate as being to-be-expired.",
+      "type": "number",
+      "default": 15,
+      "minimum": 1
+    },
+    "check-maximum-elapsed-time": {
+      "title": "Default Maximum Elapsed Check (seconds)",
+      "description": "Default maximum elapsed time for a site to reply to be considered good.",
+      "type": "number",
+      "default": 2,
+      "minimum": 1
+    },
+    "failure-amount": {
+      "title": "Default Failure Amount",
+      "description": "Default amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages.",
+      "type": "number",
+      "default": 2,
+      "minimum": 1
+    }
+  }
+}

software/monitor/instance-monitor-edgebot.cfg.jinja2

@@ -28,13 +28,15 @@
 {%- set part_list = [] %}
 {%- for class, slave_instance_list in slave_instance_dict.items() %}
 {#- class is used to separate surykatka with different timeouts #}
+{%-   set URL_LIST = [] %}
 {%-   for slave in slave_instance_list | sort(attribute='-slave-title') %}
+{%-     do URL_LIST.append(slave['url']) %}
 {%-     set part_id = 'http-query-' ~ hashlib_module.md5(slave['-slave-reference'].encode('utf-8')).hexdigest() ~ '-promise' %}
 {%-     do part_list.append(part_id) %}
 {%-     set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}
 [{{part_id}}]
 <= monitor-promise-base
-module = check_surykatka_json
+promise = check_surykatka_json
 name = {{ safe_name }}.py
 config-report = http_query
 config-url = {{ slave['url'] }}
@@ -49,7 +51,7 @@ config-json-file = ${surykatka-config-{{ class }}:json}
 
 [surykatka-bot-promise-{{ class }}]
 <= monitor-promise-base
-module = check_surykatka_json
+promise = check_surykatka_json
 name = surykatka-bot-promise-{{ class }}.py
 config-report = bot_status
 config-json-file = ${surykatka-config-{{ class }}:json}
@@ -59,7 +61,7 @@ recipe = slapos.recipe.template:jinja2
 db = ${directory:srv}/surykatka-{{ class }}.db
 rendered = ${directory:etc}/surykatka-{{ class }}.ini
 template = {{ template_surykatka_ini }}
-slave_instance_list = {{ dumps(slave_instance_list) }}
+url_list = {{ dumps(URL_LIST) }}
 nameserver_list = {{ dumps(CONFIGURATION['nameserver-list']) }}
 json = ${directory:srv}/surykatka-{{ class }}.json
 {#- timeout is just a bit bigger than class time #}
@@ -69,7 +71,7 @@ context =
   import json_module json
   key db :db
   key nameserver_list :nameserver_list
-  key slave_instance_list :slave_instance_list
+  key url_list :url_list
   key timeout :timeout
 
 [surykatka-{{ class }}]
@@ -84,11 +86,7 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 recipe = slapos.recipe.template:jinja2
 json = ${surykatka-config-{{ class }}:json}
 template = inline:#!/bin/sh
-  if {{ surykatka_binary }} --run status --configuration ${surykatka-{{ class }}:config} --output json > ${:json}.tmp ; then
-    mv -f ${:json}.tmp ${:json}
-  else
-    rm -f ${:json}.tmp
-  fi
+  {{ surykatka_binary }} --run status --configuration ${surykatka-{{ class }}:config} --output json --stdout ${:json}
 rendered = ${monitor-directory:bin}/${:_buildout_section_name_}
 mode = 0755
 

software/monitor/instance-monitor-edgetest-basic.cfg.jinja2

+{#- PREPARE #}
+{%- set DEFAULT_DICT = {
+  'check-status-code': 200,
+  'check-http-header-dict': {},
+  'check-certificate-expiration-days': 15,
+  'failure-amount': 2,
+  'check-maximum-elapsed-time': 2,
+  'check-frontend-ip-list': [],
+  'nameserver-list': []
+} %}
+{%- for default_key in ['check-frontend-ip-list', 'nameserver-list', 'check-status-code', 'check-certificate-expiration-days', 'check-maximum-elapsed-time', 'failure-amount'] %}
+{%-   if default_key in slapparameter_dict %}
+{%-     do DEFAULT_DICT.__setitem__(default_key, slapparameter_dict[default_key]) %}
+{%-   endif %}
+{%- endfor %}
+{%- set CHECK_DICT = {} %}
+{%- for check_name, check_definition in slapparameter_dict['check-dict'].items() %}
+{%-   if 'url-list' in check_definition %}
+{%-     set check = DEFAULT_DICT.copy()  %}
+{%-     set url_list = [] %}
+{%-     for url in check_definition.pop('url-list') %}
+{%-       if len(url.strip()) > 0 %}
+{#-         Support various URL entries #}
+{%-         if url.startswith('http:') or url.startswith('https:') %}
+{%-           if url not in url_list %}
+{%-             do url_list.append(url) %}
+{%-           endif %}
+{%-         else %}
+{%-           for schema in ['http://', 'https://'] %}
+{%-             set add_url = '%s%s' % (schema, url) %}
+{%-             if add_url not in url_list %}
+{%-               do url_list.append('%s%s' % (schema, url)) %}
+{%-             endif %}
+{%-           endfor %}
+{%-         endif %}
+{%-       endif %}
+{%-     endfor %}
+{%-     do check.update(check_definition) %}
+{%-     do check.__setitem__('url-list', url_list) %}
+{%-     set class = check['check-maximum-elapsed-time'] %}
+{%-     if class not in CHECK_DICT %}
+{%-       do CHECK_DICT.__setitem__(class, []) %}
+{%-     endif %}
+{%-     do check.__setitem__('-name', check_name) %}
+{%-     do CHECK_DICT[class].append(check) %}
+{%-   endif %}
+{%- endfor %}
+
+{%- set PART_LIST = [] %}
+{%- for class, class_check_list in CHECK_DICT.items() %}
+{#- class is used to separate surykatka with different timeouts #}
+{%-   set CLASS_URL_LIST = [] %}
+{%-   for check in class_check_list | sort(attribute='-name') %}
+{%-     do CLASS_URL_LIST.extend(check['url-list']) %}
+{%-     for url in check['url-list'] %}
+{%-       set promise_name = 'http-query-' ~ hashlib_module.md5(check['-name'].encode('utf-8')).hexdigest() ~ '-' ~ hashlib_module.md5(url.encode('utf-8')).hexdigest() %}
+{%-       set part_name = promise_name ~ '-promise' %}
+{%-       do PART_LIST.append(part_name) %}
+[{{part_name}}]
+<= monitor-promise-base
+promise = check_surykatka_json
+name = {{ promise_name }}.py
+config-report = http_query
+config-url = {{ url }}
+config-status-code = {{ check['check-status-code'] }}
+config-http-header-dict = {{ json_module.dumps(check['check-http-header-dict']) }}
+config-certificate-expiration-days = {{ check['check-certificate-expiration-days'] }}
+config-failure-amount = {{ check['failure-amount'] }}
+config-maximum-elapsed-time = {{ check['check-maximum-elapsed-time'] }}
+config-ip-list = {{ ' '.join(check['check-frontend-ip-list']) }}
+config-json-file = ${surykatka-config-{{ class }}:json}
+{%-     endfor %} {#- for url in check['url-list'] #}
+{%-   endfor %} {#- for check in class_check_list | sort(attribute='-name') #}
+
+{%-     do PART_LIST.append('surykatka-bot-%i-promise' % (class,)) %}
+[surykatka-bot-{{ class }}-promise]
+<= monitor-promise-base
+promise = check_surykatka_json
+name = surykatka-bot-{{ class }}.py
+config-report = bot_status
+config-json-file = ${surykatka-config-{{ class }}:json}
+
+[surykatka-config-{{ class }}]
+recipe = slapos.recipe.template:jinja2
+db = ${directory:srv}/surykatka-{{ class }}.db
+rendered = ${directory:etc}/surykatka-{{ class }}.ini
+template = {{ template_surykatka_ini }}
+url_list = {{ dumps(CLASS_URL_LIST) }}
+nameserver_list = {{ dumps(DEFAULT_DICT['nameserver-list']) }}
+json = ${directory:srv}/surykatka-{{ class }}.json
+{#- timeout is just a bit bigger than class time #}
+timeout = {{ int(class) + 2 }}
+
+context =
+  import json_module json
+  key db :db
+  key nameserver_list :nameserver_list
+  key url_list :url_list
+  key timeout :timeout
+
+{%-     do PART_LIST.append('surykatka-%i'% (class,)) %}
+[surykatka-{{ class }}]
+recipe = slapos.cookbook:wrapper
+config = ${surykatka-config-{{ class }}:rendered}
+command-line =
+  {{ surykatka_binary }} --run crawl --reload --configuration ${:config}
+wrapper-path = ${monitor-directory:service}/${:_buildout_section_name_}
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+
+{%-     do PART_LIST.append('surykatka-json-%i-promise'% (class,)) %}
+[surykatka-json-{{ class }}-promise]
+<= monitor-promise-base
+promise = check_file_state
+name = surykatka-json-{{ class }}.py
+config-filename = ${surykatka-config-{{ class }}:json}
+config-state = not-empty
+# workaround for bug in check_file_state
+config-url =
+
+[surykatka-status-json-{{ class }}]
+recipe = slapos.recipe.template:jinja2
+json = ${surykatka-config-{{ class }}:json}
+template = inline:#!/bin/sh
+  {{ surykatka_binary }} --run status --configuration ${surykatka-{{ class }}:config} --output json --stdout ${:json}
+rendered = ${monitor-directory:bin}/${:_buildout_section_name_}
+mode = 0755
+
+{%-     do PART_LIST.append('cron-entry-surykatka-status-%i' % (class,)) %}
+[cron-entry-surykatka-status-{{ class }}]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${directory:etc}/cron.d
+name = surykatka-status-{{ class }}
+frequency = */2 * * * *
+command = ${surykatka-status-json-{{ class }}:rendered}
+{%- endfor %} {#- for class, class_check_list in CHECK_DICT.items() #}
+
+[buildout]
+extends = {{ instance_base_monitor }}
+parts +=
+{% for part_id in sorted(PART_LIST) %}
+  {{ part_id }}
+{% endfor %}

software/monitor/instance-monitor.cfg.jinja2

@@ -82,7 +82,7 @@ wrapper-path = ${monitor-directory:bin}/monitor-collect-csv-dump
 
 [monitor-check-memory-usage]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = check-computer-memory-usage.py
 config-command = {{ buildout_bin}}/check-computer-memory -db ${monitor-instance-parameter:collector-db} --threshold ${slap-parameter:memory-percent-threshold} --unit percent
 
@@ -90,11 +90,8 @@ config-command = {{ buildout_bin}}/check-computer-memory -db ${monitor-instance-
 recipe = slapos.cookbook:promise.plugin
 eggs =
   slapos.toolbox
-file = ${monitor-conf-parameters:promise-output-file}
-content = 
-  from slapos.promise.plugin.check_server_cpu_load import RunPromise
+module = slapos.promise.plugin.check_server_cpu_load
 output = ${directory:plugins}/system-CPU-load-check.py
-mode = 600
 config-cpu-load-threshold = ${slap-parameter:cpu-load-threshold}
 
 [publish-connection-information]

software/monitor/instance.cfg

@@ -10,6 +10,7 @@ recipe = slapos.cookbook:switch-softwaretype
 default = instance-base-monitor:rendered
 edgetest = instance-base-edgetest:rendered
 edgebot = instance-base-edgebot:rendered
+edgetest-basic = instance-edgetest-basic:rendered
 RootSoftwareInstance = $${:default}
 
 [instance-base-monitor]
@@ -27,6 +28,34 @@ context = key develop_eggs_directory buildout:develop-eggs-directory
           raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
 mode = 0644
 
+[instance-template]
+recipe = slapos.recipe.template:jinja2
+extensions = jinja2.ext.do
+rendered = $${buildout:directory}/$${:_buildout_section_name_}.cfg
+context =
+  import json_module json
+  import hashlib_module hashlib
+  key develop_eggs_directory buildout:develop-eggs-directory
+  key eggs_directory buildout:eggs-directory
+  key slapparameter_dict slap-configuration:configuration
+  key slap_software_type slap-configuration:slap-software-type
+  key instance_base_monitor instance-base-monitor:rendered
+  raw buildout_bin ${buildout:bin-directory}
+  $${:extra-context}
+mode = 0644
+
+[surykatka]
+binary = ${buildout:bin-directory}/${surykatka:script-name}
+ini = ${template-surykatka-ini:target}
+
+[instance-edgetest-basic]
+<= instance-template
+template = ${template-monitor-edgetest-basic:target}
+extra-context =
+  raw software_type edgetest-basic
+  key template_surykatka_ini surykatka:ini
+  key surykatka_binary surykatka:binary
+
 [instance-base-edgetest]
 recipe = slapos.recipe.template:jinja2
 template = ${template-monitor-edgetest:target}
@@ -51,8 +80,6 @@ recipe = slapos.recipe.template:jinja2
 template = ${template-monitor-edgebot:target}
 rendered = $${buildout:directory}/template-monitor-edgebot.cfg
 extensions = jinja2.ext.do
-surykatka-binary = ${buildout:bin-directory}/${surykatka:script-name}
-template-surykatka-ini = ${template-surykatka-ini:target}
 
 context = import json_module json
           import hashlib_module hashlib
@@ -62,8 +89,8 @@ context = import json_module json
           key slapparameter_dict slap-configuration:configuration
           key slap_software_type slap-configuration:slap-software-type
           raw software_type edgebot
-          key surykatka_binary :surykatka-binary
-          key template_surykatka_ini :template-surykatka-ini
+          key surykatka_binary surykatka:binary
+          key template_surykatka_ini surykatka:ini
           raw buildout_bin ${buildout:bin-directory}
           raw monitor_template_output ${monitor-template:output}
           raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}

software/monitor/software.cfg

@@ -31,6 +31,11 @@ recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:_update_hash_filename_}
 mode = 0644
 
+[template-monitor-edgetest-basic]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+mode = 0644
+
 [template-monitor-edgetest]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:_update_hash_filename_}
@@ -77,11 +82,13 @@ scripts =
 
 [versions]
 
-surykatka = 0.5.0
+surykatka = 0.7.1
 
-# For surykatka 0.5.0
-click = 7.0
-dnspython = 1.16.0
+# For surykatka 0.7.1
+click = 8.0.1
+dnspython = 2.1.0
 forcediphttpsadapter = 1.0.1
 miniupnpc = 2.0.2
-peewee = 3.13.1
+peewee = 3.14.4
+python-whois = 0.7.3
+future = 0.18.2

software/monitor/software.cfg.json

@@ -10,23 +10,31 @@
       "response": "instance-default-output-schema.json",
       "index": 0
     },
+    "edgetest-basic": {
+      "title": "Edge Test Basic",
+      "description": "Basic URL monitoring configuration",
+      "request": "instance-edgetest-basic-input-schema.json",
+      "response": "instance-default-output-schema.json",
+      "serialisation": "json-in-xml",
+      "index": 1
+    },
     "edgetest": {
-      "title": "Edge Test",
-      "description": "Cluster of bots to perform a distributed monitoring ",
+      "title": "Edge Test [OBSOLETE]",
+      "description": "Cluster of bots to perform a distributed monitoring. OBSOLETE: Use Edge Test Basic.",
       "request": "instance-edgetest-input-schema.json",
       "response": "instance-default-output-schema.json",
       "serialisation": "json-in-xml",
-      "index": 1
+      "index": 2
     },
     "edgetest-slave": {
-      "title": "Edge Test Slave",
+      "title": "Edge Test Slave [OBSOLETE]",
       "shared": true,
       "software-type": "edgetest",
-      "description": "Cluster of bots to perform a distributed monitoring ",
+      "description": "Cluster of bots to perform a distributed monitoring. OBSOLETE: Use Edge Test Basic.",
       "request": "instance-edgetest-slave-input-schema.json",
       "response": "instance-default-output-schema.json",
       "serialisation": "json-in-xml",
-      "index": 2
+      "index": 3
     }
   }
 }

software/monitor/surykatka.ini.jinja2

@@ -9,9 +9,6 @@ NAMESERVER =
 {%-   endfor %}
 {% endif %}
 URL =
-{%- for slave in slave_instance_list | sort(attribute='-slave-title') %}
-{%-   if 'url' in slave %}
-  {{ slave['url'] }}
-{%-   endif -%}
+{%- for url in sorted(url_list) %}
+  {{ url -}}
 {% endfor %}
-

software/monitor/test/test.py

@@ -137,89 +137,13 @@ class MonitorTestMixin:
     )
 
 
-class EdgeSlaveMixin(MonitorTestMixin):
+class EdgeMixin(object):
   __partition_reference__ = 'edge'
   instance_max_retry = 20
   expected_connection_parameter_dict = {}
 
-  @classmethod
-  def setUpClass(cls):
-    # XXX we run these tests with --all as a workaround for the fact that after
-    # requesting new shared instances we don't have promise to wait for the
-    # processing of these shared instances to be completed.
-    # The sequence is something like this:
-    #   - `requestEdgetestSlaves` will request edgetest partition
-    #   - first `waitForInstance` will process the edgetest partition, which will
-    #     request a edgebot partition, but without promise to wait for the
-    #     processing to be finished, so the first run of `slapos node instance`
-    #     exits with success code and `waitForInstance` return.
-    #   - second `waitForInstance` process the edgebot partition.
-    # Once we implement a promise (or something similar) here, we should not
-    # have to use --all
-    cls.slap._force_slapos_node_instance_all = True
-    return super(EdgeSlaveMixin, cls).setUpClass()
-
-  @classmethod
-  def getInstanceSoftwareType(cls):
-    return 'edgetest'
-
-  def requestEdgetestSlave(self, partition_reference, partition_parameter_kw):
-    software_url = self.getSoftwareURL()
-    return self.slap.request(
-      software_release=software_url,
-      software_type='edgetest',
-      partition_reference=partition_reference,
-      partition_parameter_kw={'_': json.dumps(partition_parameter_kw)},
-      shared=True
-    )
-
-  def updateSurykatkaDict(self):
-    for instance_reference in self.surykatka_dict:
-      for class_ in self.surykatka_dict[instance_reference]:
-        update_dict = {}
-        update_dict['ini-file'] = os.path.join(
-          self.slap.instance_directory, instance_reference, 'etc',
-          'surykatka-%s.ini' % (class_,))
-        update_dict['json-file'] = os.path.join(
-          self.slap.instance_directory, instance_reference, 'srv',
-          'surykatka-%s.json' % (class_,))
-        update_dict['status-json'] = os.path.join(
-          self.slap.instance_directory, instance_reference, 'bin',
-          'surykatka-status-json-%s' % (class_,))
-        update_dict['bot-promise'] = 'surykatka-bot-promise-%s.py' % (class_,)
-        update_dict['status-cron'] = os.path.join(
-          self.slap.instance_directory, instance_reference, 'etc',
-          'cron.d', 'surykatka-status-%s' % (class_,))
-        update_dict['db_file'] = os.path.join(
-          self.slap.instance_directory, instance_reference, 'srv',
-          'surykatka-%s.db' % (class_,))
-        self.surykatka_dict[instance_reference][class_].update(update_dict)
-
-  def setUpMonitorConfigurationList(self):
-    self.monitor_configuration_list = [
-      {
-        'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
-        'version': 'RSS',
-        'title': 'testing partition 0',
-        'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
-        'text': 'testing partition 0',
-        'type': 'rss',
-        'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
-      },
-      {
-        'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
-        'version': 'RSS',
-        'title': 'edgebot-1',
-        'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
-        'text': 'edgebot-1',
-        'type': 'rss',
-        'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
-      }
-    ]
-
   def setUp(self):
     self.updateSurykatkaDict()
-    self.setUpMonitorConfigurationList()
 
   def assertSurykatkaIni(self):
     expected_init_path_list = []
@@ -252,11 +176,6 @@ class EdgeSlaveMixin(MonitorTestMixin):
 
     self.assertTrue(content in promise)
 
-  def assertHttpQueryPromiseContent(self, instance_reference, name, content):
-    hashed = 'http-query-%s-promise.py' % (
-      hashlib.md5(('_' + name).encode('utf-8')).hexdigest(),)
-    self.assertPromiseContent(instance_reference, hashed, content)
-
   def assertSurykatkaBotPromise(self):
     for instance_reference in self.surykatka_dict:
       for info_dict in self.surykatka_dict[instance_reference].values():
@@ -297,6 +216,25 @@ class EdgeSlaveMixin(MonitorTestMixin):
           status_json = json.load(fh)
         self.assertIn('bot_status', status_json)
 
+
+class EdgeSlaveMixin(EdgeMixin, MonitorTestMixin):
+  @classmethod
+  def setUpClass(cls):
+    # XXX we run these tests with --all as a workaround for the fact that after
+    # requesting new shared instances we don't have promise to wait for the
+    # processing of these shared instances to be completed.
+    # The sequence is something like this:
+    #   - `requestEdgetestSlaves` will request edgetest partition
+    #   - first `waitForInstance` will process the edgetest partition, which
+    #     will request a edgebot partition, but without promise to wait for the
+    #     processing to be finished, so the first run of `slapos node instance`
+    #     exits with success code and `waitForInstance` return.
+    #   - second `waitForInstance` process the edgebot partition.
+    # Once we implement a promise (or something similar) here, we should not
+    # have to use --all
+    cls.slap._force_slapos_node_instance_all = True
+    return super().setUpClass()
+
   def assertConnectionParameterDict(self):
     serialised = self.requestDefaultInstance().getConnectionParameterDict()
     connection_parameter_dict = json.loads(serialised['_'])
@@ -309,6 +247,73 @@ class EdgeSlaveMixin(MonitorTestMixin):
       connection_parameter_dict
     )
 
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'edgetest'
+
+  def updateSurykatkaDict(self):
+    for instance_reference in self.surykatka_dict:
+      for class_ in self.surykatka_dict[instance_reference]:
+        update_dict = {}
+        update_dict['ini-file'] = os.path.join(
+          self.slap.instance_directory, instance_reference, 'etc',
+          'surykatka-%s.ini' % (class_,))
+        update_dict['json-file'] = os.path.join(
+          self.slap.instance_directory, instance_reference, 'srv',
+          'surykatka-%s.json' % (class_,))
+        update_dict['status-json'] = os.path.join(
+          self.slap.instance_directory, instance_reference, 'bin',
+          'surykatka-status-json-%s' % (class_,))
+        update_dict['bot-promise'] = 'surykatka-bot-promise-%s.py' % (class_,)
+        update_dict['status-cron'] = os.path.join(
+          self.slap.instance_directory, instance_reference, 'etc',
+          'cron.d', 'surykatka-status-%s' % (class_,))
+        update_dict['db_file'] = os.path.join(
+          self.slap.instance_directory, instance_reference, 'srv',
+          'surykatka-%s.db' % (class_,))
+        self.surykatka_dict[instance_reference][class_].update(update_dict)
+
+  def assertHttpQueryPromiseContent(self, instance_reference, name, content):
+    hashed = 'http-query-%s-promise.py' % (
+      hashlib.md5(('_' + name).encode('utf-8')).hexdigest(),)
+    self.assertPromiseContent(instance_reference, hashed, content)
+
+  def requestEdgetestSlave(self, partition_reference, partition_parameter_kw):
+    software_url = self.getSoftwareURL()
+    return self.slap.request(
+      software_release=software_url,
+      software_type='edgetest',
+      partition_reference=partition_reference,
+      partition_parameter_kw={'_': json.dumps(partition_parameter_kw)},
+      shared=True
+    )
+
+  def setUpMonitorConfigurationList(self):
+    self.monitor_configuration_list = [
+      {
+        'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
+        'version': 'RSS',
+        'title': 'testing partition 0',
+        'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
+        'text': 'testing partition 0',
+        'type': 'rss',
+        'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
+      },
+      {
+        'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
+        'version': 'RSS',
+        'title': 'edgebot-1',
+        'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
+        'text': 'edgebot-1',
+        'type': 'rss',
+        'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
+      }
+    ]
+
+  def setUp(self):
+    super().setUp()
+    self.setUpMonitorConfigurationList()
+
   def test(self):
     # Note: Those tests do not run surykatka and do not do real checks, as
     #       this depends too much on the environment and is really hard to
@@ -950,7 +955,7 @@ URL =
         self.surykatka_dict['edge3'][2]['json-file'],))
 
   def test(self):
-    super(TestEdgeRegion, self).test()
+    super().test()
     self.assertSlaveConnectionParameterDict()
 
   maxDiff = None
@@ -1101,7 +1106,7 @@ class TestEdgeRegionDestroyed(TestEdgeRegion):
     ]
 
   def test(self):
-    super(TestEdgeRegionDestroyed, self).test()
+    super().test()
     # hack around @classmethod
     self.__class__.instance_parameter_dict[
       'region-dict']['Region Three']['state'] = 'destroyed'
@@ -1256,7 +1261,7 @@ class TestEdgeRegionAdded(TestEdgeRegion):
     ]
 
   def test(self):
-    super(TestEdgeRegionAdded, self).test()
+    super().test()
     self.__class__.instance_parameter_dict['region-dict']['Region Four'] = {
       'sla-computer_guid': 'local',
       'state': 'started',
@@ -1408,3 +1413,320 @@ URL =
   'status-code': '200',
   'url': 'https://www.all.org/'}""" % (
         self.surykatka_dict['edge4'][2]['json-file'],))
+
+
+class TestEdgeBasic(EdgeMixin, SlapOSInstanceTestCase):
+  surykatka_dict = {}
+
+  def assertConnectionParameterDict(self):
+    connection_parameter_dict = self.requestDefaultInstance(
+      ).getConnectionParameterDict()
+    # tested elsewhere
+    connection_parameter_dict.pop('monitor-setup-url', None)
+    # comes from instance-monitor.cfg.jinja2, not needed here
+    connection_parameter_dict.pop('server_log_url', None)
+    self.assertEqual(
+      self.expected_connection_parameter_dict,
+      connection_parameter_dict
+    )
+
+  def assertHttpQueryPromiseContent(
+    self, instance_reference, name, url, content):
+    hashed = 'http-query-%s-%s.py' % (
+      hashlib.md5((name).encode('utf-8')).hexdigest(),
+      hashlib.md5((url).encode('utf-8')).hexdigest(),
+    )
+    self.assertPromiseContent(instance_reference, hashed, content)
+
+  def updateSurykatkaDict(self):
+    for instance_reference in self.surykatka_dict:
+      for class_ in self.surykatka_dict[instance_reference]:
+        update_dict = {}
+        update_dict['ini-file'] = os.path.join(
+          self.slap.instance_directory, instance_reference, 'etc',
+          'surykatka-%s.ini' % (class_,))
+        update_dict['json-file'] = os.path.join(
+          self.slap.instance_directory, instance_reference, 'srv',
+          'surykatka-%s.json' % (class_,))
+        update_dict['status-json'] = os.path.join(
+          self.slap.instance_directory, instance_reference, 'bin',
+          'surykatka-status-json-%s' % (class_,))
+        update_dict['bot-promise'] = 'surykatka-bot-%s.py' % (class_,)
+        update_dict['status-cron'] = os.path.join(
+          self.slap.instance_directory, instance_reference, 'etc',
+          'cron.d', 'surykatka-status-%s' % (class_,))
+        update_dict['db_file'] = os.path.join(
+          self.slap.instance_directory, instance_reference, 'srv',
+          'surykatka-%s.db' % (class_,))
+        self.surykatka_dict[instance_reference][class_].update(update_dict)
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'_': json.dumps({
+      'nameserver-list': ['127.0.1.1', '127.0.1.2'],
+      'check-frontend-ip-list': ['127.0.0.1', '127.0.0.2'],
+      "check-maximum-elapsed-time": 5,
+      "check-certificate-expiration-days": 7,
+      "check-status-code": 201,
+      "failure-amount": 1,
+      "check-dict": {
+        "path-check": {
+           "url-list": [
+             "https://path.example.com/path",
+           ]
+        },
+        "domain-check": {
+           "url-list": [
+             "domain.example.com",
+           ]
+        },
+        "frontend-check": {
+           "url-list": [
+             "https://frontend.example.com",
+           ],
+           "check-frontend-ip-list": ['127.0.0.3'],
+        },
+        "frontend-empty-check": {
+           "url-list": [
+             "https://frontendempty.example.com",
+           ],
+           "check-frontend-ip-list": [],
+        },
+        "status-check": {
+           "url-list": [
+             "https://status.example.com",
+           ],
+           "check-status-code": 202,
+        },
+        "certificate-check": {
+           "url-list": [
+             "https://certificate.example.com",
+           ],
+           "check-certificate-expiration-days": 11,
+        },
+        "time-check": {
+           "url-list": [
+             "https://time.example.com",
+           ],
+           "check-maximum-elapsed-time": 11,
+        },
+        "failure-check": {
+           "url-list": [
+             "https://failure.example.com",
+           ],
+           "failure-amount": 3,
+        },
+        "header-check": {
+           "url-list": [
+             "https://header.example.com",
+           ],
+           'check-http-header-dict': {"A": "AAA"},
+        },
+      }
+    })}
+
+  surykatka_dict = {
+    'edge0': {
+      5: {'expected_ini': """[SURYKATKA]
+INTERVAL = 120
+TIMEOUT = 7
+SQLITE = %(db_file)s
+NAMESERVER =
+  127.0.1.1
+  127.0.1.2
+
+URL =
+  http://domain.example.com
+  https://certificate.example.com
+  https://domain.example.com
+  https://failure.example.com
+  https://frontend.example.com
+  https://frontendempty.example.com
+  https://header.example.com
+  https://path.example.com/path
+  https://status.example.com
+"""},
+      11: {'expected_ini': """[SURYKATKA]
+INTERVAL = 120
+TIMEOUT = 13
+SQLITE = %(db_file)s
+NAMESERVER =
+  127.0.1.1
+  127.0.1.2
+
+URL =
+  https://time.example.com
+"""},
+    }
+  }
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'edgetest-basic'
+
+  def assertSurykatkaPromises(self):
+    self.assertHttpQueryPromiseContent(
+      'edge0',
+      'path-check',
+      'https://path.example.com/path',
+      """extra_config_dict = { 'certificate-expiration-days': '7',
+  'failure-amount': '1',
+  'http-header-dict': '{}',
+  'ip-list': '127.0.0.1 127.0.0.2',
+  'json-file': '%s',
+  'maximum-elapsed-time': '5',
+  'report': 'http_query',
+  'status-code': '201',
+  'url': 'https://path.example.com/path'}""" % (
+        self.surykatka_dict['edge0'][5]['json-file'],))
+
+    self.assertHttpQueryPromiseContent(
+      'edge0',
+      'domain-check',
+      'https://domain.example.com',
+      """extra_config_dict = { 'certificate-expiration-days': '7',
+  'failure-amount': '1',
+  'http-header-dict': '{}',
+  'ip-list': '127.0.0.1 127.0.0.2',
+  'json-file': '%s',
+  'maximum-elapsed-time': '5',
+  'report': 'http_query',
+  'status-code': '201',
+  'url': 'https://domain.example.com'}""" % (
+        self.surykatka_dict['edge0'][5]['json-file'],))
+
+    self.assertHttpQueryPromiseContent(
+      'edge0',
+      'domain-check',
+      'http://domain.example.com',
+      """extra_config_dict = { 'certificate-expiration-days': '7',
+  'failure-amount': '1',
+  'http-header-dict': '{}',
+  'ip-list': '127.0.0.1 127.0.0.2',
+  'json-file': '%s',
+  'maximum-elapsed-time': '5',
+  'report': 'http_query',
+  'status-code': '201',
+  'url': 'http://domain.example.com'}""" % (
+        self.surykatka_dict['edge0'][5]['json-file'],))
+
+    self.assertHttpQueryPromiseContent(
+      'edge0',
+      'frontend-check',
+      'https://frontend.example.com',
+      """extra_config_dict = { 'certificate-expiration-days': '7',
+  'failure-amount': '1',
+  'http-header-dict': '{}',
+  'ip-list': '127.0.0.3',
+  'json-file': '%s',
+  'maximum-elapsed-time': '5',
+  'report': 'http_query',
+  'status-code': '201',
+  'url': 'https://frontend.example.com'}""" % (
+        self.surykatka_dict['edge0'][5]['json-file'],))
+
+    self.assertHttpQueryPromiseContent(
+      'edge0',
+      'frontend-empty-check',
+      'https://frontendempty.example.com',
+      """extra_config_dict = { 'certificate-expiration-days': '7',
+  'failure-amount': '1',
+  'http-header-dict': '{}',
+  'ip-list': '',
+  'json-file': '%s',
+  'maximum-elapsed-time': '5',
+  'report': 'http_query',
+  'status-code': '201',
+  'url': 'https://frontendempty.example.com'}""" % (
+        self.surykatka_dict['edge0'][5]['json-file'],))
+
+    self.assertHttpQueryPromiseContent(
+      'edge0',
+      'status-check',
+      'https://status.example.com',
+      """extra_config_dict = { 'certificate-expiration-days': '7',
+  'failure-amount': '1',
+  'http-header-dict': '{}',
+  'ip-list': '127.0.0.1 127.0.0.2',
+  'json-file': '%s',
+  'maximum-elapsed-time': '5',
+  'report': 'http_query',
+  'status-code': '202',
+  'url': 'https://status.example.com'}""" % (
+        self.surykatka_dict['edge0'][5]['json-file'],))
+
+    self.assertHttpQueryPromiseContent(
+      'edge0',
+      'certificate-check',
+      'https://certificate.example.com',
+      """extra_config_dict = { 'certificate-expiration-days': '11',
+  'failure-amount': '1',
+  'http-header-dict': '{}',
+  'ip-list': '127.0.0.1 127.0.0.2',
+  'json-file': '%s',
+  'maximum-elapsed-time': '5',
+  'report': 'http_query',
+  'status-code': '201',
+  'url': 'https://certificate.example.com'}""" % (
+        self.surykatka_dict['edge0'][5]['json-file'],))
+
+    self.assertHttpQueryPromiseContent(
+      'edge0',
+      'time-check',
+      'https://time.example.com',
+      """extra_config_dict = { 'certificate-expiration-days': '7',
+  'failure-amount': '1',
+  'http-header-dict': '{}',
+  'ip-list': '127.0.0.1 127.0.0.2',
+  'json-file': '%s',
+  'maximum-elapsed-time': '11',
+  'report': 'http_query',
+  'status-code': '201',
+  'url': 'https://time.example.com'}""" % (
+        self.surykatka_dict['edge0'][11]['json-file'],))
+
+    self.assertHttpQueryPromiseContent(
+      'edge0',
+      'failure-check',
+      'https://failure.example.com',
+      """extra_config_dict = { 'certificate-expiration-days': '7',
+  'failure-amount': '3',
+  'http-header-dict': '{}',
+  'ip-list': '127.0.0.1 127.0.0.2',
+  'json-file': '%s',
+  'maximum-elapsed-time': '5',
+  'report': 'http_query',
+  'status-code': '201',
+  'url': 'https://failure.example.com'}""" % (
+        self.surykatka_dict['edge0'][5]['json-file'],))
+
+    self.assertHttpQueryPromiseContent(
+      'edge0',
+      'header-check',
+      'https://header.example.com',
+      """extra_config_dict = { 'certificate-expiration-days': '7',
+  'failure-amount': '1',
+  'http-header-dict': '{"A": "AAA"}',
+  'ip-list': '127.0.0.1 127.0.0.2',
+  'json-file': '%s',
+  'maximum-elapsed-time': '5',
+  'report': 'http_query',
+  'status-code': '201',
+  'url': 'https://header.example.com'}""" % (
+        self.surykatka_dict['edge0'][5]['json-file'],))
+
+  def test(self):
+    # Note: Those tests do not run surykatka and do not do real checks, as
+    #       this depends too much on the environment and is really hard to
+    #       mock
+    #       So it is possible that some bugs might slip under the radar
+    #       Nevertheless the surykatka and check_surykatka_json are heavily
+    #       unit tested, and configuration created by the profiles is asserted
+    #       here, so it shall be enough as reasonable status
+    self.initiateSurykatkaRun()
+    self.assertSurykatkaStatusJSON()
+    self.assertSurykatkaIni()
+    self.assertSurykatkaBotPromise()
+    self.assertSurykatkaPromises()
+    self.assertSurykatkaCron()
+    self.assertConnectionParameterDict()

software/nayuos/buildout.hash.cfg

@@ -14,4 +14,4 @@
 # not need these here).
 [template-instance]
 filename = instance.cfg
-md5sum = 0974248c0b0ad5da45670386a5301e47
+md5sum = 14132bba14a1e66e7abb1a7c58b333e5

software/nayuos/instance.cfg

@@ -76,7 +76,7 @@ command-line = sudo -V
 [promise-sudo-on-host]
 # assert sudo is installed, as it is required to enter the chroot 'cros_sdk'
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = ${:_buildout_section_name_}.py
 config-command = ${promise-sudo-on-host-bin:wrapper-path}
 

software/neoppod/buildout.hash.cfg

@@ -22,7 +22,7 @@ md5sum = c13b4f1a5aa526a8d3f8e02bf6baf785
 
 [instance-neo-admin]
 filename = instance-neo-admin.cfg.in
-md5sum = dabc1e50475055b3ee9184dcace5e8d2
+md5sum = b6e1ccb1d90160110202e5111eec2afa
 
 [instance-neo-master]
 filename = instance-neo-master.cfg.in

software/neoppod/instance-neo-admin.cfg.in

@@ -18,7 +18,7 @@ plugin =  ${:etc}/plugin
 
 [monitor-neo-health]
 <= monitor-promise-base
-module = check_neo_health
+promise = check_neo_health
 name = ${:_buildout_section_name_}.py
 config-neoctl = ${neoctl:wrapper-path}
 {%- if bang_on_problem != None %}

software/neoppod/stress-testing/runTestSuite.in

@@ -135,8 +135,9 @@ mount.neoppod = {{parameter_dict['neoppod']}}
       status_dict['stderr'] = ''.join(stderr)
 
       if not returncode:
+        os.rename(partdir, dist)
         try:
-          with open(os.path.join(partdir, 'status')) as f:
+          with open(os.path.join(dist, 'status')) as f:
             status = int(f.read())
         except Exception:
           pass
@@ -145,7 +146,7 @@ mount.neoppod = {{parameter_dict['neoppod']}}
           try:
             curses.setupterm('screen')
             rmkx = curses.tigetstr('rmkx')
-            with open(os.path.join(partdir, 'stdout')) as f:
+            with open(os.path.join(dist, 'stdout')) as f:
               stdout = f.read()
             status_dict['stdout'] = stdout[stdout.rindex(rmkx)+len(rmkx):]
           except Exception:

software/neoppod/stress-testing/software.cfg

@@ -71,6 +71,7 @@ location = ${:repository}
 [vm-debian]
 # Mainly for logs, otherwise 2G would enough (<100M for the SR).
 size = 8Gi
+dists += debian-bullseye
 late-command +=
   echo tmpfs /tmp tmpfs mode=1777,size=90% 0 0 >>/etc/fstab
 packages +=
@@ -78,8 +79,6 @@ packages +=
   ca-certificates file g++ libc6-dev make patch python
 # speed up build by using the following components from the OS
   git liblzma-dev libssl-dev pkg-config python-dev
-# for pygolang
-  python-greenlet-dev
 # extra requirements for NEO
   libnetfilter-queue-dev nftables
 # extra requirements for this SR

software/neoppod/stress-testing/stress.cfg

@@ -41,9 +41,6 @@ command =
   [versions]
   slapos.cookbook =
   NetfilterQueue = 0.8.1
-  greenlet = 0.4.15
-  gevent = 1.3.7
-  cffi = 1.11.5
   # use the following components from the OS
   # and don't build dependencies for nothing
   [git]

software/nextcloud/buildout.hash.cfg

@@ -26,4 +26,4 @@ md5sum = 6f42f0a8c5e5c0c657541a65c4d9ee57
 
 [template-nextcloud-instance]
 filename = nextcloud-instance.cfg.in
-md5sum = 86a92f542e516ac92802908b85354073
+md5sum = 05f946a6523677e5dcf80e9fad230d1c

software/nextcloud/nextcloud-instance.cfg.in

@@ -28,7 +28,7 @@ depend  =
 
 [redis-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = redis.py
 config-command = ${service-redis:promise_wrapper}
 
@@ -106,7 +106,7 @@ depends =
 
 [nextcloud-install-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = check-nextcloud-install.py
 config-command = ${nc-install-wrapper:output}
 

software/nginx-push-stream/buildout.hash.cfg

 [template]
 filename = instance.cfg.in
-md5sum = f9b6d01e29f2edddd9d6f99591976c33
+md5sum = 56e986c74ef236f261834c57f5861ce0
 
 [template-nginx-configuration]
 filename = template-nginx.cfg.in

software/nginx-push-stream/instance.cfg.in

@@ -75,7 +75,7 @@ promises =
 
 [nginx-available-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = $${:_buildout_section_name_}.py
 config-url = $${nginx-configuration:base-url}/status
 

software/plantuml/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = c962079a88a6ce97d8ce20fa4e8edfd1
+md5sum = de38ed0348a9d50e01dbf383a661d53e
 
 [tomcat-server-xml]
 filename = server.xml.in

software/plantuml/instance.cfg.in

@@ -87,7 +87,7 @@ instance-promises =
 
 [tomcat-listen-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = $${:_buildout_section_name_}.py
 config-host = $${tomcat-instance:ip}
 config-port = $${tomcat-instance:port}

software/powerdns/buildout.hash.cfg

@@ -18,15 +18,15 @@ md5sum = fddea033e1aa9d6147a1a47bd7cc4b62
 
 [template-powerdns]
 filename = instance-powerdns.cfg
-md5sum = c04c3b490e7f9f35af3d204a9df51f35
+md5sum = a6fcfcef942cd9b57c2b0c69e318362c
 
 [template-pdns-configuration]
 _update_hash_filename_ = template/pdns.conf.jinja2
-md5sum = 20c37ea06a8fa405bc02470d5115fd11
+md5sum = 851353e1d4dd562ace58b3345c2da515
 
 [template-dns-replicate]
 _update_hash_filename_ = instance-powerdns-replicate.cfg.jinja2
-md5sum = 4ff993a39da03d9d66d7c0f98efeb1e0
+md5sum = 5b4b46136c6547c27508c4789ac5d0ee
 
 [iso-list]
 _update_hash_filename_ = template/zz.countries.nexedi.dk.rbldnsd
@@ -34,4 +34,4 @@ md5sum = c4dc8c141d81b92d92cdb82ca67a13ee
 
 [template-zones-file]
 _update_hash_filename_ = template/zones-file.yml.jinja2
-md5sum = 612de569ac3d1e8cc10b830683ff92ae
+md5sum = 1fab79102f296a1259ce4ac9d054be9f

software/powerdns/instance-powerdns-replicate.cfg.jinja2

@@ -66,14 +66,14 @@ name = {{dns_name}}
 state = {{ slapparameter_dict.pop(state_key) }}
 {%   endif%}
 config-supported-zone-list = {{ ' '.join(supported_zone_list) }}
-config-soa = {{ "%s,%s" % (dns_domain, server_admin) }}
+config-soa = {{ "%s,%s,0,10800,3600,604800,3600" % (dns_domain, server_admin) }}
 {%   for parameter in sla_parameters -%}
 sla-{{ parameter }} = {{ slapparameter_dict.pop( sla_key + parameter ) }}
 {%   endfor -%}
 
 [{{promise_section_title}}]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = {{promise_section_title}}.py
 config-host = {{ '${' ~ request_section_title ~ ':connection-powerdns-ipv6}' }}
 config-port = {{ '${' ~ request_section_title ~ ':connection-powerdns-port}' }}

software/powerdns/instance-powerdns.cfg

@@ -137,7 +137,7 @@ extra-context =
 # Promises
 [pdns-promise-listen-port]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = pdns-port-listening.py
 config-host = $${pdns:ipv4}
 config-port = $${pdns:port}

software/powerdns/template/pdns.conf.jinja2

 # -------------------------------------------------------------------------
 
 # Configure ip/port binding
-local-address={{ pdns.get('ipv4') }}
-local-ipv6={{ pdns.get('ipv6') }}
+local-address={{ pdns.get('ipv4') }}, {{ pdns.get('ipv6') }}
 local-port={{ pdns.get('port') }}
 
 
@@ -48,4 +47,4 @@ edns-subnet-processing=yes
 geoip-database-files={{ geo['database'] }}
 geoip-zones-file={{ geo['zones-file'] }}
 
-# -------------------------------------------------------------------------
\ No newline at end of file
+# -------------------------------------------------------------------------

software/powerdns/template/zones-file.yml.jinja2

@@ -22,7 +22,7 @@ domains:
   # Note: For each domain, one record of the domain name MUST exist with a soa record.
   records:
     {{ zone }}:
-      - soa: {{ slapparameter_dict.get('soa', 'ns0.example.com,admin@example.com').replace(',', ' ') }}
+      - soa: {{ slapparameter_dict.get('soa', 'ns0.example.com,admin@example.com,0,10800,3600,604800,3600').replace(',', ' ') }}
       {%- for ns in slapparameter_dict.get('ns-record', 'ns0.example.com,ns1.example.com').split(',') %}
       - ns: {{ ns }}
       {%- endfor %}

software/proftpd/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = efb4238229681447aa7fe73898dffad4
 
 [instance-default]
 filename = instance-default.cfg.in
-md5sum = c6dce31a36e4e13de62687e9888aeb77
+md5sum = f6c583d24940a3a6838bd421dbb84a20
 
 [proftpd-config-file]
 filename = proftpd-config-file.cfg.in

software/proftpd/instance-default.cfg.in

@@ -86,7 +86,7 @@ template = inline:{{ slapparameter_dict['ssh-key'] | indent }}
 
 [proftpd-listen-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = ${:_buildout_section_name_}.py
 config-host = ${proftpd:ipv6}
 config-port = ${proftpd:sftp-port}

software/proftpd/test/test.py

@@ -31,6 +31,7 @@ from urllib.parse import urlparse, parse_qs
 import tempfile
 import io
 import subprocess
+import time
 from http.server import BaseHTTPRequestHandler
 import logging
 
@@ -152,7 +153,12 @@ class TestSFTPOperations(ProFTPdTestCase):
 
       with self.assertRaises(IOError):
         sftp.sftp_client.putfo(ErrorFile(), "destination")
+
     # no half uploaded file is kept
+    for _ in range(10):
+      if not os.listdir(self.upload_dir):
+        break
+      time.sleep(0.1)
     self.assertEqual([], os.listdir(self.upload_dir))
 
   def test_user_cannot_escape_home(self):

software/pureftpd/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [instance-profile]
 filename = instance.cfg.in
-md5sum = 500b773d1a63a6a895f9b8038a582b05
+md5sum = 9c4336f1f5143d3281c6706ff14abdd3

software/pureftpd/instance.cfg.in

@@ -33,7 +33,7 @@ pureftpd-dir = ${:srv}/pureftpd/
 
 [check-port-listening-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = check_nginx_port.py
 
 [pureftpd-listen-promise]

software/re6stnet/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 71531ed9c9b79fa769ab367e7ea2d2a5
 
 [template-re6stnet]
 filename = instance-re6stnet.cfg.in
-md5sum = 870c34cf58acaaee21c71182dd3cb0cf
+md5sum = 98f86d2a10d909215ae88ba6a602da27
 
 [template-apache-conf]
 filename = apache.conf.in

software/re6stnet/instance-re6stnet.cfg.in

@@ -170,14 +170,14 @@ context =
 
 [re6st-registry-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = re6st-registry.py
 config-host = ${re6st-registry:ipv4}
 config-port = ${re6st-registry:port}
 
 [apache-registry-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = apache-re6st-registry.py
 config-host = ${apache-conf:ipv6}
 config-port = ${apache-conf:port}

software/repman/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 8a08be95a04f1a47098c4fdef80bdfed
 
 [instance-repman.cfg]
 _update_hash_filename_ = instance-repman.cfg.jinja2.in
-md5sum = 839642d7a56447b3f08fa69729faca61
+md5sum = 697a1b546c883da45c14dbcd2d73b2b9
 
 [config-toml.in]
 _update_hash_filename_ = templates/config.toml.in
@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
 
 [template-mariadb.cfg]
 _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
-md5sum = 21a29a41768b2370d671d3086b3ef2bb
+md5sum = a5c204cac552754520aee0570d379723
 
 [template-my-cnf]
 _update_hash_filename_ = templates/my.cnf.in

software/repman/instance-mariadb.cfg.jinja2.in

@@ -327,13 +327,13 @@ dash = {{ dumps(dash) }}
 
 [{{ section('promise-check-computer-memory') }}]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = check-computer-memory.py
 config-command = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${monitor-instance-parameter:collector-db} --threshold "{{ slapparameter_dict["computer-memory-percent-threshold"] }}" --unit percent
 
 [{{ section('promise') }}]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = mariadb.py
 config-command = "{{ parameter_dict['bin-directory'] }}/is-local-tcp-port-opened" "{{ ip }}" "{{ port }}"
 

software/repman/instance-repman.cfg.jinja2.in

@@ -216,21 +216,21 @@ depends =
 
 [proxysql-{{ name }}-admin-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = proxysql-{{ name }}-admin-port-listening.py
 config-host =  {{ ipv4 }}
 config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-admin-port}' }}
 
 [proxysql-{{ name }}-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = proxysql-{{ name }}-port-listening.py
 config-host =  {{ ipv4 }}
 config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
 
 [proxysql-{{ name }}-ipv6-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = proxysql-{{ name }}-ipv6-port-listening.py
 config-host =  {{ ip }}
 config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
@@ -403,14 +403,14 @@ context =
 
 [repman-listen-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = repman_service_listen.py
 config-host = ${repman-parameter:ipv4}
 config-port = ${repman-parameter:port}
 
 [repman-listen-ssl-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = repman_service_ssl_listen.py
 config-host = ${repman-parameter:ipv4}
 config-port = ${repman-parameter:secure-port}
@@ -508,13 +508,13 @@ return = domain secure_access
 
 [repman-frontend-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = check_repman_frontend.py
 config-url = https://${repman-frontend:connection-domain}
 
 [repman-backend-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = check_repman_backend.py
 config-url = ${nginx-parameter:backend-ssl-url}
 

software/seleniumserver/buildout.hash.cfg

@@ -19,4 +19,4 @@ md5sum = 0084214fae4ee1aad2c878aa393757af
 
 [template-selenium]
 filename = instance-selenium.cfg.in
-md5sum = 884196ea35de35fa9159517912441ce6
+md5sum = 35ba19f7cb4fe7fc9469611f2446c94e

software/seleniumserver/instance-selenium.cfg.in

@@ -283,7 +283,7 @@ instance-promises =
 
 [check-port-listening-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = $${:_buildout_section_name_}.py
 
 [sshd-listen-promise]
@@ -304,7 +304,7 @@ config-port = $${selenium-server-hub-instance:port}
 # Promise waiting for all nodes to be registered
 [selenium-server-hub-nodes-registered-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = $${:_buildout_section_name_}.py
 config-command =
   $${selenium-server-check-nodes-registered:rendered} $${selenium-server-hub-instance:api-url} $${:expected-node-count}

software/seleniumserver/test/test.py

@@ -337,8 +337,8 @@ class TestFrontend(WebServerMixin, SeleniumServerTestCase):
 class TestSSHServer(SeleniumServerTestCase):
   @classmethod
   def getInstanceParameterDict(cls):
-    cls.ssh_key = paramiko.RSAKey.generate(1024)
-    return {'ssh-authorized-key': 'ssh-rsa {}'.format(cls.ssh_key.get_base64())}
+    cls.ssh_key = paramiko.ECDSAKey.generate(bits=384)
+    return {'ssh-authorized-key': 'ecdsa-sha2-nistp384 {}'.format(cls.ssh_key.get_base64())}
 
   def test_connect(self):
     parameter_dict = self.computer_partition.getConnectionParameterDict()

software/slapos-in-partition/README.rst

-XXX TODO

software/slapos-in-partition/common.cfg

-[buildout]
-extends =
-  ../../component/apache/buildout.cfg
-  ../../component/bash/buildout.cfg
-  ../../component/dcron/buildout.cfg
-  ../../component/dropbear/buildout.cfg
-  ../../component/gzip/buildout.cfg
-  ../../component/logrotate/buildout.cfg
-  ../../stack/slapos.cfg
-
-parts =
-  instance-profile
-  slapos-cookbook
-  eggs
-
-# Add hosting location of testing version of slapos.core
-find-links +=
-  http://www.nexedi.org/static/packages/source/slapos.core-testing/
-
-[environment]
-recipe = collective.recipe.environment
-
-
-[instance-profile]
-# 3 advantages of using jinja2 for ALL templates:
-# 1/ Explicit scope (pythonic style, we explicitely list what we want to be in the scope)
-# 2/ No troubles between $ and $$ (more simple)
-# 3/ We can explicitely define the path of executables (i.e
-#    in software, define httpd-executable = ${apache:location}/bin/httpd
-#    and in instance, just use httpd-executable without bother where it is actually
-#    (location can change inside of the component, from bin to sbin for example).
-recipe = slapos.recipe.template:jinja2
-template = ${:_profile_base_location_}/instance.cfg.jinja2
-rendered = ${buildout:directory}/instance.cfg
-#md5sum = 4861be4a581686feef9f9edea865d7ee
-mode = 0644
-context =
-    key bin_directory buildout:bin-directory
-    key develop_eggs_directory buildout:develop-eggs-directory
-    key eggs_directory buildout:eggs-directory
-    key path environment:PATH
-    raw httpd_executable ${apache:location}/bin/httpd
-    raw bash_executable ${bash:location}/bin/bash
-    raw dcron_executable ${dcron:location}/sbin/crond
-    raw dropbear_executable ${dropbear:location}/sbin/dropbear
-    raw dropbearkey_executable ${dropbear:location}/bin/dropbearkey
-    raw gzip_executable ${gzip:location}/bin/gzip
-    raw gunzip_executable ${gzip:location}/bin/gunzip
-    raw logrotate_executable ${logrotate:location}/usr/sbin/logrotate
-    raw slapos_configuration_file_template_path ${slapos-configuration-file-template:target}
-    raw httpd_configuration_file_template_path ${httpd-configuration-file-template:target}
-
-[slapos-configuration-file-template]
-# Download the template of slapos.cfg
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/template/slapos.cfg.in
-#md5sum = 
-target = ${buildout:directory}/slapos.cfg.in
-mode = 0644
-
-
-[httpd-configuration-file-template]
-# Download the template of httpd.conf
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/template/httpd.conf.in
-mode = 0644
-#md5sum = 
-#target = ${
-
-[eggs]
-recipe = zc.recipe.egg
-eggs =
-  collective.recipe.template

software/slapos-in-partition/instance.cfg.jinja2

-[buildout]
-parts =
-  slapos-configuration-file
-  cron-entry-slapos
-  slapos-node-status-wrapper
-  slapos-node-format-wrapper-script
-
-  httpd-wrapper
-
-  cron
-  logrotate
-  logrotate-entry-httpd
-  logrotate-entry-slapos
-
-  sshkeys-dropbear
-  dropbear-server-add-authorized-key
-  sshkeys-authority
-  publish-connection-informations
-
-  dropbear-promise
-  httpd-promise
-  slapos-promise
-
-
-eggs-directory = {{ eggs_directory }}
-develop-eggs-directory = {{ develop_eggs_directory }}
-offline = true
-
-[instance-parameter]
-recipe = slapos.cookbook:slapconfiguration
-computer = ${slap_connection:computer_id}
-partition = ${slap_connection:partition_id}
-url = ${slap_connection:server_url}
-key = ${slap_connection:key_file}
-cert = ${slap_connection:cert_file}
-
-configuration.master-url = https://slap.vifib.com
-configuration.authorized-key =
-
-# Create all needed directories
-[directory]
-recipe = slapos.cookbook:mkdirectory
-mode = 0750
-
-etc = ${buildout:directory}/etc/
-var = ${buildout:directory}/var/
-srv = ${buildout:directory}/srv/
-bin = ${buildout:directory}/bin/
-
-sshkeys = ${:srv}/sshkeys
-service = ${:etc}/service/
-script = ${:etc}/run/
-ssh = ${:etc}/ssh/
-log = ${:var}/log/
-run = ${:var}/run/
-backup = ${:srv}/backup/
-promises = ${:etc}/promise/
-
-slapos-partitions-certificate-repository = ${:var}/pki
-software-root = ${:srv}/slapos-software
-instance-root = ${:srv}/slapos-instance
-slapos-log = ${:log}/slapos
-
-{% for i in range(0,10) %}
-slappart{{i}} = ${:instance-root}/slappart{{i}}
-{% endfor %}
-
-cron-entries = ${:etc}/cron.d
-crontabs = ${:etc}/crontabs
-cronstamps = ${:etc}/cronstamps
-
-logrotate-entries = ${:etc}/logrotate.d
-logrotate-backup = ${:backup}/logrotate
-
-httpd-log = ${:log}/httpd
-
-
-########
-# Deploy slapos.cfg, computer certificates and slapos node wrapper
-########
-[slapos-computer-certificate-file]
-recipe = collective.recipe.template
-input = inline:${instance-parameter:configuration.computer-certificate}
-output = ${directory:var}/slapos-computer.crt
-
-[slapos-computer-key-file]
-recipe = collective.recipe.template
-input = inline:${instance-parameter:configuration.computer-key}
-output = ${directory:var}/slapos-computer.key
-
-[computer-definition-file]
-recipe = collective.recipe.template
-input = inline:
-  [computer]
-{% for i in range(0,10|int) %}
-  [partition_{{i}}]
-  address = ${instance-parameter:ipv4-random}/255.255.255.0 ${instance-parameter:ipv6-random}/64
-  pathname = slappart{{i}}
-  user = dummy
-  network_interface = dummy
-{% endfor %}
-output = ${directory:etc}/slapos-computer-definition.cfg
-
-[slapos-configuration-file]
-recipe = slapos.recipe.template
-url = {{ slapos_configuration_file_template_path }}
-output = ${directory:etc}/slapos.cfg
-#md5sum = 4861be4a581686feef9f9edea865d7ee
-software-root = ${directory:software-root}
-instance-root = ${directory:instance-root}
-master-url = ${instance-parameter:configuration.master-url}
-computer-id = ${instance-parameter:configuration.computer-id}
-# XXX should be a parameter
-partition-amount = 10
-computer-definition-file = ${computer-definition-file:output}
-computer-xml = ${directory:var}/slapos.xml
-computer-key-file = ${slapos-computer-key-file:output}
-computer-certificate-file = ${slapos-computer-certificate-file:output}
-certificate-repository-path = ${directory:slapos-partitions-certificate-repository}
-
-[slapos-node-instance-wrapper]
-recipe = slapos.cookbook:wrapper
-command-line = {{ bin_directory }}/slapos node instance --cfg ${slapos-configuration-file:output} --pidfile ${directory:run}/slapos-instance.pid --logfile ${directory:slapos-log}/slapos-instance.log
-wrapper-path = ${directory:bin}/slapos-node-instance
-
-[slapos-node-software-wrapper]
-recipe = slapos.cookbook:wrapper
-command-line = {{ bin_directory }}/slapos node software --cfg ${slapos-configuration-file:output} --pidfile ${directory:run}/slapos-software.pid --logfile ${directory:slapos-log}/slapos-software.log
-wrapper-path = ${directory:bin}/slapos-node-software
-
-[slapos-node-report-wrapper]
-recipe = slapos.cookbook:wrapper
-command-line = {{ bin_directory }}/slapos node report --cfg ${slapos-configuration-file:output} --pidfile ${directory:run}/slapos-report.pid --logfile ${directory:slapos-log}/slapos-report.log
-wrapper-path = ${directory:bin}/slapos-node-report
-
-[slapos-node-status-wrapper]
-recipe = slapos.cookbook:wrapper
-command-line = {{ bin_directory }}/slapos node status --cfg ${slapos-configuration-file:output}
-wrapper-path = ${directory:bin}/slapos-node-status
-
-[slapos-node-format-wrapper]
-recipe = slapos.cookbook:wrapper
-command-line = {{ bin_directory }}/slapos node format --cfg ${slapos-configuration-file:output} --logfile=${directory:slapos-log}/slapos-node-format.log --now
-wrapper-path = ${directory:bin}/slapos-node-format
-
-[slapos-node-format-wrapper-script]
-# Create a wrapper of the wrapper in etc/run
-recipe = collective.recipe.template
-input = inline:#!{{ bash_executable }}
-  false
-  while [ ! $? -eq 0 ]; do
-    ${slapos-node-format-wrapper:wrapper-path}
-  done
-output = ${directory:script}/slapos-node-format
-mode = 700
-
-
-#########
-# Deploy some http server to see logs online
-#########
-# XXX could it be something lighter?
-[httpd-configuration-file]
-recipe = slapos.recipe.template
-url = {{ httpd_configuration_file_template_path }}
-output = ${directory:etc}/httpd.conf
-# md5sum =
-listening-ip = ${instance-parameter:ipv6-random}
-listening-port = 8080
-htdocs = ${directory:log}
-pid-file = ${directory:run}/httpd.pid
-access-log = ${directory:httpd-log}/access-log
-error-log = ${directory:httpd-log}/error-log
-document-root = ${directory:log}
-
-# XXX logrotate for httpd
-
-[httpd-wrapper]
-recipe = slapos.cookbook:wrapper
-apache-executable = {{ httpd_executable }}
-command-line = ${:apache-executable} -f ${httpd-configuration-file:output} -DFOREGROUND
-wrapper-path = ${directory:service}/httpd
-# generated parameter containing url to use for other sections
-url = http://[${httpd-configuration-file:listening-ip}]/
-
-
-#########
-# Deploy logrotate
-#########
-[logrotate]
-recipe = slapos.cookbook:logrotate
-# Binaries
-logrotate-binary = {{ logrotate_executable }}
-gzip-binary = {{ gzip_executable }}
-gunzip-binary = {{ gunzip_executable }}
-# Directories
-wrapper = ${directory:bin}/logrotate
-conf = ${directory:etc}/logrotate.conf
-logrotate-entries = ${directory:logrotate-entries}
-backup = ${directory:logrotate-backup}
-state-file = ${directory:srv}/logrotate.status
-
-[logrotate-entry-httpd]
-<= logrotate
-recipe = slapos.cookbook:logrotate.d
-name = httpd
-log = ${httpd-configuration-file:access-log} ${httpd-configuration-file:error-log}
-frequency = daily
-rotate-num = 30
-post = {{ bin_directory }}/killpidfromfile $${apache-configuration:pid-file} SIGUSR1
-sharedscripts = true
-notifempty = true
-create = true
-
-[logrotate-entry-slapos]
-<= logrotate
-recipe = slapos.cookbook:logrotate.d
-name = slapos
-log = ${directory:slapos-log}/*.log
-frequency = daily
-rotate-num = 30
-#post = {{ bin_directory }}/killpidfromfile ${nginx-configuration:pid-file} SIGUSR1
-sharedscripts = true
-notifempty = true
-create = true
-
-###########
-# Deploy cron and configure it
-###########
-[cron-simplelogger]
-recipe = slapos.cookbook:simplelogger
-wrapper = ${directory:bin}/cron_simplelogger
-log = ${directory:log}/crond.log
-
-[cron]
-recipe = slapos.cookbook:cron
-dcrond-binary = {{ dcron_executable }}
-cron-entries = ${directory:cron-entries}
-crontabs = ${directory:crontabs}
-cronstamps = ${directory:cronstamps}
-catcher = ${cron-simplelogger:wrapper}
-binary = ${directory:service}/crond
-
-[cron-entry-slapos]
-recipe = collective.recipe.template
-# Add current PATH to environment, otherwise, gcc is not able to find its own cc1.
-# We don't add it in the top of the script, because dcron disallow it.
-# XXX: maybe it works if we take PATH from instance, not software.
-input = inline:
-  * * * * * PATH={{ path }} ${slapos-node-instance-wrapper:wrapper-path} > /dev/null 2>&1
-  * * * * * PATH={{ path }} ${slapos-node-software-wrapper:wrapper-path} > /dev/null 2>&1
-  * * * * * PATH={{ path }} ${slapos-node-report-wrapper:wrapper-path} > /dev/null 2>&1
-output = ${directory:cron-entries}/slapos
-
-[cron-entry-logrotate]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = logrotate
-frequency = 0 0 * * *
-command = $${logrotate:wrapper}
-
-
-
-# XXX what to do for slapformat?
-
-#########
-# Deploy dropbear (minimalist SSH server)
-#########
-[sshkeys-directory]
-recipe = slapos.cookbook:mkdirectory
-requests = ${directory:sshkeys}/requests/
-keys = ${directory:sshkeys}/keys/
-
-[sshkeys-authority]
-recipe = slapos.cookbook:sshkeys_authority
-request-directory = ${sshkeys-directory:requests}
-keys-directory = ${sshkeys-directory:keys}
-wrapper = ${directory:service}/sshkeys_authority
-keygen-binary = {{ dropbearkey_executable }}
-
-[dropbear-server]
-recipe = slapos.cookbook:dropbear
-host = ${instance-parameter:ipv6-random}
-port = 2222
-home = ${directory:ssh}
-wrapper = ${directory:bin}/raw_sshd
-shell = {{ bash_executable }}
-rsa-keyfile = ${directory:ssh}/server_key.rsa
-dropbear-binary = {{ dropbear_executable }}
-
-[sshkeys-dropbear]
-<= sshkeys-authority
-recipe = slapos.cookbook:sshkeys_authority.request
-name = dropbear
-type = rsa
-executable = ${dropbear-server:wrapper}
-public-key = ${dropbear-server:rsa-keyfile}.pub
-private-key = ${dropbear-server:rsa-keyfile}
-wrapper = ${directory:service}/sshd
-
-[dropbear-server-add-authorized-key]
-<= dropbear-server
-recipe = slapos.cookbook:dropbear.add_authorized_key
-key = ${instance-parameter:configuration.authorized-key}
-
-
-#########
-# Send informations to SlapOS Master
-#########
-[publish-connection-informations]
-recipe = slapos.cookbook:publish
-log-viewer-url = http://[${httpd-configuration-file:listening-ip}]:${httpd-configuration-file:listening-port}
-ssh_command = ssh ${dropbear-server:host} -p ${dropbear-server:port}
-
-#########
-# Deploy promises scripts
-#########
-[dropbear-promise]
-recipe = slapos.cookbook:check_port_listening
-path = ${directory:promises}/dropbear
-hostname = ${dropbear-server:host}
-port = ${dropbear-server:port}
-
-[httpd-promise]
-recipe = slapos.cookbook:check_port_listening
-path = ${directory:promises}/httpd
-hostname = ${httpd-configuration-file:listening-ip}
-port = ${httpd-configuration-file:listening-port}
-
-[slapos-promise]
-recipe = collective.recipe.template
-input = inline:#!/{{ bash_executable }}
-  {{ bin_directory }}/slapgrid-supervisorctl ${slapos-configuration-file:output} status watchdog | grep RUNNING
-output = ${directory:promises}/slapos
-mode = 0700
-

software/slapos-in-partition/software.cfg

-# Production profile of slapos-in-partition
-# Exactly the same as common.cfg, but:
-# 1/ Use a defined set of Python eggs instead of using the latest available
-#    ones from Pypi, to ensure stability;
-# 2/ Define list of trusted certificates for the cache.
-
-[buildout]
-extends = common.cfg
-
-[versions]
-Pygments = 1.6
-collective.recipe.environment = 0.2.0
-collective.recipe.template = 1.10

software/slapos-in-partition/template/httpd.conf.in

-# Apache static configuration
-# Automatically generated
-
-# Basic server configuration
-PidFile "${:pid-file}"
-Listen [${:listening-ip}]:${:listening-port}
-ServerAdmin someone@email
-DefaultType text/plain
-TypesConfig conf/mime.types
-AddType application/x-compress .Z
-AddType application/x-gzip .gz .tgz
-
-DocumentRoot "${:document-root}"
-
-# Log configuration
-ErrorLog "${:error-log}"
-LogLevel warn
-LogFormat "%h %{REMOTE_USER}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
-LogFormat "%h %{REMOTE_USER}i %l %u %t \"%r\" %>s %b" common
-CustomLog "${:access-log}" common
-
-# Allow cross site scripting
-Header set Access-Control-Allow-Origin "*"
-
-# List of modules
-LoadModule unixd_module modules/mod_unixd.so
-LoadModule access_compat_module modules/mod_access_compat.so
-LoadModule authz_core_module modules/mod_authz_core.so
-LoadModule authz_host_module modules/mod_authz_host.so
-LoadModule log_config_module modules/mod_log_config.so
-LoadModule setenvif_module modules/mod_setenvif.so
-LoadModule version_module modules/mod_version.so
-LoadModule proxy_module modules/mod_proxy.so
-LoadModule proxy_http_module modules/mod_proxy_http.so
-LoadModule mime_module modules/mod_mime.so
-LoadModule negotiation_module modules/mod_negotiation.so
-LoadModule rewrite_module modules/mod_rewrite.so
-LoadModule headers_module modules/mod_headers.so
-LoadModule dir_module modules/mod_dir.so
-LoadModule alias_module modules/mod_alias.so
-LoadModule ssl_module modules/mod_ssl.so
-LoadModule autoindex_module modules/mod_autoindex.so
-
-<Directory />
-    Options Indexes FollowSymLinks
-    IndexOptions FancyIndexing
-    order allow,deny
-    Allow from All
-</Directory>

software/slapos-in-partition/template/slapos.cfg.in

-[slapos]
-software_root = ${:software-root}
-instance_root = ${:instance-root}
-master_url = ${:master-url}
-
-key_file = ${:computer-key-file}
-cert_file = ${:computer-certificate-file}
-certificate_repository_path = ${:certificate-repository-path}
-
-computer_id = ${:computer-id}
-maximal_delay = 0
-
-# Don't check if we are using root
-root_check = false
-
-[slapformat]
-alter_user = false
-alter_network = false
-input_definition_file = ${:computer-definition-file}
-computer_xml = ${:computer-xml}
-partition_amount = ${:partition-amount}
-create_tap = false

software/slapos-master/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 84f099cc9852c4f53a075dccbb3880f0
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = c7c0bb9abbd0f8cc6c7956d83a61c4b3
+md5sum = f565956476c31881b6e51ae1c27793ad
 
 [template-apache-backend-conf]
 filename = apache-backend.conf.in

software/slapos-master/instance-balancer.cfg.in

@@ -203,7 +203,7 @@ input = inline:
 [{{ section('apache-promise') }}]
 # Check any apache port in ipv4, expect other ports and ipv6 to behave consistently
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = apache.py
 config-host = {{ ipv4 }}
 config-port = {{ apache_dict.values()[0][0] }}
@@ -297,13 +297,13 @@ promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
 
 [{{ section('monitor-promise-apachedex-result') }}]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = check-apachedex-result.py
 config-command = "{{ parameter_dict['promise-check-apachedex-result'] }}" --apachedex_path "${directory:apachedex}" --status_file ${monitor-directory:private}/apachedex.report.json --threshold "${apachedex-parameters:promise-threshold}"
 
 [{{ section('promise-check-computer-memory') }}]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = check-computer-memory.py
 config-command = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${monitor-instance-parameter:collector-db} --threshold "{{ slapparameter_dict["computer-memory-percent-threshold"] }}" --unit percent
 

software/slapos-sr-testing/README.md

@@ -2,7 +2,7 @@
 
 This software release is used to run integration test of slapos softwares.
 
-The approach is to use setuptools' integrated test runner, `python setup.py test`, to run tests.
+The approach is to use python unittest runner to run tests.
 
 The `python` used in this command will be a `zc.recipe.egg` interpreter with
 all eggs pre-installed by this software release.
@@ -17,6 +17,7 @@ changes to the code, run tests and publish changes.
 
 ```bash
 # install this software release and request an instance
+# use software-py3.cfg instead of software.cfg if the SR you want to test is written in Python 3
 SR=https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/slapos-sr-testing/software.cfg
 COMP=slaprunner
 INSTANCE_NAME=$COMP
@@ -32,7 +33,7 @@ slapos request --node=computer_guid=$COMP $INSTANCE_NAME $SR
 source ( environment-script from step above )
 
 # The source code is a git clone working copy on the instance
-cd ~/srv/runner/instance/slappartXXX/parts/slapos/
+cd ~/srv/runner/instance/slappartXXX/software_release/parts/slapos-repository/
 
 # change directory to the directory containing test for this software
 cd ./software/helloworld/test/
@@ -44,22 +45,25 @@ SLAPOS_TEST_DEBUG=1 python_for_test -m unittest discover -v
 
 ## Environment variables
 
-The `environment-script` set all variabels except `SLAPOS_TEST_DEBUG` and `SLAPOS_TEST_VERBOSE` for you, but for reference, here is the list of variables which control the test runner:
+The `environment-script` set all variables except `SLAPOS_TEST_DEBUG`, `SLAPOS_TEST_VERBOSE`, `SLAPOS_TEST_SKIP_SOFTWARE_CHECK` and `SLAPOS_TEST_SKIP_SOFTWARE_REBUILD` for you, but for reference, here is the list of variables which control the test runner:
 
 | Variable | Description |
 | --- | --- |
+| `SLAPOS_TEST_VERBOSE` | If set to 1, debugging output will be printed on console. This also adjust the log level of python process running tests. When running on test nodes, this is not set, so keep this difference in mind if test rely on python logger |
+| `SLAPOS_TEST_DEBUG` | If set to 1, `slapos node instance` and `slapos node software` will run with `--buildout-debug` flag, which will invoke python debugger on error. |
+| `SLAPOS_TEST_SKIP_SOFTWARE_CHECK` | If set to 1, software checks will not be performed. This can be used to speed up running tests locally, as checking software is an expensive operation. |
+| `SLAPOS_TEST_SKIP_SOFTWARE_REBUILD` | If set to 1, software will not be rebuilt before running the test. The default is to use `slapos node software --all`, which unconditionally rebuild all softwares and to use `slapos node software`, which installs the software only if it's not yet available when the environment variable is set to 1. |
 | `SLAPOS_TEST_IPV6` | ipv6 used by this instance. Usually you want to use a global address here to be able to connect to this instance. |
 | `SLAPOS_TEST_IPV4` | ipv4 used by this instance. |
 | `SLAPOS_TEST_WORKING_DIR` | Path to use as a working directory to hold the standalone SlapOS. |
 | `SLAPOS_TEST_SHARED_PART_LIST` | A `:` separated of paths to look for already installed shared parts. The SlapOS used in the test will not write in these, but will use a dedicated directory in `$SLAPOS_TEST_WORKING_DIR` |
-| `SLAPOS_TEST_VERBOSE` | If set to 1, debugging output will be printed on console. This also adjust the log level of python process running tests. When running on test nodes, this is not set, so keep this difference in mind if test rely on python logger |
-| `SLAPOS_TEST_DEBUG` | If set to 1, `slapos node instance` and `slapos node software` will run with `--buildout-debug` flag, which will invoke python debugger on error. |
+
 
 ## Frequently Asked Questions
 
 ### Where to find docs about the testing framework ?
 
-Please refere to the docstrings from `slapos.testing` module, from `slapos.core` package.
+Please refer to the docstrings from `slapos.testing` module, from `slapos.core` package.
 
 This uses python unittest module from standard library, especially the setup hooks:
  - `setUpModule` installs the software and perform some static checks
@@ -69,7 +73,7 @@ This uses python unittest module from standard library, especially the setup hoo
 ### Can I run slapos commands to debug ?
 
 The standalone slapos is created in `$SLAPOS_TEST_WORKING_DIR`. In this directory you will have a `bin/slapos` that you can run to start or stop services.
-It's fine to use this command during development, but to programatically interract with the environment within the test, the recommended approach is to use supervisor XML-RPC API.
+It's fine to use this command during development, but to programmatically interact with the environment within the test, the recommended approach is to use supervisor XML-RPC API.
 
 ### How to use a development version of `slapos.cookbook` ?
 
@@ -82,3 +86,7 @@ At the end of the test, a snapshot of the slapos instances is created. Sometimes
 
 Most of the time, problems are because on test nodes paths are very long. One advanced technique to reproduce the problem in your development environment is to set `SLAPOS_TEST_WORKING_DIR` environment variable to a path with the same length as the ones on test nodes.
 One way to make instances uses a slightly shorter path is to define `__partition_reference__` class attribute, so that the instances uses this as prefix instead of the class name.
+
+### Can I run only specific test ?
+
+Yes, please refer to python unittest documentation. For example, you can use `python_for_test -m unittest -v test_module.TestClass.test_function` to run only test function.

software/slapos-sr-testing/software.cfg

@@ -337,6 +337,7 @@ tests =
   theia ${slapos.test.theia-setup:setup}
   metabase ${slapos.test.metabase-setup:setup}
   nginx-push-stream ${slapos.test.nginx-push-stream-setup:setup}
+  erp5 ${slapos.test.erp5-setup:setup}
 ###
   ${:extra}
 
@@ -345,7 +346,6 @@ extra =
 #          You should not add more lines here.
   backupserver ${slapos.test.backupserver-setup:setup}
   caddy-frontend ${slapos.test.caddy-frontend-setup:setup}
-  erp5 ${slapos.test.erp5-setup:setup}
   upgrade_erp5 ${slapos.test.upgrade_erp5-setup:setup}
   htmlvalidatorserver ${slapos.test.htmlvalidatorserver-setup:setup}
   slapos-master ${slapos.test.slapos-master-setup:setup}

software/slaprunner/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 8d6878ff1d2e75010c50a1a2b0c13b24
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = 2a09b11c7dbade65d50e66287bf4c7b9
+md5sum = 384285ab789396b6e674a8125ce2d030
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2
@@ -26,7 +26,7 @@ md5sum = f2e2493bc5da90a53f86e5bcf64d2d57
 
 [instance-runner-import]
 filename = instance-runner-import.cfg.in
-md5sum = ea7667f9af952bc4bdf43aad4520759f
+md5sum = a4ebf6918a2c68c02898b2142357f490
 
 [instance-runner-export]
 filename = instance-runner-export.cfg.in

software/slaprunner/instance-runner-import.cfg.in

@@ -134,7 +134,7 @@ mode = 755
 
 [importer-consistency-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = importer-consistency-promise.py
 config-command = ${importer-consistency-promise-bin:output}
 
@@ -158,7 +158,7 @@ mode = 755
 
 [software-release-deployment-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = software-release-deployment-promise.py
 config-command =${software-release-deployment-bin:output}
 

software/slaprunner/instance-runner.cfg

@@ -87,7 +87,7 @@ return = site_url domain
 
 [custom-frontend-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = custom_frontend_promise.py
 config-url = https://$${request-custom-frontend:connection-domain}
 {% if slapparameter_dict.get('custom-frontend-basic-auth') -%}
@@ -111,7 +111,7 @@ template = inline:
 
 [custom-frontend-url-ready-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = custom_frontend_ready_promise.py
 config-command = $${custom-frontend-url-ready-promise-bin:rendered}
 
@@ -436,7 +436,7 @@ mode = 700
 
 [apache-httpd-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = $${:filename}.py
 filename = apache-httpd-listening-on-tcp
 config-url = $${apache-httpd:access-url}
@@ -537,7 +537,7 @@ return = site_url domain
 
 [slaprunner-frontend-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = slaprunner_frontend.py
 config-url = https://$${request-frontend:connection-domain}/login
 
@@ -556,7 +556,7 @@ return = secure_access domain
 
 [httpd-frontend-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = slaprunner-apache-http-frontend.py
 config-url = $${request-httpd-frontend:connection-secure_access}
 
@@ -619,14 +619,14 @@ monitor-password = $${monitor-publish-parameters:monitor-password}
 
 [slaprunner-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = slaprunner.py
 config-host = $${slaprunner:ipv6}
 config-port = $${slaprunner:runner_port}
 
 [runner-sshd-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = runner-sshd.py
 config-host = $${slap-network-information:global-ipv6}
 config-port = $${runner-sshd-port:port}
@@ -863,20 +863,20 @@ log = $${runnerdirectory:home}/instance/*/.slapgrid/log/instance.log $${runnerdi
 
 [supervisord-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = supervisord.py
 config-host = $${slaprunner:ipv4}
 config-port = $${supervisord:port}
 
 [slapos-supervisord-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = instance_supervisord.py
 config-command = ${buildout:bin-directory}/slapos node supervisorctl --cfg=$${slaprunner:slapos.cfg} pid
 
 [slapos-proxy-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = slaproxy.py
 config-host = $${slaprunner:ipv4}
 config-port = $${slaprunner:proxy_port}

software/slaprunner/test/test.py

@@ -339,9 +339,9 @@ class TestWeb(SlaprunnerTestCase):
 class TestSSH(SlaprunnerTestCase):
   @classmethod
   def getInstanceParameterDict(cls):
-    cls.ssh_key_list = [paramiko.RSAKey.generate(1024) for i in range(2)]
+    cls.ssh_key_list = [paramiko.ECDSAKey.generate(bits=384) for i in range(2)]
     return {
-        'user-authorized-key': 'ssh-rsa {}\nssh-rsa {}'.format(
+        'user-authorized-key': 'ecdsa-sha2-nistp384 {}\necdsa-sha2-nistp384 {}'.format(
           *[key.get_base64() for key in cls.ssh_key_list]
           )
     }
@@ -363,7 +363,7 @@ class TestSSH(SlaprunnerTestCase):
     username, fingerprint_from_url = ssh_info.split(';fingerprint=')
     client = paramiko.SSHClient()
 
-    self.assertTrue(fingerprint_from_url.startswith('ssh-rsa-'), '')
+    self.assertTrue(fingerprint_from_url.startswith('ssh-rsa-'), fingerprint_from_url)
     fingerprint_from_url = fingerprint_from_url[len('ssh-rsa-'):]
 
     class KeyPolicy(object):

software/test-slave-instance-deployment/instance-default.cfg

-[buildout]
-parts =
-  publish-connection-informations
-  request-slave-instance
-  check-connection-informations-promise
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
-offline = true
-
-[directories]
-recipe = slapos.cookbook:mkdirectory
-bin = $${buildout:directory}/bin
-etc = $${buildout:directory}/etc
-scripts = $${:etc}/run
-services = $${:etc}/service
-promises = $${:etc}/promise
-
-[slap-configuration]
-recipe = slapos.cookbook:slapconfiguration.serialised
-computer = $${slap-connection:computer-id}
-partition = $${slap-connection:partition-id}
-url = $${slap-connection:server-url}
-key = $${slap-connection:key-file}
-cert = $${slap-connection:cert-file}
-
-[request-test-instance]
-# Deploy a dummy instance
-<= slap-connection
-recipe = slapos.cookbook:request
-name = Test Root Instance
-software-url = $${slap-connection:software-release-url}
-software-type = test
-return = foo slave-instance-list
-
-[request-slave-instance]
-# Deploy a dummy instance
-<= slap-connection
-recipe = slapos.cookbook:request
-name = Test Root Instance - Slave
-software-url = $${slap-connection:software-release-url}
-software-type = test
-slave = true
-sla-instance_guid = $${request-test-instance:instance-guid}
-return = foo
-
-[publish-connection-informations]
-recipe = slapos.cookbook:publish
-note = test-instance-guid should be equal to XXX, test-instance-state should be equal to started, test-instance-slave-instance-list should be NOT empty, and root-instance-slave-instance-list SHOULD be empty, slave-instance-parameter should NOT be empty.
-test-instance-guid = $${request-test-instance:instance-guid}
-test-instance-state = $${request-test-instance:instance-state}
-test-instance-slave-instance-list = $${request-test-instance:connection-slave-instance-list}
-root-instance-slave-instance-list = $${slap-configuration:slave-instance-list}
-slave-instance-parameter = $${request-slave-instance:connection-foo}
-
-# Create a file containing connection parameters
-# Then create a promise that will check content of this file
-[dump-connection-informations]
-<= publish-connection-informations
-recipe = slapos.cookbook:jsondump
-json-output = $${buildout:directory}/connection-parameters.json
-
-[check-connection-informations-promise]
-recipe = collective.recipe.template
-input = ${promise-template:output}
-output = $${directories:promises}/checkparameters
-mode = 700

software/test-slave-instance-deployment/instance-test.cfg

-[buildout]
-parts =
-  publish-connection-informations
-  publish-slave-information
-
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
-offline = true
-
-[slap-configuration]
-recipe = slapos.cookbook:slapconfiguration
-computer = $${slap-connection:computer-id}
-partition = $${slap-connection:partition-id}
-url = $${slap-connection:server-url}
-key = $${slap-connection:key-file}
-cert = $${slap-connection:cert-file}
-
-[publish-connection-informations]
-recipe = slapos.cookbook:publish
-foo = bar
-slave-instance-list = $${slap-configuration:slave-instance-list}
-
-[publish-slave-information]
-recipe = slapos.cookbook:publish
-# XXX hardcoded
--slave-reference = slappart0_Test Root Instance - Slave
-foo = I am slave instance, and I am deployed!

software/test-slave-instance-deployment/instance.cfg

-[buildout]
-parts =
-  switch-softwaretype
-
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
-offline = true
-
-[switch-softwaretype]
-recipe = slapos.cookbook:softwaretype
-default = ${instance-default-profile:output}
-test = ${instance-test-profile:output}
-
-[slap-connection]
-# part to migrate to new - separated words
-computer-id = $${slap_connection:computer_id}
-partition-id = $${slap_connection:partition_id}
-server-url = $${slap_connection:server_url}
-software-release-url = $${slap_connection:software_release_url}
-key-file = $${slap_connection:key_file}
-cert-file = $${slap_connection:cert_file}

software/test-slave-instance-deployment/parameter-check-promise.py.in

-#!${buildout:executable}
-import sys
-import os
-import json
-
-json_file = '$${dump-connection-informations:json-output}'
-connection_parameter_list = json.loads(open(json_file, 'r').read())
-# XXX list all problems before exit
-# XXX: factor in a clean dict loop
-test_instance_guid = connection_parameter_list['test-instance-guid']
-if not test_instance_guid == 'slappart1':
-  print 'Bad test-instance-guid connection parameter: %s' % test_instance_guid
-  sys.exit(1)
-test_instance_state = connection_parameter_list['test-instance-state']
-if not test_instance_state == 'started':
-  print 'Bad test-instance-state connection parameter: %s instead of started' % test_instance_state
-  sys.exit(1)
-test_instance_slave_instance_list = connection_parameter_list['test-instance-slave-instance-list']
-if test_instance_slave_instance_list == []:
-  print 'test-instance-slave-instance-list connection parameter is empty.'
-  sys.exit(1)
-root_instance_slave_instance_list = connection_parameter_list['root-instance-slave-instance-list']
-if root_instance_slave_instance_list != []:
-  print 'root-instance-slave-instance-list connection parameter is not empty.'
-  sys.exit(1)
-slave_instance_parameter = connection_parameter_list['slave-instance-parameter']
-if slave_instance_parameter != 'I am slave instance, and I am deployed!':
-  print 'parameter slave-instance-parameter is not correct.'
-  sys.exit(1)

software/test-slave-instance-deployment/software.cfg

-# This Software Release is used to test
-# the behavior of Slave Instances in slapos and in slaprunner.
-[buildout]
-# Local development
-develop =
-  ${:parts-directory}/slapos.cookbook-repository
-extends =
-  ../../component/lxml-python/buildout.cfg
-  ../../component/git/buildout.cfg
-parts =
-  slapos.cookbook-repository
-  check-recipe
-  slapos-cookbook
-  instance-profile
-  collective-egg
-
-# Local development
-[slapos.cookbook-repository]
-recipe = slapos.recipe.build:gitclone
-repository = https://lab.nexedi.com/nexedi/slapos.git
-branch = master
-git-executable = ${git:location}/bin/git
-
-[check-recipe]
-recipe = plone.recipe.command
-stop-on-error = true
-update-command = ${:command}
-command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
-
-[instance-profile]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg
-output = ${buildout:directory}/instance.cfg
-mode = 0644
-
-[instance-default-profile]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-default.cfg
-output = ${buildout:directory}/instance-default.cfg
-mode = 0644
-
-[instance-test-profile]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-test.cfg
-output = ${buildout:directory}/instance-test.cfg
-mode = 0644
-
-[promise-template]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/parameter-check-promise.py.in
-output = ${buildout:directory}/parameter-check-promise.py
-mode = 0644
-
-[collective-egg]
-recipe = zc.recipe.egg
-eggs = collective.recipe.template
-
-[versions]
-collective.recipe.template = 1.11
-plone.recipe.command = 1.1
-slapos.recipe.build = 0.28
-
-# Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.
-[buildout]
-# Explicitly disable download-cache
-download-cache =
-
-# Separate from site eggs
-allowed-eggs-from-site-packages =
-include-site-packages = false
-exec-sitecustomize = false
-
-# Add location for modified non-official slapos.buildout
-find-links +=
-  http://www.nexedi.org/static/packages/source/slapos.buildout/
-  http://www.nexedi.org/static/packages/source/
-  http://www.nexedi.org/static/packages/source/hexagonit.recipe.download/
-
-# Use only quite well working sites.
-allow-hosts +=
-  *.googlecode.com
-  *.nexedi.org
-  *.python.org
-  *.sourceforge.net
-  alastairs-place.net
-  bitbucket.org
-  dist.repoze.org
-  effbot.org
-  github.com
-  launchpad.net
-  peak.telecommunity.com
-  sourceforge.net
-  www.dabeaz.com
-  www.owlfish.com
-
-# XXX: Workaround of SlapOS limitation
-# Unzippig of eggs is required, as SlapOS do not yet provide nicely working
-# development / fast switching environment for whole software
-unzip = true
-
-# Define where is defined what version of eggs we should use
-versions = versions
-
-# Define networkcache with shacache.org
-networkcache-section = networkcache
-
-# Install slapos.cookbook containing all officials recipes
-[slapos-cookbook]
-recipe = zc.recipe.egg
-eggs =
-  ${lxml-python:egg}
-  slapos.cookbook
-
-[versions]
-# Use SlapOS patched zc.buildout
-zc.buildout = 1.6.0-dev-SlapOS-013
-# zc.recipe.egg 2.x is for Buildout 2
-zc.recipe.egg = 1.3.2
-# Use own version of h.r.download to be able to open xz-like archives
-hexagonit.recipe.download = 1.7nxd002
-# Use pinned version of setuptools. Other versions work, but changing
-# version makes buildout recompile everything. Developers' nightmare.
-setuptools = 0.9.8
-# Official egg of prettytable has permission problems in EGG-INFO.
-prettytable = 0.7.3-nxd001
-
-[networkcache]
-download-cache-url = http://download.shacache.org
-download-dir-url = http://dir.shacache.org

software/theia/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance-theia]
 _update_hash_filename_ = instance-theia.cfg.jinja.in
-md5sum = e39925b69a8bc17d17be54c075ae2f88
+md5sum = f396d9a0780f4fb17016dbd32b56d7b8
 
 [instance]
 _update_hash_filename_ = instance.cfg.in
@@ -23,11 +23,11 @@ md5sum = a7d78b4002266c69ece05a476df82791
 
 [instance-import]
 _update_hash_filename_ = instance-import.cfg.jinja.in
-md5sum = 861ef130f27175c2978a9b946b138dd5
+md5sum = 57b707cf0ed83be1959d26a88c131906
 
 [instance-export]
 _update_hash_filename_ = instance-export.cfg.jinja.in
-md5sum = b3cedaa1603ca8ed83fdd94ef4b35cc8
+md5sum = 190a736471f0e0cffcb2838968e01d84
 
 [instance-resilient]
 _update_hash_filename_ = instance-resilient.cfg.jinja
@@ -47,7 +47,7 @@ md5sum = 9e8c17a4b2d802695caf0c2c052f0d11
 
 [yarn.lock]
 _update_hash_filename_ = yarn.lock
-md5sum = c65949e402ba7c47c0354e3c7438b094
+md5sum = 6faa52754c46e505912a478bc8ba3300
 
 [python-language-server-requirements.txt]
 _update_hash_filename_ = python-language-server-requirements.txt
@@ -59,7 +59,7 @@ md5sum = 8157c22134200bd862a07c6521ebf799
 
 [slapos.css.in]
 _update_hash_filename_ = slapos.css.in
-md5sum = d2930ec3ef973b7908f0fa896033fd64
+md5sum = 841141fc699b8d8918ed0669e6e61995
 
 [logo.png]
 _update_hash_filename_ = logo.png

software/theia/instance-export.cfg.jinja.in

@@ -69,7 +69,7 @@ export-promises =
 
 [export-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = resiliency-export-promise.py
 config-command = $${export-promise-script:rendered}
 

software/theia/instance-import.cfg.jinja.in

@@ -127,7 +127,7 @@ import-promises =
 
 [import-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = resiliency-import-promise.py
 config-command = $${import-promise-script:rendered}
 

software/theia/instance-theia.cfg.jinja.in

@@ -82,21 +82,21 @@ instance-promises =
 
 [theia-listen-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = $${:_buildout_section_name_}.py
 config-host = $${theia-instance:ip}
 config-port = $${theia-instance:port}
 
 [frontend-listen-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = $${:_buildout_section_name_}.py
 config-host = $${frontend-instance:ip}
 config-port = $${frontend-instance:port}
 
 [frontend-authentification-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = $${:_buildout_section_name_}.py
 username = $${frontend-instance-password:username}
 password = $${frontend-instance-password:passwd}
@@ -106,7 +106,7 @@ config-url = https://$${:username}:$${:password}@[$${:ip}]:$${:port}
 
 [remote-frontend-url-available-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = $${:_buildout_section_name_}.py
 config-url = $${remote-frontend:connection-secure_access}
 config-http-code = 401
@@ -114,7 +114,7 @@ config-http-code = 401
 {% if additional_frontend %}
 [remote-additional-frontend-url-available-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = $${:_buildout_section_name_}.py
 config-url = $${remote-additional-frontend:connection-secure_access}
 config-http-code = 401
@@ -122,7 +122,7 @@ config-http-code = 401
 
 [slapos-standalone-listen-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 # XXX promise plugins can not contain "slapos" in their names
 name = standalone-listen-promise.py
 config-host = $${slapos-standalone-instance:hostname}
@@ -130,13 +130,13 @@ config-port = $${slapos-standalone-instance:port}
 
 [slapos-standalone-ready-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = standalone-ready-promise.py
 config-abstract = $${directory:runner}/standalone_ready
 
 [slapos-autorun-promise]
 <= monitor-promise-base
-module = check_service_state
+promise = check_service_state
 name = autorun-state-promise.py
 config-service = $${slapos-autorun:service-name}
 config-expect = $${slapos-autorun:autorun}

software/theia/slapos.css.in

+/* backported fixes */
+/* https://github.com/eclipse-theia/theia/commit/616c34e1c446a706f4cb02182b2d9195ef3ea854 */
+.monaco-editor .monaco-list .monaco-list-row.focused,
+.monaco-editor .monaco-list .monaco-list-row.focused,
+.monaco-editor .monaco-list .monaco-list-row.focused .suggest-icon {
+  color: var(--theia-list-activeSelectionForeground) !important;
+  background-color: var(--theia-list-activeSelectionBackground) !important;
+}
+
 /* logo */
 .theia-icon {
     background-image: url('/{{ logo_image }}');

software/theia/software.cfg

@@ -362,6 +362,7 @@ template =
   inline:
   #!/bin/sh
   cd ${theia:location}
+  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${libsecret:location}/lib:${gettext:location}/lib:${glib:location}/lib"
   exec ${yarn:location}/bin/yarn theia start "$@"
 
 [theia-open]

software/theia/test/dummy/software.cfg

@@ -9,11 +9,12 @@ parts =
 
 [log-env]
 recipe = slapos.recipe.build
-init =
+path = ${buildout:directory}/../env.json
+update =
   import json
   import os
-  with open("${buildout:directory}/../env.json", 'w') as f:
+  with open(options['path'], 'w') as f:
     json.dump(dict(os.environ), f)
 
 [versions]
-slapos.recipe.build = 0.47
+slapos.recipe.build = 0.48

software/turnserver/buildout.hash.cfg

@@ -19,8 +19,8 @@ md5sum = b43d5e8d1fc2d0eeb54f91cefe6a5bae
 
 [template-turnserver]
 filename = instance-turnserver.cfg.jinja2.in
-md5sum = 7af3318d7249e9afe22436d9fe200159
+md5sum = 932c4d82faa8e28b62bfbfc3dfe31c02
 
 [template-insecure-turnserver]
 filename = instance-insecure-turnserver.cfg.jinja2.in
-md5sum = 3db65c3a16eb76ab438ac3817d1a5fea
+md5sum = 504f0f5ead8600b80ba43f828a0f82b6

software/turnserver/instance-insecure-turnserver.cfg.jinja2.in

@@ -57,7 +57,7 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [promise-check-turnserver-port]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = turnserver-port-listening.py
 config-host = {{ listening_ip }}
 config-port = {{ turn_port }}

software/turnserver/instance-turnserver.cfg.jinja2.in

@@ -123,14 +123,14 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [promise-check-turnserver-port]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = turnserver-port-listening.py
 config-host = {{ listening_ip }}
 config-port = {{ turn_port }}
 
 [promise-check-turnserver-tls-port]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = turnserver-tls-port-listening.py
 config-host = {{ listening_ip }}
 config-port = {{ turn_tls_port }}

software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion.py

@@ -3,7 +3,7 @@ request = portal.REQUEST
 reference = request['reference']
 data_chunk = request['data_chunk']
 
-module = portal.data_stream_module
+promise = portal.data_stream_module
 try:
   data_stream = module[reference]
 except KeyError:

stack/caucase/buildout.cfg

@@ -36,6 +36,6 @@ mode = 0644
 depends = ${caucase-jinja2-library-eggs:eggs}
 
 [versions]
-caucase = 0.9.10
+caucase = 0.9.12
 pem = 21.1.0
 PyJWT = 1.7.1

stack/caucase/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [caucase-jinja2-library]
 filename = caucase.jinja2.library
-md5sum = b3ee1414719f8a24790ae324ca023066
+md5sum = 1e3607e514320441ddccdb6d1a21f705

stack/caucase/caucase.jinja2.library

@@ -37,7 +37,7 @@ command-line = '{{ buildout_bin_directory }}/caucased'
 {% if promise -%}
 [{{ prefix }}-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = {{ prefix }}.py
 config-command = '{{ buildout_bin_directory }}/caucase-probe' 'http://{{ netloc }}'
 {%- endif %}
@@ -119,7 +119,7 @@ command-line = '{{ buildout_bin_directory }}/caucase-updater'
 {% if promise -%}
 [{{ prefix }}-promise]
 <= monitor-promise-base
-module = check_certificate
+promise = check_certificate
 name = {{ prefix }}.py
 config-certificate = {{ crt_path }}
 config-key = {{ key_path }}
@@ -133,7 +133,7 @@ recipe = plone.recipe.command
 checksum-file = '{{ template }}.md5'
 command =
   set -e
-  md5_current=$(${buildout:executable} -c "import hashlib ; print hashlib.md5(open('{{ template }}', 'rb').read()).hexdigest()")
+  md5_current=$(${buildout:executable} -c "from __future__ import print_function ; import hashlib ; print(hashlib.md5(open('{{ template }}', 'rb').read()).hexdigest())")
   md5_old=$([ -f ${:checksum-file} ] && cat ${:checksum-file} || echo none)
   if [ "$md5_current" != "$md5_old" ] || [ ! -f '{{ csr }}' ] || [ ! -f '{{ key }}' ] ; then
     '{{ buildout_bin_directory }}/caucase-rerequest' --template '{{ template }}' --csr '{{ csr }}' --key '{{ key }}'

stack/erp5/buildout.cfg

@@ -621,7 +621,7 @@ depends =
 Acquisition = 2.13.12+SlapOSPatched001
 Products.DCWorkflow = 2.2.4+SlapOSPatched001
 ocropy = 1.0+SlapOSPatched001
-pysvn = 1.7.10+SlapOSPatched002
+pysvn = 1.9.15+SlapOSPatched001
 python-ldap = 2.4.32+SlapOSPatched001
 python-magic = 0.4.12+SlapOSPatched001
 PyPDF2 = 1.26.0+SlapOSPatched001

stack/erp5/buildout.hash.cfg

@@ -26,11 +26,11 @@ md5sum = d10b8e35b02b5391cf46bf0c7dbb1196
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = 7d064777c1c4e7b275b255db4f4b1da9
+md5sum = c82ea00c4514b72fb97a6fa7ac36ec52
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
-md5sum = fed6dd2bdc389b4fc7e7b7ca32c5d4b6
+md5sum = cfe4696a67bf4886a5d8252a5274a941
 
 [template-zope-conf]
 filename = zope.conf.in
@@ -50,7 +50,7 @@ md5sum = 1102c3e37a5a2e8aa2d8a2607ab633c8
 
 [template-postfix]
 filename = instance-postfix.cfg.in
-md5sum = 2a68a3e7c5c509cbd4cfa9e670ac91c7
+md5sum = 0f666e5e7e52afda433feb9f02452717
 
 [template-postfix-master-cf]
 filename = postfix_master.cf.in
@@ -78,7 +78,7 @@ md5sum = fcc8470824c448a56e2282c43b870cb5
 
 [template-zeo]
 filename = instance-zeo.cfg.in
-md5sum = 79b6b422df512b5a075eba54a6895a01
+md5sum = 1f33f3b93da32b34e2fd11471648835d
 
 [template-zodb-base]
 filename = instance-zodb-base.cfg.in
@@ -86,15 +86,15 @@ md5sum = bc821f9f9696953b10a03ad7b59a1936
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = 58ca95f6e0c067702a03fc3be66d50c1
+md5sum = 769e81946c346530cebfce6ad7553165
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = c6c1b3e4b2f3c6f256153dcfe9fbecad
+md5sum = d6166515fda7b09df754672536b131be
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in
-md5sum = 3f4f7e49c504cbf610fc5dc462713dfc
+md5sum = 9d61e05c8578e0f17e349603ccaaf52c
 
 [template-rsyslogd-cfg]
 filename = rsyslogd.cfg.in
@@ -102,4 +102,4 @@ md5sum = 5cf0316fdd17a940031e4083bbededd8
 
 [instance-wcfs.cfg.in]
 filename = instance-wcfs.cfg.in
-md5sum = 945e8e4552a6bdf228b9609567b09399
+md5sum = eb4be2669a9a56187cc4366272e11d18

stack/erp5/haproxy.cfg.in

@@ -172,7 +172,7 @@ listen family_{{ name }}
   # logs
   capture request header Referer len 512
   capture request header User-Agent len 512
-  log-format "%{+Q}o %{-Q}ci - - [%trg] %r %ST %B %{+Q}[capture.req.hdr(0)] %{+Q}[capture.req.hdr(1)] %Tt"
+  log-format "%{+Q}o %{-Q}ci - - [%trg] %r %ST %B %{+Q}[capture.req.hdr(0)] %{+Q}[capture.req.hdr(1)] %Ta"
 
 {%   for outer_prefix, inner_prefix in family_path_routing_dict.get(name, []) + path_routing_list %}
   {%   set outer_prefix = outer_prefix.strip('/') -%}

stack/erp5/instance-balancer.cfg.in

@@ -272,7 +272,7 @@ hash-files = ${rsyslogd-cfg:rendered}
 
 [{{ section ('rsyslogd-listen-promise') }}]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = rsyslogd_listen_promise.py
 config-command = test -S ${rsyslogd-cfg-parameter-dict:log-socket}
 
@@ -303,7 +303,7 @@ certificate-and-key = ${directory:etc}/certificate-and-key-generated.pem
 [{{ section('haproxy-promise') }}]
 <= monitor-promise-base
 # Check any haproxy port in ipv4, expect other ports and ipv6 to behave consistently
-module = check_socket_listening
+promise = check_socket_listening
 name = haproxy.py
 config-host = {{ ipv4 }}
 config-port = {{ haproxy_dict.values()[0][0] }}
@@ -382,13 +382,13 @@ promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
 
 [{{ section('monitor-promise-apachedex-result') }}]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = check-apachedex-result.py
 config-command = "{{ parameter_dict['promise-check-apachedex-result'] }}" --apachedex_path "${directory:apachedex}" --status_file ${monitor-directory:private}/apachedex.report.json --threshold "${apachedex-parameters:promise-threshold}"
 
 [{{ section('promise-check-computer-memory') }}]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = check-computer-memory.py
 config-command = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${monitor-instance-parameter:collector-db} --threshold "{{ slapparameter_dict["computer-memory-percent-threshold"] }}" --unit percent
 

stack/erp5/instance-kumofs.cfg.in

@@ -86,7 +86,7 @@ rendered = ${directory:srv}/exporter.exclude
 # Deploy zope promises scripts
 [promise-template]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 config-host = ${kumofs-instance:ip}
 config-port = ${kumofs-instance:server-listen-port}
 
@@ -112,7 +112,7 @@ config-port = ${kumofs-instance:manager-port}
 
 [promise-check-computer-memory]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = check-computer-memory.py
 config-command = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${monitor-instance-parameter:collector-db} --threshold "{{ slapparameter_dict["computer-memory-percent-threshold"] }}" --unit percent
 

stack/erp5/instance-mariadb.cfg.in

@@ -332,7 +332,7 @@ context =
 {%if slapparameter_dict.get('max-slowqueries-threshold') and slapparameter_dict.get('slowest-query-threshold') %}
 [{{ section('monitor-promise-slowquery-result') }}]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = check-slow-query-pt-digest-result.py
 config-command = "{{ parameter_dict['promise-check-slow-queries-digest-result'] }}" --ptdigest_path "${directory:slowquery}" --status_file ${monitor-directory:private}/mariadb_slow_query.report.json --max_queries_threshold "${:max_queries_threshold}" --slowest_query_threshold "${:slowest_queries_threshold}"
 max_queries_threshold = {{ slapparameter_dict['max-slowqueries-threshold'] }}
@@ -341,13 +341,13 @@ slowest_queries_threshold = {{ slapparameter_dict['slowest-query-threshold'] }}
 
 [{{ section('promise-check-computer-memory') }}]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = check-computer-memory.py
 config-command = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${monitor-instance-parameter:collector-db} --threshold "{{ slapparameter_dict["computer-memory-percent-threshold"] }}" --unit percent
 
 [{{ section('promise') }}]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = mariadb.py
 config-command = "${binary-wrap-mysql:wrapper-path}"  --execute ';' {% if database_list and database_list[0].get('user') %} --host="${my-cnf-parameters:ip}" --port="${my-cnf-parameters:port}" --user="{{ database_list[0]['user'] }}" --password="{{ database_list[0]['password'] }}" {% endif %}
 

stack/erp5/instance-postfix.cfg.in

@@ -80,7 +80,7 @@ wrapper-path = ${directory:run}/munnel
 
 [{{ section('munnel-promise') }}]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = munnel.py
 config-host = {{ ip }}
 config-port = {{ milter_port }}
@@ -262,14 +262,14 @@ wrapper-path = ${directory:run}/postfix-master
 
 [{{ section('postfix-promise') }}]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = postfix.py
 config-host = {{ ip }}
 config-port = {{ tcpv4_port }}
 
 [{{ section('promise-check-computer-memory') }}]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = check-computer-memory.py
 config-command = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${monitor-instance-parameter:collector-db} --threshold "{{ slapparameter_dict["computer-memory-percent-threshold"] }}" --unit percent
 

stack/erp5/instance-wcfs.cfg.in

@@ -52,7 +52,7 @@ wrapper-path = ${directory:service-on-watch}/wcfs
 
 [wcfs-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = ${:_buildout_section_name_}.py
 config-command = {{ bin_directory }}/wcfs status {{ zurl }}
 

stack/erp5/instance-zeo.cfg.in

@@ -55,7 +55,7 @@ post = test ! -s {{ "${" ~ zeo_section_name ~":pid-path}" }} || {{ bin_directory
 
 [{{ section(zeo_section_name ~ "-promise") }}]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = zeo-{{ family }}.py
 config-host = {{ "${" ~ zeo_section_name ~ ":ip}" }}
 config-port = {{ "${" ~ zeo_section_name ~ ":port}" }}
@@ -89,7 +89,7 @@ tidstorage-wrapper = ${directory:services}/tidstoraged
 
 [{{ section("promise-tidstorage") }}]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = tidstorage.py
 config-host = ${tidstorage:ip}
 config-port = ${tidstorage:port}
@@ -177,7 +177,7 @@ mode = 755
 
 [{{ section('promise-check-computer-memory') }}]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = check-computer-memory.py
 config-command = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${monitor-instance-parameter:collector-db} --threshold "{{ slapparameter_dict["computer-memory-percent-threshold"] }}" --unit percent
 

stack/erp5/instance-zope.cfg.in

@@ -358,7 +358,7 @@ hash-existing-files =
 
 [{{ section("promise-" ~ name) }}]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = {{ name }}.py
 config-host = {{ ipv4 }}
 config-port = {{ port }}
@@ -373,7 +373,7 @@ ipv4-port = {{ port }}
 
 [{{ section("promise-tunnel-" ~ name) }}]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = {{ zope_tunnel_base_name }}.py
 config-host = {{ '${' ~ zope_tunnel_section_name ~ ':ipv6}' }}
 config-port = {{ '${' ~ zope_tunnel_section_name ~ ':ipv6-port}' }}
@@ -384,7 +384,7 @@ config-port = {{ '${' ~ zope_tunnel_section_name ~ ':ipv6-port}' }}
 {% if longrequest_logger_interval > 0 -%}
 [{{ section('promise-check-' ~name ~ '-longrequest-error-log') }}]
 <= monitor-promise-base
-module = check_error_on_zope_longrequest_log
+promise = check_error_on_zope_longrequest_log
 name = {{'check-' ~ name ~ '-longrequest-error-log.py'}}
 config-log-file = {{ '${' ~ conf_parameter_name ~ ':longrequest-logger-file}' }}
 config-error-threshold = {{ slapparameter_dict["zope-longrequest-logger-error-threshold"] }}
@@ -528,7 +528,7 @@ expected-value =
 
 [{{ section("promise-test-runner-apache-url") }}]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = ${:_buildout_section_name_}.py
 config-command = ${promise-test-runner-apache-url-executable:path}
 
@@ -537,7 +537,7 @@ config-command = ${promise-test-runner-apache-url-executable:path}
 
 [{{ section('promise-check-computer-memory') }}]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = check-computer-memory.py
 config-command = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${monitor-instance-parameter:collector-db} --threshold "{{ slapparameter_dict["computer-memory-percent-threshold"] }}" --unit percent
 

stack/lamp/buildout.cfg

@@ -23,7 +23,6 @@ extends =
   ../../component/mariadb/buildout.cfg
   ../../component/perl/buildout.cfg
   ../../component/sqlite3/buildout.cfg
-  ../../component/stunnel/buildout.cfg
   ../../component/xz-utils/buildout.cfg
   ../../component/zlib/buildout.cfg
   ../erp5/buildout.cfg
@@ -83,7 +82,6 @@ context =
     key logrotate_cfg template-logrotate-base:rendered
     key gzip_location gzip:location
     key xz_utils_location xz-utils:location
-    key stunnel_location stunnel:location
     key template_monitor monitor2-template:rendered
     key mariadb_link_binary template-mariadb:link-binary
     key mariadb_location mariadb:location

stack/lamp/buildout.hash.cfg

@@ -18,11 +18,11 @@ md5sum = e4e070f93adaf917f9427ae9f35573d9
 
 [instance-apache-php]
 filename = instance-apache-php.cfg.in
-md5sum = 4afee4377fa9cbc1e4ff80647b2f279c
+md5sum = e7a14c01e6314e2bffebd7d80cf1c488
 
 [instance-lamp]
 filename = instance-lamp.cfg.jinja2.in
-md5sum = 79f562260895df2665a85df5cb442193
+md5sum = e0e2e88b6deeb011b998b78e4e468555
 
 [template-apache.conf]
 filename = apache.conf.in

stack/lamp/instance-apache-php.cfg.in

@@ -207,7 +207,7 @@ backend-url = ${apache-php-configuration:url}
 [promise]
 # Check any apache port in ipv4, expect other ports and ipv6 to behave consistently
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = apache-httpd-port-listening.py
 config-host = ${apache-php-configuration:ip}
 config-port = ${apache-php-configuration:port}

stack/lamp/instance-lamp.cfg.jinja2.in

@@ -77,7 +77,7 @@ return = domain secure_access
 
 [lamp-frontend-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = lamp-http-frontend.py
 url = ${request-frontend:connection-secure_access}
 config-url = ${:url}

stack/monitor/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [monitor2-template]
 filename = instance-monitor.cfg.jinja2.in
-md5sum = d4185c191e8b9df20e1f98cd8c556b1d
+md5sum = 3cba541a8b0b22c2648848ed1d259174
 
 [monitor-httpd-conf]
 _update_hash_filename_ = templates/monitor-httpd.conf.in

stack/monitor/instance-monitor.cfg.jinja2.in

@@ -302,14 +302,12 @@ pre = {{ monitor_statistic }} --history_folder ${monitor-directory:public}
 recipe = slapos.cookbook:promise.plugin
 eggs =
   slapos.toolbox
-content =
-  from slapos.promise.plugin.${:module} import RunPromise
-mode = 600
+module = slapos.promise.plugin.${:promise}
 output = ${directory:plugins}/${:name}
 
 [monitor-httpd-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = monitor-httpd-listening-on-tcp.py
 config-url = ${monitor-httpd-conf-parameter:url}
 config-http-code = 401
@@ -360,7 +358,7 @@ return = domain secure_access
 # credentials.
 [check-monitor-password-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = check-monitor-frontend-password.py
 url = ${monitor-frontend:connection-secure_access}
 config-url = ${:url}
@@ -371,7 +369,7 @@ config-password = ${monitor-instance-parameter:password}
 # supplied.
 [monitor-frontend-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = monitor-http-frontend.py
 url = ${monitor-frontend:connection-secure_access}
 config-url = ${:url}
@@ -379,7 +377,7 @@ config-http-code = 401
 
 [monitor-bootstrap-promise]
 <= monitor-promise-base
-module = monitor_bootstrap_status
+promise = monitor_bootstrap_status
 name = monitor-bootstrap-status.py
 config-process-pid-file = ${monitor-conf-parameters:pid-file}
 config-process-name = ${start-monitor:name}
@@ -387,13 +385,13 @@ config-status-file = ${monitor-conf-parameters:promise-output-file}
 
 [promise-check-slapgrid]
 <= monitor-promise-base
-module = check_partition_deployment_state
+promise = check_partition_deployment_state
 name = buildout-${slap-connection:partition-id}-status.py
 config-monitor-url = ${monitor-instance-parameter:monitor-base-url}
 
 [promise-check-free-disk-space]
 <= monitor-promise-base
-module = check_free_disk_space
+promise = check_free_disk_space
 name = check-free-disk-space.py
 config-collectordb = ${monitor-instance-parameter:collector-db}
 config-threshold-file = ${directory:etc}/min-free-disk-size

stack/resilient/buildout.hash.cfg

@@ -14,23 +14,23 @@
 # not need these here).
 [pbsready]
 filename = pbsready.cfg.in
-md5sum = 005125621d157b3ae04c428ea6060e37
+md5sum = 1d3aba1ba770ad3fcc2ab6c0b9266409
 
 [pbsready-import]
 filename = pbsready-import.cfg.in
-md5sum = dd8f0728e53b49822eed5d613839558f
+md5sum = a8c9821951425bedbdea30a870fb5138
 
 [pbsready-export]
 filename = pbsready-export.cfg.in
-md5sum = 2b0c71b085cfe8017f28098c160b1f49
+md5sum = 8f15263c4a27ec315eb3a12dbf7a7b34
 
 [template-pull-backup]
 filename = instance-pull-backup.cfg.in
-md5sum = b240dc76a663190304d8bcb9cabcda8f
+md5sum = 4425db50d551fb8a974e547308990bac
 
 [template-replicated]
 filename = template-replicated.cfg.in
-md5sum = 0917aaacb752526e6f114839a3e6e1de
+md5sum = c4012ccc2c473ae5c7cad9dcac61e0f1
 
 [template-parts]
 filename = template-parts.cfg.in

stack/resilient/instance-pull-backup.cfg.in

@@ -217,7 +217,7 @@ wrapper-path = $${rootdirectory:bin}/stalled-pull-push
 
 [pull-push-stalled-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = stalled-pull-push.py
 config-command = $${pull-push-stalled-promise-bin:wrapper-path}
 
@@ -233,7 +233,7 @@ context =
 
 [notifier-feed-status-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = notifier-feed-check-malformed-or-failure.py
 config-command = $${notifier-feed-status-promise-bin:rendered}
 

stack/resilient/pbsready-export.cfg.in

@@ -60,7 +60,7 @@ rendered = ${rootdirectory:bin}/exporter-status
 
 [notifier-exporter-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = exporter-status.py
 config-command = ${notifier-exporter-promise-bin:rendered}
 

stack/resilient/pbsready-import.cfg.in

@@ -81,7 +81,7 @@ mode = 700
 
 [backup-checksum-integrity-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = backup-checksum-integrity.py
 config-command = $${backup-checksum-integrity-promise-bin:rendered}
 
@@ -159,7 +159,7 @@ wrapper-path = $${basedirectory:services}/resilient-web-takeover-httpd
 
 [resilient-web-takeover-httpd-promise]
 <= monitor-promise-base
-module = check_url_available
+promise = check_url_available
 name = resilient-web-takeover-httpd.py
 config-url = http://[$${resilient-web-takeover-httpd-configuration-file:listening-ip}]:$${resilient-web-takeover-httpd-configuration-file:listening-port}/
 

stack/resilient/pbsready.cfg.in

@@ -165,7 +165,7 @@ wrapper-path = $${rootdirectory:bin}/stalled-notifier-callbacks
 
 [notifier-stalled-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = stalled-notifier-callbacks.py
 config-command = $${notifier-stalled-promise-bin:wrapper-path}
 
@@ -217,7 +217,7 @@ wrapper-path = $${basedirectory:scripts}/sshd-graceful
 
 [sshd-promise]
 <= monitor-promise-base
-module = check_socket_listening
+promise = check_socket_listening
 name = sshd.py
 config-host = $${slap-network-information:global-ipv6}
 config-port = $${sshd-port:port}
@@ -261,7 +261,7 @@ mode = 700
 
 [resilient-sshkeys-sshd-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = public-key-existence.py
 config-command = $${resilient-sshkeys-sshd-promise-bin:output}
 
@@ -281,7 +281,7 @@ context =
 
 [notifier-feed-status-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = notifier-feed-check-malformed-or-failure.py
 config-command = $${notifier-feed-status-promise-bin:rendered}
 #----------------

stack/resilient/template-replicated.cfg.in

@@ -42,11 +42,7 @@ config-name = {{namebase}}0
 # Bubble up all the instance parameters to the requested export instance.
 {% if slapparameter_dict is defined %}
 {% for parameter_name, parameter_value in six.iteritems(slapparameter_dict) %}
-{% if parameter_value is string %}
-config-{{parameter_name}} = {{ parameter_value.split('\n') | join('\n  ') }}
-{% else %}
-config-{{parameter_name}} = {{ parameter_value }}
-{% endif %}
+config-{{ parameter_name }} = {{ dumps(parameter_value) }}
 {% endfor %}
 {% endif %}
 {% for key, value in six.iteritems(monitor_dict) -%}
@@ -152,7 +148,7 @@ mode = 700
 
 [resilient-request-{{namebase}}-public-key-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = resilient-request-{{namebase}}-public-key.py
 config-command = ${resilient-request-{{namebase}}-public-key-promise-bin:output}
 
@@ -174,7 +170,7 @@ mode = 700
 
 [resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key.py
 config-command = ${resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key-promise-bin:output}
 
@@ -242,7 +238,7 @@ mode = 700
 
 [resilient-request-pbs-{{namebase}}-{{id}}-public-key-promise]
 <= monitor-promise-base
-module = check_command_execute
+promise = check_command_execute
 name = resilient-request-pbs-{{namebase}}-{{id}}-public-key.py
 config-command = ${resilient-request-pbs-{{namebase}}-{{id}}-public-key-promise-bin:output}
 

stack/slapos.cfg

@@ -132,14 +132,14 @@ eggs =
 [versions]
 setuptools = 44.1.1
 # Use SlapOS patched zc.buildout
-zc.buildout = 2.7.1+slapos013
+zc.buildout = 2.7.1+slapos014
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
 zc.recipe.egg = 2.0.3+slapos003
 # Use own version of h.r.download to be able to open .xz and .lz archives
 hexagonit.recipe.download = 1.7.post4
 
 traitlets = 4.3.3
-Jinja2 = 2.11.2
+Jinja2 = 2.11.3
 Importing = 1.10
 MarkupSafe = 1.0
 PyYAML = 5.4.1
@@ -157,7 +157,7 @@ collective.recipe.shelloutput = 0.1
 collective.recipe.template = 2.0
 configparser = 4.0.2:whl
 contextlib2 = 0.6.0.post1
-cryptography = 3.3.1
+cryptography = 3.3.2
 dateparser = 0.7.6
 decorator = 4.3.0
 funcsigs = 1.0.2
@@ -188,18 +188,18 @@ requests = 2.24.0
 scandir = 1.10.0
 setproctitle = 1.1.10
 setuptools-dso = 1.7
-rubygemsrecipe  = 0.4.2
+rubygemsrecipe  = 0.4.3
 six = 1.12.0
-slapos.cookbook = 1.0.197
-slapos.core = 1.6.18
+slapos.cookbook = 1.0.213
+slapos.core = 1.6.19
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.20
 slapos.rebootstrap = 4.5
-slapos.recipe.build = 0.47
+slapos.recipe.build = 0.49
 slapos.recipe.cmmi = 0.17
 slapos.recipe.template = 4.6
-slapos.toolbox = 0.124
+slapos.toolbox = 0.126
 stevedore = 1.21.0:whl
 subprocess32 = 3.5.4
 unicodecsv = 0.14.1
@@ -224,7 +224,7 @@ atomize = 0.2.0
 croniter = 0.3.25
 dnspython = 1.16.0
 enum34 = 1.1.10
-erp5.util = 0.4.71
+erp5.util = 0.4.72
 feedparser = 5.2.1
 functools32 = 3.2.3.post2
 attrs = 18.2.0