Merge remote-tracking branch 'origin/master' into t

* origin/master: (91 commits)
  NEO: fix stress test against recent zodbtools update
  version up: MariaDB 10.3.26 & 10.4.16
  software/kvm: Fix textarea definition
  Revert "software/kvm: Allow many CD-ROMs to boot from"
  stack/erp5: Make bt5 default value a string at the last step.
  software/helloworld: move to python 3
  stack/slapos.cfg: upgrade rubygemsrecipe version
  software/theia: version up theia 1.7.0
  software/plantuml: move to python3
  component/caddy: Switch to Go 1.14
  Add fmtlib component
  stack/slapos: slapos.cookbook version up (1.0.167)
  Release slapos.cookbook (1.0.167)
  stack/slapos.cfg: version up erp5.util 0.4.69
  lamp: remove redundant items in buildout:parts
  fixup! Clean up usage of CMake
  erp5testnode: request a frontend for logs
  Make easier to use MariaDB 10.4
  version up: GCC 8.4
  version up: CMake 3.18.4
  ...

component/caddy/buildout.cfg

@@ -11,7 +11,7 @@ parts =
 command = bash -c ". ${gowork:env.sh} && cd ${go_github.com_caddyserver_caddy:location} && GO111MODULE=on go install -v $(echo -n '${gowork:install}' |tr '\n' ' ')"
 
 [gowork]
-golang  = ${golang1.12:location}
+golang  = ${golang1.14:location}
 install =
   github.com/caddyserver/caddy/...
 

component/cmake/buildout.cfg

@@ -9,8 +9,8 @@ parts =
 [cmake]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://cmake.org/files/v3.7/cmake-3.7.2.tar.gz
-md5sum = 79bd7e65cd81ea3aa2619484ad6ff25a
+url = https://cmake.org/files/v3.18/cmake-3.18.4.tar.gz
+md5sum = 0380beaee1c39a22455db02651abe7be
 environment =
   CMAKE_INCLUDE_PATH=${ncurses:location}/include:${openssl:location}/include
   CMAKE_LIBRARY_PATH=${ncurses:location}/lib:${openssl:location}/lib

component/cuneiform/buildout.cfg

-[buildout]
-extends =
-  ../cmake/buildout.cfg
-  ../imagemagick/buildout.cfg
-parts = cuneiform
-
-[cuneiform]
-recipe = slapos.recipe.cmmi
-url = http://launchpad.net/cuneiform-linux/1.0/1.0/+download/cuneiform-linux-1.0.0.tar.bz2
-md5sum = 785232ffffad7d82446fbac08a1c3ef9
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-configure-command =
-  mkdir build && cd build && \
-  ${cmake:location}/bin/cmake \
-    -DCMAKE_INSTALL_RPATH=${:location}/lib64:${:location}/lib \
-    -DCMAKE_INSTALL_PREFIX=${:location} \
-    -DNO_SYSTEM_ENVIRONMENT_PATH=ON \
-    -DCMAKE_INCLUDE_PATH=${imagemagick:location}/include \
-    -DCMAKE_LIBRARY_PATH=${imagemagick:location}/lib \
-    -DCMAKE_INSTALL_RPATH=${:location}/lib64:${:location}/lib:${imagemagick:location}/lib \
-    -DCMAKE_BUILD_TYPE=release \
-    ..
-make-binary =
-  cd build && make

component/defaults.cfg

@@ -34,7 +34,7 @@ depends =
   ${patch:recipe}
 recipe = slapos.recipe.build
 # Latest version provided by SlapOS.
-part = gcc-8.2
+part = gcc-8.4
 # Minimum version for all components that might be required for
 # slapos.rebootstrap (see https://bugs.python.org/issue34112 about Python 3.7+).
 min_version = 5.4

component/fmtlib/buildout.cfg

+[buildout]
+extends =
+  ../cmake/buildout.cfg
+parts =
+  fmtlib
+
+[fmtlib]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://github.com/fmtlib/fmt/archive/7.0.3.tar.gz
+md5sum = 57392b7ea09592a2b237a02950f35bb0
+configure-command = ${cmake:location}/bin/cmake
+configure-options =
+  -DCMAKE_INSTALL_PREFIX=@@LOCATION@@
+  -DBUILD_SHARED_LIBS=ON
+  -DFMT_TEST=OFF

component/gcc/buildout.cfg

@@ -10,7 +10,7 @@ extends =
   ../binutils/buildout.cfg
 
 parts =
-  gcc-8.2
+  gcc-8.4
 
 [gcc-common]
 recipe = slapos.recipe.cmmi
@@ -65,16 +65,10 @@ patch-options = -p1
 patches =
   ${:_profile_base_location_}/libsanitizer_Use_pre-computed_size_of_struct_ustat_for_Linux.patch#1e5f33e89f9fe1ca3e406eabcc621762
 
-[gcc-8.2]
+[gcc-8.4]
 <= gcc-common
-version = 8.2.0
-md5sum = 4ab282f414676496483b3e1793d07862
-patch-binary = ${patch:location}/bin/patch
-patch-options = -p1
-# glibc-2.31-libsanitizer comes from https://github.com/spack/spack/pull/15403
-patches =
-  ${:_profile_base_location_}/glibc-2.31-libsanitizer-1.patch
-  ${:_profile_base_location_}/glibc-2.31-libsanitizer-2.patch
+version = 8.4.0
+md5sum = bb815a8e3b7be43c4a26fa89dbbd9795
 
 [gcc-minimal]
 <= gcc-5.5

component/gcc/glibc-2.31-libsanitizer-1.patch

-From ce9568e9e9cf6094be30e748821421e703754ffc Mon Sep 17 00:00:00 2001
-From: Jakub Jelinek <jakub@redhat.com>
-Date: Fri, 8 Nov 2019 19:53:18 +0100
-Subject: [PATCH] backport: re PR sanitizer/92154 (new glibc breaks arm
- bootstrap due to libsanitizer)
-
-	Backported from mainline
-	2019-10-22  Tamar Christina  <tamar.christina@arm.com>
-
-	PR sanitizer/92154
-	* sanitizer_common/sanitizer_platform_limits_posix.cc:
-	Cherry-pick compiler-rt revision r375220.
-
-From-SVN: r277981
----
- libsanitizer/ChangeLog                                   | 9 +++++++++
- .../sanitizer_common/sanitizer_platform_limits_posix.cc  | 6 +++++-
- 2 files changed, 14 insertions(+), 1 deletion(-)
-
-diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc
-index 6cd4a5bac8b0..06a605ff4670 100644
---- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc
-+++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc
-@@ -1156,8 +1156,12 @@ CHECK_SIZE_AND_OFFSET(ipc_perm, uid);
- CHECK_SIZE_AND_OFFSET(ipc_perm, gid);
- CHECK_SIZE_AND_OFFSET(ipc_perm, cuid);
- CHECK_SIZE_AND_OFFSET(ipc_perm, cgid);
--#if !defined(__aarch64__) || !SANITIZER_LINUX || __GLIBC_PREREQ (2, 21)
-+#if (!defined(__aarch64__) || !SANITIZER_LINUX || __GLIBC_PREREQ (2, 21)) && \
-+    !defined(__arm__)
- /* On aarch64 glibc 2.20 and earlier provided incorrect mode field.  */
-+/* On Arm glibc 2.31 and later provide a different mode field, this field is
-+   never used by libsanitizer so we can simply ignore this assert for all glibc
-+   versions.  */
- CHECK_SIZE_AND_OFFSET(ipc_perm, mode);
- #endif
- 

component/gcc/glibc-2.31-libsanitizer-2.patch

-From 75003cdd23c310ec385344e8040d490e8dd6d2be Mon Sep 17 00:00:00 2001
-From: Jakub Jelinek <jakub@redhat.com>
-Date: Fri, 20 Dec 2019 17:58:35 +0100
-Subject: [PATCH] backport: re PR sanitizer/92154 (new glibc breaks arm
- bootstrap due to libsanitizer)
-
-	Backported from mainline
-	2019-11-26  Jakub Jelinek  <jakub@redhat.com>
-
-	PR sanitizer/92154
-	* sanitizer_common/sanitizer_platform_limits_posix.h: Cherry-pick
-	llvm-project revision 947f9692440836dcb8d88b74b69dd379d85974ce.
-	* sanitizer_common/sanitizer_platform_limits_posix.cc: Likewise.
-
-From-SVN: r279653
----
- libsanitizer/ChangeLog                            | 10 ++++++++++
- .../sanitizer_platform_limits_posix.cc            |  9 +++------
- .../sanitizer_platform_limits_posix.h             | 15 +--------------
- 3 files changed, 14 insertions(+), 20 deletions(-)
-
-diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc
-index 06a605ff4670..d823a12190c0 100644
---- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc
-+++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc
-@@ -1156,12 +1156,9 @@ CHECK_SIZE_AND_OFFSET(ipc_perm, uid);
- CHECK_SIZE_AND_OFFSET(ipc_perm, gid);
- CHECK_SIZE_AND_OFFSET(ipc_perm, cuid);
- CHECK_SIZE_AND_OFFSET(ipc_perm, cgid);
--#if (!defined(__aarch64__) || !SANITIZER_LINUX || __GLIBC_PREREQ (2, 21)) && \
--    !defined(__arm__)
--/* On aarch64 glibc 2.20 and earlier provided incorrect mode field.  */
--/* On Arm glibc 2.31 and later provide a different mode field, this field is
--   never used by libsanitizer so we can simply ignore this assert for all glibc
--   versions.  */
-+#if !SANITIZER_LINUX || __GLIBC_PREREQ (2, 31)
-+/* glibc 2.30 and earlier provided 16-bit mode field instead of 32-bit
-+   on many architectures.  */
- CHECK_SIZE_AND_OFFSET(ipc_perm, mode);
- #endif
- 
-diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h
-index 73af92af1e8f..6a673a7c9959 100644
---- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h
-+++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h
-@@ -211,26 +211,13 @@ namespace __sanitizer {
-     u64 __unused1;
-     u64 __unused2;
- #elif defined(__sparc__)
--#if defined(__arch64__)
-     unsigned mode;
--    unsigned short __pad1;
--#else
--    unsigned short __pad1;
--    unsigned short mode;
-     unsigned short __pad2;
--#endif
-     unsigned short __seq;
-     unsigned long long __unused1;
-     unsigned long long __unused2;
--#elif defined(__mips__) || defined(__aarch64__) || defined(__s390x__)
--    unsigned int mode;
--    unsigned short __seq;
--    unsigned short __pad1;
--    unsigned long __unused1;
--    unsigned long __unused2;
- #else
--    unsigned short mode;
--    unsigned short __pad1;
-+    unsigned int mode;
-     unsigned short __seq;
-     unsigned short __pad2;
- #if defined(__x86_64__) && !defined(_LP64)

component/h2o/buildout.cfg

@@ -14,7 +14,7 @@ recipe = slapos.recipe.cmmi
 shared = True
 url = https://github.com/h2o/h2o/archive/v2.2.6.tar.gz
 md5sum = 075283f92347fcb7b2f87dcc5251b922
-configure-command = ${cmake:location}/bin/cmake .
+configure-command = ${cmake:location}/bin/cmake
 configure-options =
   -DCMAKE_INSTALL_PREFIX=@@LOCATION@@
   -DWITH_MRUBY=on

component/inkscape/buildout.cfg

@@ -41,11 +41,10 @@ recipe = slapos.recipe.cmmi
 shared = true
 url = https://inkscape.org/gallery/item/13330/inkscape-0.92.4_A6N0YOn.tar.bz2
 md5sum = ac30f6d5747fd9c620c00dad500f414f
-location = @@LOCATION@@
 pkg_config_depends = ${freetype:location}/lib/pkgconfig:${gtkmm:location}/lib/pkgconfig:${gtkmm:pkg_config_depends}:${gsl:location}/lib/pkgconfig:${popt:location}/lib/pkgconfig:${garbage-collector:location}/lib/pkgconfig:${libxslt:location}/lib/pkgconfig
 configure-command = ${cmake:location}/bin/cmake
 configure-options =
-  -DCMAKE_INSTALL_PREFIX=${:location}
+  -DCMAKE_INSTALL_PREFIX=@@LOCATION@@
   -DENABLE_POPPLER=OFF
   -DWITH_GNOME_VFS=OFF
   -DWITH_IMAGE_MAGICK=OFF

component/libzip/buildout.cfg

@@ -13,17 +13,16 @@ shared = true
 url = https://libzip.org/download/libzip-1.5.2.tar.xz
 md5sum = f9dd38d273bcdec5d3d1498fe6684f42
 location = @@LOCATION@@
-configure-command =
-  mkdir build && cd build && \
-  ${cmake:location}/bin/cmake \
-    -DCMAKE_INSTALL_PREFIX=${:location} \
-    -DCMAKE_INCLUDE_PATH=${zlib:location}/include:${bzip2:location}/include \
-    -DCMAKE_LIBRARY_PATH=${zlib:location}/lib:${bzip2:location}/lib \
-    ..
-make-binary =
-  cd build && make
+configure-command = cmake
+configure-options =
+  -Bbuild
+  -DCMAKE_BUILD_TYPE=Release
+  -DCMAKE_INSTALL_PREFIX=${:location}
+  -DCMAKE_INCLUDE_PATH=${zlib:location}/include:${bzip2:location}/include
+  -DCMAKE_LIBRARY_PATH=${zlib:location}/lib:${bzip2:location}/lib
+make-options = -C build
 environment =
   PATH=${cmake:location}/bin:%(PATH)s
   CMAKE_INCLUDE_PATH=${zlib:location}/include:${bzip2:location}/include
   CMAKE_LIBRARY_PATH=${zlib:location}/lib:${bzip2:location}/lib
-  LDFLAGS=-L${:location}/lib -Wl,-rpath=${:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib
+  LDFLAGS=-Wl,-rpath=${:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib

component/llvm/buildout.cfg

+[buildout]
+extends =
+    ../cmake/buildout.cfg
+    ../git/buildout.cfg
+
+parts = llvm
+
+[llvm]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://github.com/llvm/llvm-project/releases/download/llvmorg-${:version}/llvm-${:version}.src.tar.xz
+version = 11.0.0
+md5sum = 85844102335b2e01b3c64b6734fb56f2
+configure-command = ${cmake:location}/bin/cmake
+configure-options =
+    -Bbuild
+    -DCMAKE_INSTALL_PREFIX=@@LOCATION@@
+    -DCMAKE_BUILD_TYPE=Release
+    -DLLVM_INSTALL_UTILS=ON
+    -DCMAKE_C_FLAGS="${:CMAKE_CFLAGS}"
+    -DCMAKE_CXX_FLAGS="${:CMAKE_CFLAGS}"
+make-options = -C build
+CMAKE_CFLAGS = -I${libxml2:location}/include/libxml2 -I${ncurses:location}/include -I${zlib:location}/include
+environment =
+    PATH=${git:location}/bin:%(PATH)s
+    LDFLAGS=-L${libxml2:location}/lib -L${ncurses:location}/lib -L${zlib:location}/lib -Wl,-rpath=${libxml2:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${zlib:location}/lib

component/mariadb/buildout.cfg

@@ -26,12 +26,12 @@ extends =
 parts =
   mariadb
 
-[mariadb]
+[mariadb-10.4]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
-version = 10.4.14
-md5sum = 9801120ae8acb33904ab4a3366a7714f
+url = https://archive.mariadb.org//mariadb-${:version}/source/mariadb-${:version}.tar.gz
+version = 10.4.16
+md5sum = e7e5071cc879de5358f60789f53df99e
 location = @@LOCATION@@
 pre-configure =
   set '\bSET(PLUGIN_AUTH_PAM YES CACHE BOOL "")' cmake/build_configurations/mysql_release.cmake
@@ -129,14 +129,10 @@ environment =
 
 ### XXX keep using mariadb 10.3.22 because 10.4 cause crash
 ### (we just override here for easier revert)
-[mariadb]
-version = 10.3.22
-url = https://downloads.mariadb.org/f/mariadb-10.3.22/source/mariadb-10.3.22.tar.gz/from/http%3A//ftp.hosteurope.de/mirror/archive.mariadb.org/?serve
-md5sum = f712a5e6fde038d0c9c6d2a2cd88b84e
-pre-configure =
-  set -e '\bSET(PLUGIN_AUTH_PAM YES)' cmake/build_configurations/mysql_release.cmake
-  grep -q "$@"
-  sed -i "/$1/d" "$2"
+[mariadb-10.3]
+<= mariadb-10.4
+version = 10.3.26
+md5sum = 7a3eda171db192c15ec31ac2520551ad
 post-install =
   ldd=`ldd ${:location}/lib/plugin/ha_rocksdb.so`
   for x in ${lz4:location} ${snappy:location} ${zstd:location}
@@ -145,14 +141,6 @@ post-install =
   mkdir -p ${:location}/include/wsrep &&
   cp -p wsrep/wsrep_api.h ${:location}/include/wsrep
 
-[mroonga-mariadb]
-pre-configure =
-  rm -rf fake_mariadb_source
-  mkdir -p fake_mariadb_source
-  cd fake_mariadb_source
-  ln -s ${mariadb:location}/wsrep-lib
-  cp -a ${mariadb:location}/include/mysql/server include
-  cp -a include/private sql
-  chmod -R a+w include sql # so that buildout can delete this compile-dir after install
-  mkdir -p ${:plugin-dir}
+[mariadb]
+location = ${mariadb-10.3:location}
 ### /XXX keep using mariadb 10.3.22 because 10.4 cause crash

component/proftpd/buildout.cfg

@@ -32,7 +32,8 @@ configure-options =
   --disable-cap
   --with-modules=mod_sftp:mod_ban
   --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
-environment = 
+environment =
+  CFLAGS=-DPR_RUN_DIR=\"/proc/self/cwd/var\"
   CPPFLAGS=-I${zlib:location}/include -I${openssl:location}/include
   LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib
   install_user=${proftpd-environment:USER}

component/rust/buildout.cfg

 [buildout]
-parts =
-  rustc
-
 extends =
-    ../cmake/buildout.cfg
-    ../curl/buildout.cfg
-    ../git/buildout.cfg
+    ../llvm/buildout.cfg
     ../openssl/buildout.cfg
     ../pkgconfig/buildout.cfg
 
+parts = rustc
 
 [rustc]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://static.rust-lang.org/dist/rustc-1.45.2-src.tar.gz
-md5sum = 1c67d7c3f211e49e12e7c20abab08e70
+url = https://static.rust-lang.org/dist/rustc-1.47.0-src.tar.gz
+md5sum = a460bed79b92f6a7833ba6e6390ee6ae
 # --sysconfdir is a workaround for https://github.com/rust-lang/rust/issues/63915
-configure-options = --enable-extended --sysconfdir=@@LOCATION@@/etc/
-environment = 
-    PATH=${cmake:location}/bin/:${curl:location}/bin/:${git:location}/bin/:${pkgconfig:location}/bin/:%(PATH)s
-    PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:
+configure-options =
+    --enable-extended
+    --llvm-config=${llvm:location}/bin/llvm-config
+    --sysconfdir=@@LOCATION@@/etc
+environment =
+    PATH=${curl:location}/bin/:${git:location}/bin:${pkgconfig:location}/bin:%(PATH)s
+    PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig
+    RUSTFLAGS=-C link-arg=-Wl,-rpath=${openssl:location}/lib -C link-arg=-L${libxml2:location}/lib -C link-arg=-L${ncurses:location}/lib -C link-arg=-L${zlib:location}/lib -C link-arg=-Wl,-rpath=${libxml2:location}/lib -C link-arg=-Wl,-rpath=${ncurses:location}/lib -C link-arg=-Wl,-rpath=${zlib:location}/lib

component/snappy/buildout.cfg

@@ -9,10 +9,8 @@ recipe = slapos.recipe.cmmi
 shared = true
 url = https://github.com/google/snappy/archive/1.1.8.tar.gz
 md5sum = 70e48cba7fecf289153d009791c9977f
-location = @@LOCATION@@
 configure-command = ${cmake:location}/bin/cmake
 configure-options =
-  -DCMAKE_INSTALL_PREFIX=${:location}
+  -DCMAKE_BUILD_TYPE=Release
+  -DCMAKE_INSTALL_PREFIX=@@LOCATION@@
   -DBUILD_SHARED_LIBS=ON
-environment =
-  CMAKE_PROGRAM_PATH=${cmake:location}/bin

component/srslte/buildout.cfg

@@ -15,28 +15,22 @@ parts =
 recipe = slapos.recipe.cmmi
 url = https://github.com/srsLTE/srsLTE/archive/release_18_09.zip
 md5sum = d9007ff9bd03ba67c6ed8acfd9bf2a7a 
-pre-configure = 
-  mkdir -p build
-configure-command = cd build && ${cmake:location}/bin/cmake ../  
-
-make-binary =
-make-targets= cd build &&
-    make DESTDIR=@@LOCATION@@ -j1 install
-
-
-configure-options = 
+configure-command = cmake
+configure-options =
+  -Bbuild
+  -DCMAKE_INSTALL_PREFIX=@@LOCATION@@
   -DCMAKE_C_FLAGS="${:CMAKE_CFLAGS}"
   -DCMAKE_CXX_FLAGS="${:CMAKE_CFLAGS}"
   -DCMAKE_INSTALL_RPATH=${:CMAKE_LIBRARY_PATH}
   -DCMAKE_INCLUDE_PATH=${mbedtls:location}/include
   -DCMAKE_LIBRARY_PATH=${mbedtls:location}/lib
+make-options = -C build
 
 CMAKE_CFLAGS = -I${bzip2:location}/include -I${fftw3:location}/include -I${mbedtls:location}/include -I${lksctp-tools:location}/include -I${libconfig:location}/include -I${boost-lib:location}/include 
 CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${mbedtls:location}/lib:${lksctp-tools:location}/lib:${libconfig:location}/lib:${boost-lib:location}/lib:${fftw3:location}/lib
 
 environment =
   CMAKE_INSTALL_PREFIX=${buildout:directory}/bin
-  CMAKE_PROGRAM_PATH=${cmake:location}/bin
   CMAKE_INCLUDE_PATH=${bzip2:location}/include:${mbedtls:location}/include:${lksctp-tools:location}/include:${libconfig:location}/include:${boost-lib:location}/include:${fftw3:location}/include
   CMAKE_LIBRARY_PATH=${:CMAKE_LIBRARY_PATH}
   CPPFLAGS=${:CMAKE_CFLAGS}

component/tomcat/buildout.cfg

@@ -10,7 +10,7 @@ parts =
 <= tomcat9
 
 [tomcat7]
-recipe = hexagonit.recipe.download
+recipe = slapos.recipe.build:download-unpacked
 ignore-existing = true
 strip-top-level-dir = true
 url = https://archive.apache.org/dist/tomcat/tomcat-7/v${:version}/bin/apache-tomcat-${:version}.tar.gz
@@ -18,7 +18,7 @@ version = 7.0.100
 md5sum = 79be4ba5a6e770730a4be3d5cb3c7862
 
 [tomcat9]
-recipe = hexagonit.recipe.download
+recipe = slapos.recipe.build:download-unpacked
 ignore-existing = true
 strip-top-level-dir = true
 url = https://archive.apache.org/dist/tomcat/tomcat-9/v${:version}/bin/apache-tomcat-${:version}.tar.gz

component/yaml-cpp/buildout.cfg

@@ -10,11 +10,9 @@ recipe = slapos.recipe.cmmi
 shared = true
 url = https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-0.6.3.tar.gz
 md5sum = b45bf1089a382e81f6b661062c10d0c2
-location = @@LOCATION@@
-configure-command =
-  mkdir build && cd build && \
-  ${cmake:location}/bin/cmake \
-    -DYAML_BUILD_SHARED_LIBS=ON \
-    -DCMAKE_INSTALL_PREFIX=${:location} \
-    ..
+configure-command = ${cmake:location}/bin/cmake
+configure-options =
+    -Bbuild
+    -DYAML_BUILD_SHARED_LIBS=ON
+    -DCMAKE_INSTALL_PREFIX=@@LOCATION@@
 make-options = -C build

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.152'
+version = '1.0.167'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read()
 

slapos/recipe/erp5testnode/__init__.py

@@ -58,7 +58,6 @@ class Recipe(GenericBaseRecipe):
           "\npath_list = %s" % ",".join(software_path_list) 
     CONFIG['computer_id'] = self.buildout['slap-connection']['computer-id']
     CONFIG['server_url'] = self.buildout['slap-connection']['server-url']
-    CONFIG['frontend_url'] = self.buildout['testnode-frontend']['connection-secure_access']
     configuration_file = self.createFile(
       self.options['configuration-file'],
       self.substituteTemplate(

slapos/recipe/erp5testnode/template/erp5testnode.cfg.in

@@ -20,6 +20,7 @@ httpd_software_access_port = %(httpd_software_access_port)s
 computer_id = %(computer_id)s
 server_url = %(server_url)s
 frontend_url = %(frontend_url)s
+log_frontend_url = %(log_frontend_url)s
 keep_log_days = %(keep_log_days)s
 
 # Binaries

slapos/recipe/switch_softwaretype.py

@@ -25,8 +25,45 @@
 #
 ##############################################################################
 
-import os, subprocess, sys
-import six
+
+from zc.buildout.buildout import Buildout
+
+
+class SubBuildout(Buildout):
+  """Run buildout in buildout, partially copied from infrae.buildout
+  """
+  def __init__(self, main_buildout, config, options, **kwargs):
+    # Use same logger
+    self._logger = main_buildout._logger
+    self._log_level = main_buildout._log_level
+
+    # Use same options
+    for opt in (
+        'offline',
+        'verbosity',
+        'newest',
+        'directory',
+        'eggs-directory',
+        'develop-eggs-directory',
+    ):
+      if opt in main_buildout['buildout']:
+        options.append((
+            'buildout',
+            opt,
+            main_buildout['buildout'][opt],
+        ))
+    # Use same slap connection
+    for k, v in main_buildout["slap-connection"].items():
+      options.append(('slap-connection', k, v))
+
+    Buildout.__init__(self, config, options, **kwargs)
+
+  def _setup_logging(self):
+    """We don't want to setup any logging, since it's already done
+    by the main buildout.
+    """
+    pass
+
 
 class Recipe:
 
@@ -39,26 +76,23 @@ class Recipe:
     self.base = self.buildout[section][key]
 
   def install(self):
-    # XXX-Antoine: We gotta find a better way to do this. I tried to check
-    # out how slapgrid-cp was running buildout. But it is worse than that.
-    args = sys.argv[:]
-    for x in six.iteritems(self.buildout["slap-connection"]):
-      args.append("slap-connection:%s=%s" % x)
-    for x in "directory", "eggs-directory", "develop-eggs-directory":
-      args.append("buildout:%s=%s" % (x, self.buildout["buildout"][x]))
-    args.append("buildout:installed=.installed-%s.cfg" % self.name)
-    # Options.get (from zc.buildout) should deserialize.
+    options = [("buildout", "installed", ".installed-%s.cfg" % self.name)]
+    profile = self.base
     try:
-      override = self.options["override"][self.software_type]
+      # XXX this assume using slapos.buildout, which serializes arbitrary python objects for options
+      extended_profile = self.options["override"][self.software_type]
     except (KeyError, TypeError):
-      buildout = self.base
+      pass
     else:
-      # unfortunately, buildout:extends does not work when given at command line
-      buildout = os.path.join(self.buildout["buildout"]["parts-directory"],
-                              self.name + ".cfg")
-      with open(override) as src, open(buildout, "w", 0) as dst:
-        dst.write("[buildout]\nextends = %s\n\n" % self.base + src.read())
-    subprocess.check_call(args + ["-oc", buildout])
-    return []
+      options.append(["buildout", "extends", profile])
+      profile = extended_profile
+
+    sub_buildout = SubBuildout(
+        self.buildout,
+        profile,
+        options,
+    )
+
+    sub_buildout.install([])
 
   update = install

software/caddy-frontend/README.caddy_frontend.rst

@@ -448,6 +448,15 @@ This allows backends to:
 Technical notes
 ===============
 
+Profile development guidelines
+------------------------------
+
+Keep the naming in instance profiles:
+
+ * ``software_parameter_dict`` for values coming from software
+ * ``instance_parameter_dict`` for **local** values generated by the instance, except ``configuration``
+ * ``slapparameter_dict`` for values coming from SlapOS Master
+
 Instantiated cluster structure
 ------------------------------
 

software/caddy-frontend/buildout.hash.cfg

@@ -14,36 +14,32 @@
 # not need these here).
 [template]
 filename = instance.cfg.in
-md5sum = bfb647325103640c19c38e7b7f6e2833
+md5sum = 28bf0c4c75c028bed79fc38786831b3e
 
-[template-common]
+[profile-common]
 filename = instance-common.cfg.in
 md5sum = 5784bea3bd608913769ff9a8afcccb68
 
-[template-apache-frontend]
+[profile-caddy-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = 02aac183352a6fd6ddd336d2e3757405
+md5sum = e7d7e1448b6420657e953026573311ca
 
-[template-caddy-replicate]
+[profile-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = 54c0648c8593699dae0c565bc7dd8629
+md5sum = b70f9ce80dd927ead51b4526997b75ed
 
-[template-slave-list]
+[profile-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = 2690fb2b5b6cefb7de53f35c214bdd52
+md5sum = ab143bfa2e20725aa35940c9033fa0ee
 
-[template-replicate-publish-slave-information]
+[profile-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
-md5sum = 7e3ee70c447f8203273d78f66ab519c3
+md5sum = de268251dafa5ad83ebf5b20636365d9
 
-[template-caddy-frontend-configuration]
+[profile-caddy-frontend-configuration]
 _update_hash_filename_ = templates/Caddyfile.in
 md5sum = 2503056e35463e045db3329bb8b6fae8
 
-[caddy-backend-url-validator]
-filename = templates/caddy-backend-url-validator.in
-md5sum = 0979a03476e86bf038516c9565dadc17
-
 [template-not-found-html]
 _update_hash_filename_ = templates/notfound.html
 md5sum = 88af61e7abbf30dc99a1a2526161128d
@@ -54,7 +50,7 @@ md5sum = 266f175dbdfc588af7a86b0b1884fe73
 
 [template-backend-haproxy-configuration]
 _update_hash_filename_ = templates/backend-haproxy.cfg.in
-md5sum = 80081a7ded0029bb24b4fe2d06b6ae95
+md5sum = bf40f8d0a049a8dd924ccc731956c87e
 
 [template-log-access]
 _update_hash_filename_ = templates/template-log-access.conf.in
@@ -112,13 +108,13 @@ md5sum = 8e1c6c06c09beb921965b3ce98c67c9e
 filename = caddyprofiledummy.py
 md5sum = 38792c2dceae38ab411592ec36fff6a8
 
-[template-kedifa]
+[profile-kedifa]
 filename = instance-kedifa.cfg.in
-md5sum = d76fe7bf062410eda7049446ed06a736
+md5sum = 3daebc4b37088fa01183a853920d4143
 
 [template-backend-haproxy-rsyslogd-conf]
 _update_hash_filename_ = templates/backend-haproxy-rsyslogd.conf.in
-md5sum = be899b04e1aa652ed510f20d4ea523dd
+md5sum = 3ec9e088817f6a0e3b3b71919590e6b3
 
 [template-slave-introspection-httpd-nginx]
 _update_hash_filename_ = templates/slave-introspection-httpd-nginx.conf.in

software/caddy-frontend/common.cfg

-[buildout]
-extends =
-  buildout.hash.cfg
-  ../../stack/slapos.cfg
-  ../../component/dash/buildout.cfg
-  ../../component/caddy/buildout.cfg
-  ../../component/gzip/buildout.cfg
-  ../../component/logrotate/buildout.cfg
-  ../../component/rdiff-backup/buildout.cfg
-  ../../component/trafficserver/buildout.cfg
-  ../../component/6tunnel/buildout.cfg
-  ../../component/xz-utils/buildout.cfg
-  ../../component/rsyslogd/buildout.cfg
-  ../../component/haproxy/buildout.cfg
-  ../../component/nginx/buildout.cfg
-  ../../component/numpy/buildout.cfg
-
-  ../../stack/caucase/buildout.cfg
-# Monitoring stack (keep on bottom)
-  ../../stack/monitor/buildout.cfg
-
-parts +=
-  caucase-eggs
-  template
-  template-caddy-frontend
-  template-caddy-replicate
-  caddy
-
-  logrotate
-  rdiff-backup
-  caddyprofiledeps
-
-  kedifa-develop
-  kedifa
-
-[kedifa-repository]
-recipe = slapos.recipe.build:gitclone
-repository = https://lab.nexedi.com/nexedi/kedifa.git
-git-executable = ${git:location}/bin/git
-revision = d6bbd7db215e12871c1536f22a8fbf994227270c
-
-[kedifa-develop]
-recipe = zc.recipe.egg:develop
-setup = ${kedifa-repository:location}
-
-[kedifa]
-recipe = zc.recipe.egg
-eggs =
-  ${python-cryptography:egg}
-  kedifa
-
-[caddyprofiledeps-setup]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/setup.py
-
-[caddyprofiledeps-dummy]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/caddyprofiledummy.py
-
-[caddyprofiledeps-prepare]
-recipe = plone.recipe.command
-stop-on-error = True
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-update-command = ${:command}
-command =
-  rm -fr ${:location} &&
-  mkdir -p ${:location} &&
-  cp ${caddyprofiledeps-setup:target} ${:location}/ &&
-  cp ${caddyprofiledeps-dummy:target} ${:location}/
-
-[caddyprofiledeps-develop]
-recipe = zc.recipe.egg:develop
-setup = ${caddyprofiledeps-prepare:location}
-
-[caddyprofiledeps]
-depends = ${caddyprofiledeps-develop:recipe}
-recipe = zc.recipe.egg
-eggs =
-  caddyprofiledeps
-  websockify
-  collective.recipe.shelloutput
-
-[template-common]
-recipe = slapos.recipe.template:jinja2
-template = ${:_profile_base_location_}/instance-common.cfg.in
-rendered = ${buildout:directory}/instance-common.cfg
-mode = 0644
-context =
-  key develop_eggs_directory buildout:develop-eggs-directory
-  key eggs_directory buildout:eggs-directory
-
-[template-frontend-parameter-section]
-common_profile = ${template-common:rendered}
-logrotate_base_instance = ${template-logrotate-base:rendered}
-
-bin_directory = ${buildout:bin-directory}
-
-sixtunnel = ${6tunnel:location}
-nginx = ${nginx-output:nginx}
-nginx_mime = ${nginx-output:mime}
-caddy = ${caddy:output}
-caddy_location = ${caddy:location}
-haproxy_executable = ${haproxy:location}/sbin/haproxy
-rsyslogd_executable = ${rsyslogd:location}/sbin/rsyslogd
-curl = ${curl:location}
-dash = ${dash:location}
-gzip = ${gzip:location}
-logrotate = ${logrotate:location}
-openssl = ${openssl:location}/bin/openssl
-openssl_cnf = ${openssl:location}/etc/ssl/openssl.cnf
-trafficserver = ${trafficserver:location}
-sha256sum = ${coreutils:location}/bin/sha256sum
-kedifa = ${:bin_directory}/kedifa
-kedifa-updater = ${:bin_directory}/kedifa-updater
-kedifa-csr = ${:bin_directory}/kedifa-csr
-xz_location = ${xz-utils:location}
-htpasswd = ${:bin_directory}/htpasswd
-
-monitor_template = ${monitor-template:output}
-template_backend_haproxy_configuration = ${template-backend-haproxy-configuration:target}
-template_backend_haproxy_rsyslogd_conf = ${template-backend-haproxy-rsyslogd-conf:target}
-template_caddy_frontend_configuration = ${template-caddy-frontend-configuration:target}
-template_graceful_script = ${template-graceful-script:target}
-template_validate_script = ${template-validate-script:target}
-template_rotate_script = ${template-rotate-script:target}
-template_configuration_state_script = ${template-configuration-state-script:target}
-template_caddy_lazy_script_call = ${template-caddy-lazy-script-call:target}
-template_default_slave_virtualhost = ${template-default-slave-virtualhost:target}
-template_empty = ${template-empty:target}
-template_log_access = ${template-log-access:target}
-template_not_found_html = ${template-not-found-html:target}
-template_slave_list = ${template-slave-list:target}
-template_trafficserver_records_config = ${template-trafficserver-records-config:target}
-template_trafficserver_storage_config = ${template-trafficserver-storage-config:target}
-template_trafficserver_logging_yaml = ${template-trafficserver-logging-yaml:target}
-template_wrapper = ${template-wrapper:output}
-template_slave_introspection_httpd_nginx = ${template-slave-introspection-httpd-nginx:target}
-
-[template]
-recipe = slapos.recipe.template:jinja2
-template = ${:_profile_base_location_}/instance.cfg.in
-rendered = ${buildout:directory}/template.cfg
-mode = 0644
-context =
-  key common_profile template-common:rendered
-
-  key monitor2_template monitor2-template:rendered
-  key template_caddy_frontend template-caddy-frontend:target
-  key template_caddy_replicate template-caddy-replicate:target
-  key template_kedifa template-kedifa:target
-  key template_replicate_publish_slave_information template-replicate-publish-slave-information:target
-  key caddy_backend_url_validator caddy-backend-url-validator:output
-
-  section template_frontend_parameter_dict template-frontend-parameter-section
-  key caucase_jinja2_library caucase-jinja2-library:target
-
-[template-caddy-frontend]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/instance-apache-frontend.cfg.in
-mode = 0644
-
-[caddy-backend-url-validator]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/${:filename}
-output = ${buildout:directory}/caddy-backend-url-validator
-mode = 0750
-
-[template-caddy-replicate]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
-mode = 0644
-
-[template-kedifa]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/instance-kedifa.cfg.in
-mode = 0644
-
-[download-template]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/${:_update_hash_filename_}
-mode = 640
-
-[template-slave-list]
-<=download-template
-
-[template-replicate-publish-slave-information]
-<=download-template
-
-[template-caddy-frontend-configuration]
-<=download-template
-
-[template-not-found-html]
-<=download-template
-
-[template-default-slave-virtualhost]
-<=download-template
-
-[template-backend-haproxy-configuration]
-<=download-template
-
-[template-log-access]
-<=download-template
-
-[template-empty]
-<=download-template
-
-[template-slave-introspection-httpd-nginx]
-<=download-template
-
-[template-wrapper]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/templates/wrapper.in
-output = ${buildout:directory}/template-wrapper.cfg
-mode = 0644
-
-[template-trafficserver-records-config]
-<=download-template
-
-[template-trafficserver-storage-config]
-<=download-template
-
-[template-trafficserver-logging-yaml]
-<=download-template
-
-[template-rotate-script]
-<=download-template
-
-[template-caddy-lazy-script-call]
-<=download-template
-
-[template-graceful-script]
-<=download-template
-
-[template-validate-script]
-<=download-template
-
-[template-configuration-state-script]
-<=download-template
-
-[template-backend-haproxy-rsyslogd-conf]
-<=download-template

software/caddy-frontend/development.cfg

-# Development profile of caddy-frontend.
-# Exactly the same as software.cfg, but fetch the slapos.cookbook
-# from git repository instead of fetching stable version,
-# allowing to play with bleeding edge environment.
-
-# You'll need to run buildout twice for this profile.
-
-[buildout]
-extends =
-# Extend in this order, otherwise "parts" will be taken from git profile
-  common.cfg
-
-parts +=
-  slapos.toolbox-dev
-
-[slapos.toolbox-dev]
-recipe = zc.recipe.egg:develop
-egg = slapos.toolbox
-setup = ${slapos.toolbox-repository:location}

software/caddy-frontend/instance.cfg.in

 [buildout]
-extends = {{ common_profile }}
+extends = {{ software_parameter_dict['profile_common'] }}
 
 parts =
-  dynamic-template-caddy-replicate
   switch-softwaretype
 
 [caddyprofiledeps]
@@ -11,76 +10,59 @@ recipe = caddyprofiledeps
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
 rendered = ${buildout:directory}/${:filename}
+extensions = jinja2.ext.do
 extra-context =
 context =
     import json_module json
-    key slap_software_type instance-parameter:slap-software-type
     key slapparameter_dict instance-parameter:configuration
-    key slave_instance_list instance-parameter:slave-instance-list
-    section instance_parameter instance-parameter
+    section instance_parameter_dict instance-parameter
+    section software_parameter_dict software-parameter-section
     ${:extra-context}
+caucase-jinja2-library = {{ software_parameter_dict['caucase_jinja2_library'] }}
+import-list =
+  file caucase :caucase-jinja2-library
 
 [switch-softwaretype]
 recipe = slapos.cookbook:softwaretype
-default = ${dynamic-template-caddy-replicate:rendered}
-RootSoftwareInstance = ${dynamic-template-caddy-replicate:rendered}
-custom-personal = ${dynamic-template-caddy-replicate:rendered}
-single-default = ${dynamic-template-caddy-frontend:rendered}
-single-custom-personal = ${dynamic-template-caddy-frontend:rendered}
-replicate = ${dynamic-template-caddy-replicate:rendered}
-kedifa = ${dynamic-template-kedifa:rendered}
+default = ${dynamic-profile-caddy-replicate:rendered}
+RootSoftwareInstance = ${dynamic-profile-caddy-replicate:rendered}
+custom-personal = ${dynamic-profile-caddy-replicate:rendered}
+single-default = ${dynamic-profile-caddy-frontend:rendered}
+single-custom-personal = ${dynamic-profile-caddy-frontend:rendered}
+replicate = ${dynamic-profile-caddy-replicate:rendered}
+kedifa = ${dynamic-profile-kedifa:rendered}
 
-[dynamic-template-caddy-frontend-parameters]
-{% for key,value in template_frontend_parameter_dict.iteritems() %}
+[software-parameter-section]
+{% for key,value in software_parameter_dict.iteritems() %}
 {{ key }} = {{ dumps(value) }}
 {% endfor -%}
 
-[dynamic-template-caddy-frontend]
+[dynamic-profile-caddy-frontend]
 < = jinja2-template-base
-template = {{ template_caddy_frontend }}
+template = {{ software_parameter_dict['profile_caddy_frontend'] }}
 filename = instance-caddy-frontend.cfg
-extensions = jinja2.ext.do
 extra-context =
   import furl_module furl
-  section parameter_dict dynamic-template-caddy-frontend-parameters
   raw software_type single-custom-personal
-caucase-jinja2-library = {{ caucase_jinja2_library }}
-import-list =
-  file caucase :caucase-jinja2-library
 
-[dynamic-template-caddy-replicate]
+[dynamic-profile-caddy-replicate]
 < = jinja2-template-base
 depends = ${caddyprofiledeps:recipe}
-template = {{ template_caddy_replicate }}
+template = {{ software_parameter_dict['profile_caddy_replicate'] }}
 filename = instance-caddy-replicate.cfg
-extensions = jinja2.ext.do
 extra-context =
     import subprocess_module subprocess
     import functools_module functools
     import validators validators
-    key cluster_identification instance-parameter:root-instance-title
-    raw caddy_backend_url_validator {{ caddy_backend_url_validator }}
-    raw template_publish_slave_information {{ template_replicate_publish_slave_information }}
 # Must match the key id in [switch-softwaretype] which uses this section.
     raw software_type RootSoftwareInstance-default-custom-personal-replicate
-    raw template_monitor {{ monitor2_template }}
-    raw common_profile {{ common_profile }}
-    section parameter_dict dynamic-template-caddy-frontend-parameters
-caucase-jinja2-library = {{ caucase_jinja2_library }}
-import-list =
-  file caucase :caucase-jinja2-library
 
-[dynamic-template-kedifa]
+[dynamic-profile-kedifa]
 < = jinja2-template-base
-template = {{ template_kedifa }}
+template = {{ software_parameter_dict['profile_kedifa'] }}
 filename = instance-kedifa.cfg
-extensions = jinja2.ext.do
 extra-context =
-  section parameter_dict dynamic-template-caddy-frontend-parameters
   raw software_type kedifa
-caucase-jinja2-library = {{ caucase_jinja2_library }}
-import-list =
-  file caucase :caucase-jinja2-library
 
 [instance-parameter]
 # Fetches parameters defined in SlapOS Master for this instance.

software/caddy-frontend/software.cfg

 [buildout]
-extends = common.cfg
+extends =
+  buildout.hash.cfg
+  ../../stack/slapos.cfg
+  ../../component/dash/buildout.cfg
+  ../../component/caddy/buildout.cfg
+  ../../component/gzip/buildout.cfg
+  ../../component/logrotate/buildout.cfg
+  ../../component/rdiff-backup/buildout.cfg
+  ../../component/trafficserver/buildout.cfg
+  ../../component/6tunnel/buildout.cfg
+  ../../component/xz-utils/buildout.cfg
+  ../../component/rsyslogd/buildout.cfg
+  ../../component/numpy/buildout.cfg
+  ../../component/haproxy/buildout.cfg
+  ../../component/nginx/buildout.cfg
+
+  ../../stack/caucase/buildout.cfg
+# Monitoring stack (keep on bottom)
+  ../../stack/monitor/buildout.cfg
+
+parts +=
+  caucase-eggs
+  template
+  rdiff-backup
+  caddyprofiledeps
+  kedifa-develop
+  kedifa
+
+[kedifa-repository]
+recipe = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/kedifa.git
+git-executable = ${git:location}/bin/git
+revision = d6bbd7db215e12871c1536f22a8fbf994227270c
+
+[kedifa-develop]
+recipe = zc.recipe.egg:develop
+setup = ${kedifa-repository:location}
+
+[kedifa]
+recipe = zc.recipe.egg
+eggs =
+  ${python-cryptography:egg}
+  kedifa
+
+[caddyprofiledeps-setup]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/setup.py
+
+[caddyprofiledeps-dummy]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/caddyprofiledummy.py
+
+[caddyprofiledeps-prepare]
+recipe = plone.recipe.command
+stop-on-error = True
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+update-command = ${:command}
+command =
+  rm -fr ${:location} &&
+  mkdir -p ${:location} &&
+  cp ${caddyprofiledeps-setup:target} ${:location}/ &&
+  cp ${caddyprofiledeps-dummy:target} ${:location}/
+
+[caddyprofiledeps-develop]
+recipe = zc.recipe.egg:develop
+setup = ${caddyprofiledeps-prepare:location}
+
+[caddyprofiledeps]
+depends = ${caddyprofiledeps-develop:recipe}
+recipe = zc.recipe.egg
+eggs =
+  caddyprofiledeps
+  websockify
+  collective.recipe.shelloutput
+
+[profile-common]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance-common.cfg.in
+rendered = ${buildout:directory}/instance-common.cfg
+mode = 0644
+context =
+  key develop_eggs_directory buildout:develop-eggs-directory
+  key eggs_directory buildout:eggs-directory
+
+[software-parameter-section]
+# libraries
+caucase_jinja2_library = ${caucase-jinja2-library:target}
+
+# profiles
+profile_caddy_frontend = ${profile-caddy-frontend:target}
+profile_caddy_replicate = ${profile-caddy-replicate:target}
+profile_common = ${profile-common:rendered}
+profile_kedifa = ${profile-kedifa:target}
+profile_logrotate_base = ${template-logrotate-base:rendered}
+profile_monitor = ${monitor-template:output}
+profile_monitor2 = ${monitor2-template:rendered}
+profile_replicate_publish_slave_information = ${profile-replicate-publish-slave-information:target}
+profile_slave_list = ${profile-slave-list:target}
+
+# templates
+template_backend_haproxy_configuration = ${template-backend-haproxy-configuration:target}
+template_backend_haproxy_rsyslogd_conf = ${template-backend-haproxy-rsyslogd-conf:target}
+template_caddy_frontend_configuration = ${profile-caddy-frontend-configuration:target}
+template_caddy_lazy_script_call = ${template-caddy-lazy-script-call:target}
+template_configuration_state_script = ${template-configuration-state-script:target}
+template_default_slave_virtualhost = ${template-default-slave-virtualhost:target}
+template_empty = ${template-empty:target}
+template_graceful_script = ${template-graceful-script:target}
+template_log_access = ${template-log-access:target}
+template_not_found_html = ${template-not-found-html:target}
+template_rotate_script = ${template-rotate-script:target}
+template_slave_introspection_httpd_nginx = ${template-slave-introspection-httpd-nginx:target}
+template_trafficserver_logging_yaml = ${template-trafficserver-logging-yaml:target}
+template_trafficserver_records_config = ${template-trafficserver-records-config:target}
+template_trafficserver_storage_config = ${template-trafficserver-storage-config:target}
+template_validate_script = ${template-validate-script:target}
+template_wrapper = ${template-wrapper:output}
+
+# directories
+bin_directory = ${buildout:bin-directory}
+
+# files
+sixtunnel = ${6tunnel:location}
+nginx = ${nginx-output:nginx}
+nginx_mime = ${nginx-output:mime}
+caddy = ${caddy:output}
+haproxy_executable = ${haproxy:location}/sbin/haproxy
+rsyslogd_executable = ${rsyslogd:location}/sbin/rsyslogd
+curl = ${curl:location}
+dash = ${dash:location}
+gzip = ${gzip:location}
+logrotate = ${logrotate:location}
+openssl = ${openssl:location}/bin/openssl
+openssl_cnf = ${openssl:location}/etc/ssl/openssl.cnf
+trafficserver = ${trafficserver:location}
+sha256sum = ${coreutils:location}/bin/sha256sum
+kedifa = ${:bin_directory}/kedifa
+kedifa-updater = ${:bin_directory}/kedifa-updater
+kedifa-csr = ${:bin_directory}/kedifa-csr
+xz_location = ${xz-utils:location}
+htpasswd = ${:bin_directory}/htpasswd
+
+[template]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance.cfg.in
+rendered = ${buildout:directory}/template.cfg
+mode = 0644
+context =
+  section software_parameter_dict software-parameter-section
+
+[profile-caddy-frontend]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/instance-apache-frontend.cfg.in
+mode = 0644
+
+[profile-caddy-replicate]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
+mode = 0644
+
+[profile-kedifa]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/instance-kedifa.cfg.in
+mode = 0644
+
+[download-template]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+mode = 640
+
+[profile-slave-list]
+<=download-template
+
+[profile-replicate-publish-slave-information]
+<=download-template
+
+[profile-caddy-frontend-configuration]
+<=download-template
+
+[template-not-found-html]
+<=download-template
+
+[template-default-slave-virtualhost]
+<=download-template
+
+[template-backend-haproxy-configuration]
+<=download-template
+
+[template-log-access]
+<=download-template
+
+[template-empty]
+<=download-template
+
+[template-slave-introspection-httpd-nginx]
+<=download-template
+
+[template-wrapper]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/templates/wrapper.in
+output = ${buildout:directory}/template-wrapper.cfg
+mode = 0644
+
+[template-trafficserver-records-config]
+<=download-template
+
+[template-trafficserver-storage-config]
+<=download-template
+
+[template-trafficserver-logging-yaml]
+<=download-template
+
+[template-rotate-script]
+<=download-template
+
+[template-caddy-lazy-script-call]
+<=download-template
+
+[template-graceful-script]
+<=download-template
+
+[template-validate-script]
+<=download-template
+
+[template-configuration-state-script]
+<=download-template
+
+[template-backend-haproxy-rsyslogd-conf]
+<=download-template
 
 [versions]
 # Modern KeDiFa requires zc.lockfile

software/caddy-frontend/templates/backend-haproxy-rsyslogd.conf.in

@@ -16,7 +16,7 @@ $Umask 0022
 $WorkDirectory {{ configuration['spool-directory'] }}
 
 # Setup logging per slave, by extracting the slave name from the log stream
-{%- set regex = ".*-backend (.*)-http.*" %}
+{%- set regex = ".*-backend (.*)-http.{0,1}/" %}
 template(name="extract_slave_name" type="string" string="%msg:R,ERE,1,FIELD:{{ regex }}--end%")
 set $!slave_name = exec_template("extract_slave_name");
 template(name="slave_output" type="string" string="{{ configuration['caddy-log-directory'] }}/%$!slave_name%_backend_log")

software/caddy-frontend/templates/backend-haproxy.cfg.in

@@ -14,27 +14,30 @@ defaults
   timeout connect {{ configuration['backend-connect-timeout'] }}s
   retries {{ configuration['backend-connect-retries'] }}
 
+{%- set SCHEME_PREFIX_MAPPING = { 'http': 'http_backend', 'https': 'https_backend'} %}
 {%- macro frontend_entry(slave_instance, scheme, wildcard) %}
 {#-   wildcard switch allows to put dangerous entries in the end, as haproxy parses with first match #}
-{%-   set host_list = (slave_instance.get('server-alias') or  '').split() %}
-{%-   if slave_instance.get('custom_domain') not in host_list %}
-{%-     do host_list.append(slave_instance.get('custom_domain')) %}
-{%-   endif %}
-{%-   set matched = {'count': 0} %}
-{%-   for host in host_list %}
-{#-     Match up to the end or optional port (starting with ':') #}
-{#-     Please note that this matching is quite sensitive to changes and hard to test, so avoid needless changes #}
-{%-     if wildcard and host.startswith('*.') %}
-{%-       do matched.__setitem__('count', matched['count'] + 1) %}
+{%-   if slave_instance[SCHEME_PREFIX_MAPPING[scheme]]['hostname'] and slave_instance[SCHEME_PREFIX_MAPPING[scheme]]['port'] %}
+{%-     set host_list = (slave_instance.get('server-alias') or  '').split() %}
+{%-     if slave_instance.get('custom_domain') not in host_list %}
+{%-       do host_list.append(slave_instance.get('custom_domain')) %}
+{%-     endif %}
+{%-     set matched = {'count': 0} %}
+{%-     for host in host_list %}
+{#-       Match up to the end or optional port (starting with ':') #}
+{#-       Please note that this matching is quite sensitive to changes and hard to test, so avoid needless changes #}
+{%-       if wildcard and host.startswith('*.') %}
+{%-         do matched.__setitem__('count', matched['count'] + 1) %}
 # match wildcard {{ host }}
   acl is_{{ slave_instance['slave_reference'] }} hdr_reg(host) -i {{ host[2:] }}($|:.*)
-{%-     elif not wildcard and not host.startswith('*.') %}
-{%-       do matched.__setitem__('count', matched['count'] + 1) %}
+{%-       elif not wildcard and not host.startswith('*.') %}
+{%-         do matched.__setitem__('count', matched['count'] + 1) %}
   acl is_{{ slave_instance['slave_reference'] }} hdr_reg(host) -i ^{{ host }}($|:.*)
-{%-     endif %}
-{%-   endfor %}
-{%-   if matched['count'] > 0 %}
+{%-       endif %}
+{%-     endfor %}
+{%-     if matched['count'] > 0 %}
   use_backend {{ slave_instance['slave_reference'] }}-{{ scheme }} if is_{{ slave_instance['slave_reference'] }}
+{%-     endif %}
 {%-   endif %}
 {%- endmacro %}
 
@@ -49,59 +52,58 @@ frontend statistic
 
 frontend http-backend
   bind {{ configuration['local-ipv4'] }}:{{ configuration['http-port'] }}
-{%- for slave_instance in backend_slave_list %}
+{%- for slave_instance in backend_slave_list -%}
 {{ frontend_entry(slave_instance, 'http', False) }}
 {%- endfor %}
-{%- for slave_instance in backend_slave_list %}
+{%- for slave_instance in backend_slave_list -%}
 {{ frontend_entry(slave_instance, 'http', True) }}
 {%- endfor %}
 
 frontend https-backend
   bind {{ configuration['local-ipv4'] }}:{{ configuration['https-port'] }}
-{%- for slave_instance in backend_slave_list %}
+{%- for slave_instance in backend_slave_list -%}
 {{ frontend_entry(slave_instance, 'https', False) }}
 {%- endfor %}
-{%- for slave_instance in backend_slave_list %}
+{%- for slave_instance in backend_slave_list -%}
 {{ frontend_entry(slave_instance, 'https', True) }}
 {%- endfor %}
 
 {%- for slave_instance in backend_slave_list %}
-{%-   for (scheme, prefix) in [('http', 'http_backend'), ('https', 'https_backend')] %}
+{%-   for (scheme, prefix) in SCHEME_PREFIX_MAPPING.items() %}
 {%-     set info_dict = slave_instance[prefix] %}
-{%-     if info_dict['scheme'] == 'https' %}
-{%-       set ssl = [] %}
-{%-       if slave_instance['authenticate-to-backend'] %}
-{%-         set ssl = ['crt %s' % (configuration['certificate'],)] %}
-{%-       endif %}
-{%-       do ssl.append('ssl verify') %}
-{%-       set path_to_ssl_proxy_ca_crt = slave_instance.get('path_to_ssl_proxy_ca_crt') %}
-{%-       if slave_instance['ssl_proxy_verify'] %}
-{%-         if path_to_ssl_proxy_ca_crt %}
-{%-           do ssl.append('required ca-file %s' % (path_to_ssl_proxy_ca_crt,)) %}
+{%-     if info_dict['hostname'] and info_dict['port'] %}
+{%-       set ssl_list = [] %}
+{%-       if info_dict['scheme'] == 'https' %}
+{%-         if slave_instance['authenticate-to-backend'] %}
+{%-           do ssl_list.append('crt %s' % (configuration['certificate'],)) %}
+{%-         endif %}
+{%-         do ssl_list.append('ssl verify') %}
+{%-         set path_to_ssl_proxy_ca_crt = slave_instance.get('path_to_ssl_proxy_ca_crt') %}
+{%-         if slave_instance['ssl_proxy_verify'] %}
+{%-           if path_to_ssl_proxy_ca_crt %}
+{%-             do ssl_list.append('required ca-file %s' % (path_to_ssl_proxy_ca_crt,)) %}
+{%-           else %}
+{#-           Backend SSL shall be verified, but not CA provided, disallow connection #}
+{#-           Simply dropping hostname from the dict will result with ignoring it... #}
+{%-           do info_dict.__setitem__('hostname', '') %}
+{%-           endif %}
 {%-         else %}
-{#-         Backend SSL shall be verified, but not CA provided, disallow connection #}
-{#-         Simply dropping hostname from the dict will result with ignoring it... #}
-{%-         do info_dict.__setitem__('hostname', '') %}
+{%-           do ssl_list.append('none') %}
 {%-         endif %}
-{%-       else %}
-{%-         do ssl.append('none') %}
 {%-       endif %}
-{%-       set ssl = ' '.join(ssl) %}
-{%-     else %}
-{%-       set ssl = '' %}
-{%-     endif %}
 
 backend {{ slave_instance['slave_reference'] }}-{{ scheme }}
-{%-     set hostname = info_dict['hostname'] %}
-{%-     set port = info_dict['port'] %}
-{%-     set path = info_dict['path'].rstrip('/') %}
-{%-     if hostname and port %}
+{%-       set hostname = info_dict['hostname'] %}
+{%-       set port = info_dict['port'] %}
+{%-       set path = info_dict['path'].rstrip('/') %}
+{%-       if hostname and port %}
   timeout server {{ slave_instance['request-timeout'] }}s
   timeout connect {{ slave_instance['backend-connect-timeout'] }}s
   retries {{ slave_instance['backend-connect-retries'] }}
-  server backend {{ hostname }}:{{ port }} {{ ssl }}
-{%-       if path %}
+  server {{ slave_instance['slave_reference'] }}-backend {{ hostname }}:{{ port }} {{ ' '.join(ssl_list) }}
+{%-         if path %}
   http-request set-path {{ path }}%[path]
+{%-         endif %}
 {%-       endif %}
 {%-     endif %}
 {%-   endfor %}

software/caddy-frontend/templates/caddy-backend-url-validator.in

-#!${dash:location}/bin/dash
-config="https://example.com {\n  proxy / $1 {\n  }\n}"
-
-echo -e $config | ${caddy:output} -conf stdin -validate > /dev/null 2>&1

software/caddy-frontend/templates/replicate-publish-slave-information.cfg.in

@@ -75,7 +75,7 @@ log-access-url = {{ dumps(json_module.dumps(log_access_url, sort_keys=True)) }}
 {% endfor %}
 
 [buildout]
-extends = {{ common_profile }}
+extends = {{ profile_common }}
 parts =
 {% for part in part_list %}
 {{ '  %s' % part }}

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_file_list_log-CADDY.txt

@@ -13,13 +13,10 @@ T-2/var/log/httpd/_dummy-cached_access_log
 T-2/var/log/httpd/_dummy-cached_backend_log
 T-2/var/log/httpd/_dummy-cached_error_log
 T-2/var/log/httpd/_enable-http2-default_access_log
-T-2/var/log/httpd/_enable-http2-default_backend_log
 T-2/var/log/httpd/_enable-http2-default_error_log
 T-2/var/log/httpd/_enable-http2-false_access_log
-T-2/var/log/httpd/_enable-http2-false_backend_log
 T-2/var/log/httpd/_enable-http2-false_error_log
 T-2/var/log/httpd/_enable-http2-true_access_log
-T-2/var/log/httpd/_enable-http2-true_backend_log
 T-2/var/log/httpd/_enable-http2-true_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -13,13 +13,10 @@ T-2/var/log/httpd/_dummy-cached_access_log
 T-2/var/log/httpd/_dummy-cached_backend_log
 T-2/var/log/httpd/_dummy-cached_error_log
 T-2/var/log/httpd/_enable-http2-default_access_log
-T-2/var/log/httpd/_enable-http2-default_backend_log
 T-2/var/log/httpd/_enable-http2-default_error_log
 T-2/var/log/httpd/_enable-http2-false_access_log
-T-2/var/log/httpd/_enable-http2-false_backend_log
 T-2/var/log/httpd/_enable-http2-false_error_log
 T-2/var/log/httpd/_enable-http2-true_access_log
-T-2/var/log/httpd/_enable-http2-true_backend_log
 T-2/var/log/httpd/_enable-http2-true_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_file_list_log-CADDY.txt

@@ -13,13 +13,10 @@ T-2/var/log/httpd/_dummy-cached_access_log
 T-2/var/log/httpd/_dummy-cached_backend_log
 T-2/var/log/httpd/_dummy-cached_error_log
 T-2/var/log/httpd/_enable-http2-default_access_log
-T-2/var/log/httpd/_enable-http2-default_backend_log
 T-2/var/log/httpd/_enable-http2-default_error_log
 T-2/var/log/httpd/_enable-http2-false_access_log
-T-2/var/log/httpd/_enable-http2-false_backend_log
 T-2/var/log/httpd/_enable-http2-false_error_log
 T-2/var/log/httpd/_enable-http2-true_access_log
-T-2/var/log/httpd/_enable-http2-true_backend_log
 T-2/var/log/httpd/_enable-http2-true_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -13,13 +13,10 @@ T-2/var/log/httpd/_dummy-cached_access_log
 T-2/var/log/httpd/_dummy-cached_backend_log
 T-2/var/log/httpd/_dummy-cached_error_log
 T-2/var/log/httpd/_enable-http2-default_access_log
-T-2/var/log/httpd/_enable-http2-default_backend_log
 T-2/var/log/httpd/_enable-http2-default_error_log
 T-2/var/log/httpd/_enable-http2-false_access_log
-T-2/var/log/httpd/_enable-http2-false_backend_log
 T-2/var/log/httpd/_enable-http2-false_error_log
 T-2/var/log/httpd/_enable-http2-true_access_log
-T-2/var/log/httpd/_enable-http2-true_backend_log
 T-2/var/log/httpd/_enable-http2-true_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-CADDY.txt

@@ -22,7 +22,6 @@ T-2/var/log/httpd/_auth-to-backend_access_log
 T-2/var/log/httpd/_auth-to-backend_backend_log
 T-2/var/log/httpd/_auth-to-backend_error_log
 T-2/var/log/httpd/_ciphers_access_log
-T-2/var/log/httpd/_ciphers_backend_log
 T-2/var/log/httpd/_ciphers_error_log
 T-2/var/log/httpd/_custom_domain_access_log
 T-2/var/log/httpd/_custom_domain_backend_log
@@ -43,7 +42,6 @@ T-2/var/log/httpd/_disabled-cookie-list_access_log
 T-2/var/log/httpd/_disabled-cookie-list_backend_log
 T-2/var/log/httpd/_disabled-cookie-list_error_log
 T-2/var/log/httpd/_empty_access_log
-T-2/var/log/httpd/_empty_backend_log
 T-2/var/log/httpd/_empty_error_log
 T-2/var/log/httpd/_enable-http2-default_access_log
 T-2/var/log/httpd/_enable-http2-default_backend_log
@@ -79,10 +77,8 @@ T-2/var/log/httpd/_https-only_access_log
 T-2/var/log/httpd/_https-only_backend_log
 T-2/var/log/httpd/_https-only_error_log
 T-2/var/log/httpd/_monitor-ipv4-test_access_log
-T-2/var/log/httpd/_monitor-ipv4-test_backend_log
 T-2/var/log/httpd/_monitor-ipv4-test_error_log
 T-2/var/log/httpd/_monitor-ipv6-test_access_log
-T-2/var/log/httpd/_monitor-ipv6-test_backend_log
 T-2/var/log/httpd/_monitor-ipv6-test_error_log
 T-2/var/log/httpd/_prefer-gzip-encoding-to-backend-https-only_access_log
 T-2/var/log/httpd/_prefer-gzip-encoding-to-backend-https-only_backend_log

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -22,7 +22,6 @@ T-2/var/log/httpd/_auth-to-backend_access_log
 T-2/var/log/httpd/_auth-to-backend_backend_log
 T-2/var/log/httpd/_auth-to-backend_error_log
 T-2/var/log/httpd/_ciphers_access_log
-T-2/var/log/httpd/_ciphers_backend_log
 T-2/var/log/httpd/_ciphers_error_log
 T-2/var/log/httpd/_custom_domain_access_log
 T-2/var/log/httpd/_custom_domain_backend_log
@@ -43,7 +42,6 @@ T-2/var/log/httpd/_disabled-cookie-list_access_log
 T-2/var/log/httpd/_disabled-cookie-list_backend_log
 T-2/var/log/httpd/_disabled-cookie-list_error_log
 T-2/var/log/httpd/_empty_access_log
-T-2/var/log/httpd/_empty_backend_log
 T-2/var/log/httpd/_empty_error_log
 T-2/var/log/httpd/_enable-http2-default_access_log
 T-2/var/log/httpd/_enable-http2-default_backend_log
@@ -79,10 +77,8 @@ T-2/var/log/httpd/_https-only_access_log
 T-2/var/log/httpd/_https-only_backend_log
 T-2/var/log/httpd/_https-only_error_log
 T-2/var/log/httpd/_monitor-ipv4-test_access_log
-T-2/var/log/httpd/_monitor-ipv4-test_backend_log
 T-2/var/log/httpd/_monitor-ipv4-test_error_log
 T-2/var/log/httpd/_monitor-ipv6-test_access_log
-T-2/var/log/httpd/_monitor-ipv6-test_backend_log
 T-2/var/log/httpd/_monitor-ipv6-test_error_log
 T-2/var/log/httpd/_prefer-gzip-encoding-to-backend-https-only_access_log
 T-2/var/log/httpd/_prefer-gzip-encoding-to-backend-https-only_backend_log

software/erp5/test/setup.py

@@ -33,7 +33,7 @@ with open("README.md") as f:
 
 setup(name=name,
       version=version,
-      description="Test for SlapOS' ERP5 software releae",
+      description="Test for SlapOS' ERP5 software release",
       long_description=long_description,
       long_description_content_type='text/markdown',
       maintainer="Nexedi",
@@ -50,8 +50,9 @@ setup(name=name,
         'mysqlclient',
         'backports.lzma',
         'cryptography',
+        'pexpect',
         'pyOpenSSL',
-        ],
-      zip_safe=True,
+        'typing; python_version<"3"',
+      ],
       test_suite='test',
     )

software/erp5/test/test/test_erp5.py

@@ -31,6 +31,7 @@ import glob
 import urlparse
 import socket
 import time
+import tempfile
 
 import psutil
 import requests
@@ -43,7 +44,7 @@ setUpModule # pyflakes
 class TestPublishedURLIsReachableMixin(object):
   """Mixin that checks that default page of ERP5 is reachable.
   """
-  def _checkERP5IsReachable(self, url):
+  def _checkERP5IsReachable(self, url, verify):
     # What happens is that instanciation just create the services, but does not
     # wait for ERP5 to be initialized. When this test run ERP5 instance is
     # instanciated, but zope is still busy creating the site and haproxy replies
@@ -51,7 +52,7 @@ class TestPublishedURLIsReachableMixin(object):
     # erp5 site is not created, with 500 when mysql is not yet reachable, so we
     # retry in a loop until we get a succesful response.
     for i in range(1, 60):
-      r = requests.get(url, verify=False)  # XXX can we get CA from caucase already ?
+      r = requests.get(url, verify=verify)
       if r.status_code != requests.codes.ok:
         delay = i * 2
         self.logger.warn("ERP5 was not available, sleeping for %ds and retrying", delay)
@@ -62,19 +63,36 @@ class TestPublishedURLIsReachableMixin(object):
 
     self.assertIn("ERP5", r.text)
 
+  def _getCaucaseServiceCACertificate(self):
+    ca_cert = tempfile.NamedTemporaryFile(
+        prefix="ca.crt.pem",
+        mode="w",
+        delete=False,
+    )
+    ca_cert.write(
+        requests.get(
+            urlparse.urljoin(
+                self.getRootPartitionConnectionParameterDict()['caucase-http-url'],
+                '/cas/crt/ca.crt.pem',
+            )).text)
+    self.addCleanup(os.unlink, ca_cert.name)
+    return ca_cert.name
+
   def test_published_family_default_v6_is_reachable(self):
     """Tests the IPv6 URL published by the root partition is reachable.
     """
     param_dict = self.getRootPartitionConnectionParameterDict()
     self._checkERP5IsReachable(
-      urlparse.urljoin(param_dict['family-default-v6'], param_dict['site-id']))
+      urlparse.urljoin(param_dict['family-default-v6'], param_dict['site-id']),
+      self._getCaucaseServiceCACertificate())
 
   def test_published_family_default_v4_is_reachable(self):
     """Tests the IPv4 URL published by the root partition is reachable.
     """
     param_dict = self.getRootPartitionConnectionParameterDict()
     self._checkERP5IsReachable(
-      urlparse.urljoin(param_dict['family-default'], param_dict['site-id']))
+      urlparse.urljoin(param_dict['family-default'], param_dict['site-id']),
+      self._getCaucaseServiceCACertificate())
 
 
 class TestDefaultParameters(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):

software/erp5testnode/buildout.hash.cfg

@@ -18,4 +18,4 @@ md5sum = 307663d73ef3ef94b02567ecd322252e
 
 [template-default]
 filename = instance-default.cfg
-md5sum = 40364ff26e9284cea97a58f3cd8c75e3
+md5sum = e553b582cd51db90fcd5634bd055f042

software/erp5testnode/instance-default.cfg

@@ -23,6 +23,7 @@ parts =
 recipe = slapos.cookbook:publish
 url = $${shellinabox-frontend:url}
 frontend-url = $${testnode-frontend:connection-secure_access}
+log-frontend-url = $${testnode-log-frontend:connection-secure_access}
 
 [pwgen]
 recipe = slapos.cookbook:generate.password
@@ -63,6 +64,8 @@ httpd-log-directory = $${basedirectory:log}
 httpd-software-directory = $${directory:software}
 httpd-cert-file = $${rootdirectory:etc}/httpd-public.crt
 httpd-key-file = $${rootdirectory:etc}/httpd-private.key
+frontend-url = $${testnode-frontend:connection-secure_access}
+log-frontend-url = $${testnode-log-frontend:connection-secure_access}
 
 configuration-file = $${rootdirectory:etc}/erp5testnode.cfg
 log-file = $${basedirectory:log}/erp5testnode.log
@@ -212,22 +215,31 @@ recipe = collective.recipe.template
 input = inline: **
 output = $${directory:srv}/exporter.exclude
 
-[testnode-frontend]
+[request-shared-frontend]
 <= slap-connection
 recipe = slapos.cookbook:requestoptional
-name = Test Node Frontend $${testnode:test-node-title}
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config-url = https://[$${testnode:httpd-ip}]:$${testnode:httpd-software-access-port}
 config-https-only = true
-#software-type = custom-personal
 return = domain secure_access
 
+[testnode-frontend]
+<= request-shared-frontend
+name = Test Node Frontend $${testnode:test-node-title}
+config-url = https://[$${testnode:httpd-ip}]:$${testnode:httpd-software-access-port}
+
+[testnode-log-frontend]
+<= request-shared-frontend
+name = Test Node Logs Frontend $${testnode:test-node-title}
+config-url = https://[$${testnode:httpd-ip}]:$${testnode:httpd-port}
+
+
 [promises]
 recipe =
 instance-promises =
   $${shellinabox-frontend-listen-promise:name}
+  $${testnode-log-frontend-promise:name}
 
 [shellinabox-frontend-listen-promise]
 <= monitor-promise-base
@@ -236,6 +248,12 @@ name = $${:_buildout_section_name_}.py
 config-hostname = $${shellinabox-frontend:hostname}
 config-port = $${shellinabox-frontend:port}
 
+[testnode-log-frontend-promise]
+<= monitor-promise-base
+module = check_url_available
+name = $${:_buildout_section_name_}.py
+config-url = $${testnode-log-frontend:connection-secure_access}
+
 [slap-parameter]
 node-quantity = 1
 test-suite-master-url =

software/fluentd/software.cfg

@@ -24,4 +24,3 @@ gems +=
 
 [versions]
 slapos.recipe.template = 4.4
-rubygemsrecipe = 0.2.2+slapos001

software/gitlab/software.cfg

@@ -405,6 +405,5 @@ strip-top-level-dir = true
 cns.recipe.symlink = 0.2.3
 docutils = 0.12
 plone.recipe.command = 1.1
-rubygemsrecipe = 0.2.2+slapos002
 slapos.recipe.template = 4.3
 z3c.recipe.scripts = 1.0.1

software/helloworld/software.cfg

@@ -29,6 +29,9 @@ parts =
   helloweb-go
 
 
+[python]
+part = python3
+
 # Macro for jinja templates. The filename is set in buildout.hash.cfg
 # in the section using this template
 [jinja-template]
@@ -54,4 +57,3 @@ context =
 # Pin versions of eggs used that are not already pinned by stack/slapos.cfg
 [versions]
 slapos.recipe.template = 4.4
-rubygemsrecipe  = 0.2.2+slapos001

software/kvm/buildout.hash.cfg

@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
 
 [template-kvm-run]
 filename = template/template-kvm-run.in
-md5sum = 4a6f149177a453a13436f320f6841518
+md5sum = a97b8462955dd422a30fbe02d6906172
 
 [template-kvm-controller]
 filename = template/kvm-controller-run.in

software/kvm/instance-kvm-cluster-input-schema.json

@@ -505,9 +505,9 @@
             },
             "boot-image-url-list": {
               "title": "Boot image list",
-              "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
+              "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM. Note: Maximum 3 ISOs are supported.",
               "type": "string",
-              "textarea": "true"
+              "textarea": true
             },
             "boot-image-url-select": {
               "title": "Boot image",

software/kvm/instance-kvm-cluster-simplified-input-schema.json

@@ -60,19 +60,53 @@
               ],
               "default": "fr"
             },
-            "nbd-host": {
-              "title": "NBD hostname or IP",
-              "description": "hostname (or IP) of the NBD server containing the boot image.",
-              "type": "string",
-              "format": "internet-address"
-            },
-            "nbd-port": {
-              "title": "NBD port",
-              "description": "Port of the NBD server containing the boot image.",
-              "type": "integer",
-              "default": 1024,
-              "minimum": 1,
-              "maximum": 65535
+            "boot-image-url-select": {
+              "title": "Boot image",
+              "type": "array",
+              "oneOf": [
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/0a6aee1d9aafc1ed095105c052f9fdd65ed00ea9274188c9cd0072c8e6838ab40e246d45a1e6956d74ef1b04a1fc042151762f25412e9ff0cbf49418eef7992e#a3ebc76aec372808ad80000108a2593a"
+                  ],
+                  "title": "Debian Buster 10.5 netinst x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/ce5ddfdbdaccdf929b7fe321212356347d82a02f6b7733427282b416f113d91e587682b003e9d376ac189c3b731595c50c236962aadf2720c16d9f36913577c0#23bf2a2d60271e553e63525e794415f1"
+                  ],
+                  "title": "Centos 8.2004 Minimal x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/302c990c6d69575ff24c96566e5c7e26bf36908abb0cd546e22687c46fb07bf8dba595bf77a9d4fd9ab63e75c0437c133f35462fd41ea77f6f616140cd0e5e6a#f3a306f40e4a313fb5a584d73b3dee8f"
+                  ],
+                  "title": "Ubuntu Focal 20.04.1 Live Server x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/6635269a7eb6fbd6b85fda40cd94f14a27bf53cb1fc82ffcce9fe386a025a43e1ab681db7e8cec50416bfbfc90262f0d95273686a101c74b3f17646f0a34c85b#3708a59af6cf820a95cafe0ae73ac399"
+                  ],
+                  "title": "openSUSE Leap 15.2 NET x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/fc17e8c6ae0790162f4beb8fa6226d945cff638429588999b3a08493ff27b280dc2939fba825ae04be1d9082ea8d7c3c002c5e4c39fbbcf88b8ab5104619e28a#ebcdb2223a77f098af3923fe1fa180aa"
+                  ],
+                  "title": "Arch Linux 2020.09.01 x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/c5a511f349a1146b615e6fab9c24f9be4362046adcf24f0ff82c470d361fac5f6628895e2110ebf8ff87db49d4c413a0a332699da6b1bec64275e0c17a15b999#ca7a1e555c04b4d9a549065fa2ddf713"
+                  ],
+                  "title": "Fedora Server 32-1.6 netinst x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/6c355def68b3c0427f21598cb054ffc893568902f205601ac60f192854769b31bc9cff8eeb6ce99ef975a8fb887d8d3e56fc6cd5ea5cb4b3bba1175c520047cb#57088b77f795ca44b00971e44782ee23"
+                  ],
+                  "title": "FreeBSD 12.1 RELEASE bootonly x86_64"
+                }
+              ]
             }
           },
           "type": "object"

software/kvm/instance-kvm-input-schema.json

@@ -368,9 +368,9 @@
     },
     "boot-image-url-list": {
       "title": "Boot image list",
-      "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
+      "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM. Note: Maximum 3 ISOs are supported.",
       "type": "string",
-      "textarea": "true"
+      "textarea": true
     },
     "boot-image-url-select": {
       "title": "Boot image",

software/kvm/template/template-kvm-run.in

@@ -371,7 +371,8 @@ for nbd_ip, nbd_port in nbd_list:
           '-drive',
           'file=nbd:[%s]:%s,media=cdrom' % (nbd_ip, nbd_port)])
 else:
-  index = 0
+  # Note: Do not get tempted to use virtio-scsi-pci, as it does not work with
+  #       Debian installation CDs, rendering it uninstallable
   if boot_image_url_select_json_config:
     # Support boot-image-url-select
     with open(boot_image_url_select_json_config) as fh:
@@ -381,11 +382,9 @@ else:
         link = os.path.join(image_config['destination-directory'], image['link'])
         if os.path.exists(link) and os.path.islink(link):
           kvm_argument_list.extend([
-            '-drive', 'file=%s,media=cdrom,if=none,id=cdrom%s' % (link, index),
-            '-device', 'virtio-scsi-pci,id=scsi%s' % (index,),
-            '-device', 'scsi-cd,bus=scsi%s.0,drive=cdrom%s' % (index, index)
+            '-drive',
+            'file=%s,media=cdrom' % (link,)
           ])
-          index += 1
   if boot_image_url_list_json_config:
     # Support boot-image-url-list
     with open(boot_image_url_list_json_config) as fh:
@@ -395,16 +394,12 @@ else:
         link = os.path.join(image_config['destination-directory'], image['link'])
         if os.path.exists(link) and os.path.islink(link):
           kvm_argument_list.extend([
-            '-drive', 'file=%s,media=cdrom,if=none,id=cdrom%s' % (link, index),
-            '-device', 'virtio-scsi-pci,id=scsi%s' % (index,),
-            '-device', 'scsi-cd,bus=scsi%s.0,drive=cdrom%s' % (index, index)
+            '-drive',
+            'file=%s,media=cdrom' % (link,)
           ])
-          index += 1
   # Always add by default the default image
   kvm_argument_list.extend([
-      '-drive', 'file=%s,media=cdrom,if=none,id=cdrom%s' % (default_cdrom_iso, index),
-      '-device', 'virtio-scsi-pci,id=scsi%s' % (index,),
-      '-device', 'scsi-cd,bus=scsi%s.0,drive=cdrom%s' % (index, index)
+      '-drive', 'file=%s,media=cdrom' % default_cdrom_iso
   ])
 
 

software/kvm/test/test.py

@@ -604,7 +604,7 @@ class TestBootImageUrlList(InstanceTestCase):
     # check that the image is NOT YET available in kvm
     self.assertEqual(
       ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
-       'media=cdrom,if=none,id=cdrom0'],
+       'media=cdrom'],
       getRunningImageList()
     )
 
@@ -618,12 +618,10 @@ class TestBootImageUrlList(InstanceTestCase):
     # now the image is available in the kvm, and its above default image
     self.assertEqual(
       [
-        'file=/srv/%s/image_001,media=cdrom,if=none,id=cdrom0' % (
-          self.image_directory,),
-        'file=/srv/%s/image_002,media=cdrom,if=none,id=cdrom1' % (
-          self.image_directory,),
+        'file=/srv/%s/image_001,media=cdrom' % (self.image_directory,),
+        'file=/srv/%s/image_002,media=cdrom' % (self.image_directory,),
         'file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
-        'media=cdrom,if=none,id=cdrom2'
+        'media=cdrom'
       ],
       getRunningImageList()
     )
@@ -647,7 +645,7 @@ class TestBootImageUrlList(InstanceTestCase):
     # again only default image is available in the running process
     self.assertEqual(
       ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
-       'media=cdrom,if=none,id=cdrom0'],
+       'media=cdrom'],
       getRunningImageList()
     )
 
@@ -786,7 +784,7 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
     # check that the image is NOT YET available in kvm
     self.assertEqual(
       ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
-       'media=cdrom,if=none,id=cdrom0'],
+       'media=cdrom'],
       getRunningImageList()
     )
 
@@ -800,12 +798,10 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
     # now the image is available in the kvm, and its above default image
     self.assertEqual(
       [
-        'file=/srv/boot-image-url-select-repository/image_001,media=cdrom,'
-        'if=none,id=cdrom0',
-        'file=/srv/boot-image-url-list-repository/image_001,media=cdrom,'
-        'if=none,id=cdrom1',
+        'file=/srv/boot-image-url-select-repository/image_001,media=cdrom',
+        'file=/srv/boot-image-url-list-repository/image_001,media=cdrom',
         'file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
-        'media=cdrom,if=none,id=cdrom2'
+        'media=cdrom'
       ],
       getRunningImageList()
     )
@@ -834,7 +830,7 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
     # again only default image is available in the running process
     self.assertEqual(
       ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
-       'media=cdrom,if=none,id=cdrom0'],
+       'media=cdrom'],
       getRunningImageList()
     )
 

software/neoppod/stress-testing/software.cfg

@@ -78,6 +78,8 @@ packages +=
   ca-certificates file g++ libc6-dev make patch python
 # speed up build by using the following components from the OS
   git liblzma-dev libssl-dev pkg-config python-dev
+# for pygolang
+  python-greenlet-dev
 # extra requirements for NEO
   libnetfilter-queue-dev nftables
 # extra requirements for this SR

software/plantuml/software.cfg

@@ -15,6 +15,9 @@ parts =
   slapos-cookbook
   instance
 
+[python]
+part = python3
+
 [instance]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/${:filename}

software/proftpd/software.cfg

@@ -12,6 +12,8 @@ parts =
   proftpd-config-file
   instance-profile
 
+[python]
+part = python3
 
 [download-file-base]
 recipe = slapos.recipe.build:download

software/repman/buildout.hash.cfg

@@ -14,11 +14,11 @@
 # not need these here).
 [instance.cfg]
 filename = instance.cfg.in
-md5sum = 027cdcfe251b7bba9b779fee890a9162
+md5sum = fd0df2c4a91c1e478b72cbaeb6c021c3
 
 [instance-repman.cfg]
 _update_hash_filename_ = instance-repman.cfg.jinja2.in
-md5sum = 657ecdb1dfbbcf53e4e7932b3b5708c4
+md5sum = 1d6eba3984b3e2009682f6ce49b8ac4d
 
 [config-toml.in]
 _update_hash_filename_ = templates/config.toml.in
@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
 
 [template-mariadb.cfg]
 _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
-md5sum = 189ccee60d0fb53e29431a45e0816bc1
+md5sum = 21a29a41768b2370d671d3086b3ef2bb
 
 [template-my-cnf]
 _update_hash_filename_ = templates/my.cnf.in

software/repman/instance-mariadb.cfg.jinja2.in

@@ -156,7 +156,7 @@ wrapper = ${directory:controller}/mariadb
 {% do part_list.append("supervisord-mariadb") %}
 
 [odbc-ini-text]
-text = {{ dumps(slapparameter_dict.get('odbc-ini', '').encode('base64')) }}
+text = {{ dumps(base64.b64encode( slapparameter_dict.get('odbc-ini', '').encode() )) }}
 
 [{{ section('odbc-ini') }}]
 < = jinja2-template-base

software/repman/instance-repman.cfg.jinja2.in

@@ -359,7 +359,7 @@ update-command = ${:command}
 [replication-manager-reload]
 recipe = slapos.recipe.template:jinja2
 template = {{ template_repman_manager_sh }}
-cluster-list = {{ dumps(slapparameter_dict.get('repman-cluster-dict', default_parameter_dict).keys() ) }}
+cluster-list = {{ dumps( list(slapparameter_dict.get('repman-cluster-dict', default_parameter_dict)) ) }}
 context =
   section parameter_dict repman
   key username repman-parameter:username

software/repman/instance.cfg.in

@@ -101,3 +101,4 @@ template = {{ template_mariadb }}
 filename = instance-mariadb.cfg
 extra-context =
     section parameter_dict template-mariadb-parameters
+    import base64 base64

software/slapos-master/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 2ef0ddc206c6b0982a37cfc21f23e423
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = ef86e09e44ac67a9b15939df0ab4a466
+md5sum = 4998e62351f54700ee23a2ca8cd89329
 
 [template-apache-backend-conf]
 filename = apache-backend.conf.in