Update Release Candidate

component/dcron/buildout.cfg

@@ -2,6 +2,7 @@
 extends =
   ../coreutils/buildout.cfg
   ../patch/buildout.cfg
+  ../randomsleep/buildout.cfg
 
 parts = dcron-output
 
@@ -18,6 +19,7 @@ make-options =
 post-make-hook = ${:_profile_base_location_}/dcron-hooks.py#860e914dff4108b47565965fe5ebc7b5:post_make_hook
 environment =
   PATH=${patch:location}/bin:%(PATH)s
+randomsleep_install = ${randomsleep:location}
 
 [dcron-output]
 # Shared binary location to ease migration

component/golang/buildout.cfg

@@ -34,8 +34,8 @@ make-targets= cd src && ./make.bash && cp -alf .. ${:location}
 
 [golang18]
 <= golang-common
-url = https://storage.googleapis.com/golang/go1.8.3.src.tar.gz
-md5sum = 64e9380e07bba907e26a00cf5fcbe77e
+url = https://dl.google.com/go/go1.8.7.src.tar.gz
+md5sum = c61cfe9c85e7d42f903d3fe146d7cde6
 
 # go1.8 needs go1.4 to bootstrap
 environment-extra =
@@ -43,8 +43,8 @@ environment-extra =
 
 [golang19]
 <= golang-common
-url = https://storage.googleapis.com/golang/go1.9.2.src.tar.gz
-md5sum = 44105c865a1a810464df79233a05a568
+url = https://dl.google.com/go/go1.9.4.src.tar.gz
+md5sum = 6816441fd6680c63865cdd5cb8bc1960
 
 # go1.9 needs go1.4 to bootstrap
 environment-extra =
@@ -63,6 +63,11 @@ environment-extra =
 #       lab.nexedi.com/kirr/neo/go/...  \
 #       github.com/pkg/profile          \
 #       golang.org/x/perf/cmd/benchstat
+#
+# it is possible to specify Go build flags used for compilation e.g. this way:
+#
+#   [gowork]
+#   buildflags = -race
 [gowork]
 directory = ${buildout:directory}/go.work
 src	= ${:directory}/src
@@ -72,6 +77,9 @@ depends = ${gowork.goinstall:recipe}
 # go version used for the workspace (possible to override in applications)
 golang  = ${golang19:location}
 
+# no special build flags by default
+buildflags =
+
 # everything is done by dependent parts
 recipe  = plone.recipe.command
 command = :
@@ -100,7 +108,7 @@ stop-on-error = true
 # clients should put package list to install to gowork:install ("..." requests installing everything)
 [gowork.goinstall]
 recipe  = plone.recipe.command
-command = bash -c ". ${gowork:env.sh} && go install -v ${gowork:install}"
+command = bash -c ". ${gowork:env.sh} && go install ${gowork:buildflags} -v ${gowork:install}"
 update-command = ${:command}
 stop-on-error = true
 
@@ -115,7 +123,7 @@ git-executable = ${git:location}/bin/git
 # 3) provide repository (which is not the same as importpath in general case)
 #
 # the list of go packages for a go workspace state can be automatically
-# generated with the help of go-pkg-snapshot tool.
+# generated with the help of gowork-snapshot tool.
 [go-git-package]
 <= git-repository
 location = ${gowork:src}/${:go.importpath}

component/mariadb/buildout.cfg

@@ -27,8 +27,8 @@ parts =
 [mariadb]
 recipe = slapos.recipe.cmmi
 url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
-version = 10.1.30
-md5sum = c424fd12bdff388e3da1bdecf42626c9
+version = 10.1.31
+md5sum = 14ab0398c019eb531bc29f2c437ccb51
 patch-options = -p0
 patches =
   ${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3

component/python-2.7/asyncore_poll_insteadof_select.patch

+From efa86aa5ad5b89ec329b3cde288cb119efb21c89 Mon Sep 17 00:00:00 2001
+From: Kirill Smelkov <kirr@nexedi.com>
+Date: Thu, 06 Jul 2017 21:15:31 +0200
+Subject: [PATCH] asyncore: switch default to use poll instead of select
+
+Recently we started to get the following errors on a Wendelin production
+instance:
+
+      File ".../bin/runzope", line 211, in <module>
+        sys.exit(Zope2.Startup.run.run())
+      File ".../eggs/Zope2-2.13.24-py2.7.egg/Zope2/Startup/run.py", line 26, in run
+        starter.run()
+      File ".../eggs/Zope2-2.13.24-py2.7.egg/Zope2/Startup/__init__.py", line 111, in run
+        Lifetime.loop()
+      File ".../eggs/Zope2-2.13.24-py2.7.egg/Lifetime/__init__.py", line 43, in loop
+        lifetime_loop()
+      File ".../eggs/Zope2-2.13.24-py2.7.egg/Lifetime/__init__.py", line 53, in lifetime_loop
+        asyncore.poll(timeout, map)
+      File ".../parts/python2.7/lib/python2.7/asyncore.py", line 145, in poll
+        r, w, e = select.select(r, w, e, timeout)
+    ValueError: filedescriptor out of range in select()
+
+Initially we thought the reason here is that number of file descriptors this
+process uses goes beyond allowed limit (65K open files on that instance)
+because wendelin.core uses 1 fd per an array view (opening
+/dev/shm/ramh.XXXXX) but that turned out to be not strictly that case:
+
+The reason here is that select() has limit for how many #fd it can
+monitor at all. The limit is system-specific but on Linux it is usually
+1024 - http://man7.org/linux/man-pages/man2/select.2.html#BUGS :
+
+    $ cat s.c
+    #include <sys/select.h>
+    #include <stdio.h>
+
+    int main() {
+            printf("%d\n", FD_SETSIZE);
+            return 0;
+    }
+
+    $ tcc -run s.c
+    1024
+
+Also select() can monitor only file descriptors which are by itself
+"< FD_SETSIZE", e.g. it cannot monitor fd=1025 even if there are only 10
+opened file descriptors because 1025 is not < 1024.
+
+As was said above in wendelin.core every array view uses 1 file
+descriptor, so if we are using not so small amount of arrays, even
+though #fd does not go beyond process-specific ulimit, because of
+select() usage the total number of allowed opened file descriptors
+becomes essentially 1024.
+
+So let's switch from select() to poll() which does not have this 1024
+ #fd limit. Asyncore already support using poll() out of the box -
+either via passing use_pull=True to asyncore.loop()
+
+https://docs.python.org/2/library/asyncore.html#asyncore.loop
+
+or by using asyncore.poll2() instead of asyncore.poll()
+
+https://github.com/python/cpython/blob/2.7/Lib/asyncore.py#L170
+https://github.com/python/cpython/blob/2.7/Lib/asyncore.py#L125
+
+--------
+
+One option would be to switch asyncore.poll() -> asyncore.poll2() for only 1
+place in Zope - inside lifetime_loop(). There are however many such similar
+places in Zope (e.g. in medusa) and in other software.
+
+For this reason, for me, what makes sense to do is not to patch all such
+places, but instead change via runtime-patching asyncore.poll to be
+asyncore.poll2 - this way all software will automatically benefit from
+poll() usage instead of select.
+
+P.S.
+
+What might also make sense to do in the future is to actually let
+asyncore.poll to use epoll(), as both select() and poll() are doing all
+fd registration on _every_ call, so when #fd grows this preparation
+time grows too. For epoll() file descriptors are registered only once.
+For this to work asyncore.socket_map has to be patched also, since there
+are places in code which modify this fd registry directly (e.g. remove
+fd from there etc)
+
+Original patch & discussion: https://lab.nexedi.com/kirr/Zope/commit/c6addb05
+
+---
+ Lib/asyncore.py | 9 +++++++++
+ 1 file changed, 9 insertions(+)
+
+diff --git a/Lib/asyncore.py b/Lib/asyncore.py
+index 105982f..a3025a4 100644
+--- a/Lib/asyncore.py
++++ b/Lib/asyncore.py
+@@ -661,3 +661,12 @@ if os.name == 'posix':
+             self.socket = file_wrapper(fd)
+             self._fileno = self.socket.fileno()
+             self.add_channel()
++
++# monkeypatch to switch everyone's default to use poll instead of select
++# (select has 1024 fd limit)
++# TODO it is better to use epoll
++_poll_select = poll
++_poll_poll   = poll2
++poll = _poll_poll
++
++#print >>sys.stderr, 'I(nxd): asyncore: using poll instead of select'
+-- 
+2.16.1.101.gde0f0111ea

component/python-2.7/buildout.cfg

@@ -43,6 +43,7 @@ patches =
   ${:_profile_base_location_}/fix_compiler_module_issue_20613.patch#94443a77f903e9de880a029967fa6aa7
   ${:_profile_base_location_}/pytracemalloc_pep445.patch#46662cf0ccc7cb7cfb8289bbfd68b21a
   ${:_profile_base_location_}/disabled_module_list.patch#71ad30d32bcdbc50c19cf48675b1246e
+  ${:_profile_base_location_}/asyncore_poll_insteadof_select.patch#ab6991c0ee6e25aeb8951e71f280a2f1
 url =
   http://www.python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz
 configure-options =

component/qemu-kvm/buildout.cfg

@@ -19,12 +19,11 @@ extends =
 [kvm]
 recipe = slapos.recipe.cmmi
 # qemu-kvm and qemu are now the same since 1.3.
-url = http://wiki.qemu-project.org/download/qemu-2.8.0.tar.bz2
-md5sum = 17940dce063b6ce450a12e719a6c9c43
+url = http://wiki.qemu-project.org/download/qemu-2.11.0.tar.bz2
+md5sum = 335994a755bc655e88a87aeb36bfc0b9
 configure-options =
   --target-list="$(uname -m 2>/dev/null|sed 's,^i[456]86$,i386,')-softmmu"
   --enable-system
-  --with-system-pixman
   --disable-sdl
   --disable-xen
   --disable-vnc-sasl
@@ -58,22 +57,18 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
 <= debian-netinst-base
 arch = amd64
 
-[debian-amd64-squeeze-netinst.iso]
-<= debian-amd64-netinst-base
-version = 6.0.10
-md5sum = 7f82d341561035f65933da43f94d5b52
-
-[debian-amd64-wheezy-netinst.iso]
+[debian-amd64-jessie-netinst.iso]
+# Download the installer of Debian 8 (Jessie)
 <= debian-amd64-netinst-base
-version = 7.11.0
-md5sum = 096c1c18b44c269808bd815d58c53c8f
+release = archive
+version = 8.10.0
+md5sum = 19dcfc381bd3e609c6056216d203f5bc
 
 [debian-amd64-netinst.iso]
-# Download the installer of Debian 8 (Jessie)
 <= debian-amd64-netinst-base
 release = release/current
-version = 8.6.0
-md5sum = e9f61bf327db6d8f7cee05a99f2353cc
+version = 9.3.0
+md5sum = db8ab7871bc2b7d456c4746e706fb5d3
 
 [debian-amd64-testing-netinst.iso]
 # Download the installer of Debian Stretch

component/randomsleep/buildout.cfg

+[buildout]
+extends = 
+  ../coreutils/buildout.cfg
+  ../bash/buildout.cfg
+
+[randomsleep]
+recipe = slapos.recipe.build
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+bin_dir = ${buildout:bin-directory}
+bash_script_code = 
+  if [ "$#" -ne 1 ]; then
+    echo "usage: randomsleep maxseconds"
+    exit
+  fi
+  ${coreutils:location}/bin/sleep $((RANDOM % $1)).$((RANDOM % 100))
+wrapper_script_code = 
+  #!${bash:location}/bin/bash
+  ${bash:location}/bin/bash ${:location}/randomsleep.bash "$@"
+script = 
+  os.makedirs(self.options['location'])
+
+  bash_script_path = os.path.join(self.options['location'], 'randomsleep.bash')
+  with open(bash_script_path, 'w') as f:
+    f.write(self.options['bash_script_code'])
+
+  wrapper_script_path = os.path.join(self.options['bin_dir'], 'randomsleep')
+  with open(wrapper_script_path, 'w') as f:
+    f.write(self.options['wrapper_script_code'])
+  os.chmod(wrapper_script_path, 0750)

component/tensorboard/0.4.patch → component/tensorboard/0.4.0.patch

 diff --git a/tensorboard/pip_package/build_pip_package.sh b/tensorboard/pip_package/build_pip_package.sh
-index b386d59..f03b056 100755
+index 88fb5a2..782dbc5 100755
 --- a/tensorboard/pip_package/build_pip_package.sh
 +++ b/tensorboard/pip_package/build_pip_package.sh
 @@ -26,6 +26,7 @@ function main() {
@@ -10,14 +10,14 @@ index b386d59..f03b056 100755
  
    echo $(date) : "=== Using tmpdir: ${TMPDIR}"
  
-@@ -45,8 +46,8 @@ function main() {
+@@ -44,9 +45,7 @@ function main() {
+   pushd ${TMPDIR} >/dev/null
    rm -f MANIFEST
-   echo $(date) : "=== Building wheel"
-   echo $(pwd)
--  python setup.py bdist_wheel >/dev/null
--  python3 setup.py bdist_wheel >/dev/null
+   echo $(date) : "=== Building python2 wheel in $PWD"
+-  python setup.py bdist_wheel --python-tag py2 >/dev/null
+-  echo $(date) : "=== Building python3 wheel in $PWD"
+-  python setup.py bdist_wheel --python-tag py3 >/dev/null
 +  PYTHONPATH=${WORKDIR}/${RUNFILES} $PYTHON_BIN_PATH setup.py bdist_egg >/dev/null
-+  #python3 setup.py bdist_wheel >/dev/null
    mkdir -p ${DEST}
    cp dist/* ${DEST}
-   popd
+   popd >/dev/null

component/tensorboard/buildout.cfg

@@ -11,11 +11,11 @@ parts =
 recipe = plone.recipe.command
 stop-on-error = true
 repository = https://github.com/tensorflow/tensorboard
-tag = 0.4
+tag = 0.4.0
 git-binary = ${git:location}/bin/git
 patch-binary = ${patch:location}/bin/patch
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
-command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/0.4.patch ) || (rm -fr ${:location}; exit 1)
+command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/0.4.0.patch ) || (rm -fr ${:location}; exit 1)
 
 [tensorboard-build]
 recipe = slapos.recipe.build
@@ -61,7 +61,7 @@ numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter}
 script =
   os.makedirs(location)
   workdir = self.options['tensorboard-repository-path']
-  egg_name = 'tensorflow_tensorboard-0.4.0rc3-py2.7.egg'
+  egg_name = 'tensorflow_tensorboard-0.4.0-py2.7.egg'
   dist_dir = os.path.join(workdir, 'dist')
   dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
   env = {'PATH':':'.join([self.options['bazel-bin'],

component/util-linux/buildout.cfg

@@ -5,8 +5,8 @@ extends =
 
 [util-linux]
 recipe = slapos.recipe.cmmi
-url = http://www.kernel.org/pub/linux/utils/util-linux/v2.29/util-linux-2.29.tar.xz
-md5sum = 07b6845f48a421ad5844aa9d58edb837
+url = https://www.kernel.org/pub/linux/utils/util-linux/v2.31/util-linux-2.31.1.tar.xz
+md5sum = 7733b583dcb51518944d42aa62ef19ea
 configure-options =
   --disable-static
   --enable-libuuid
@@ -29,7 +29,7 @@ configure-options =
   --disable-rename
   --disable-schedutils
   --disable-setterm
-  --disable-libsmartcols
+  --enable-libsmartcols
   --disable-switch_root
   --disable-tinfo
   --disable-tls

component/zbar/buildout.cfg

+[buildout]
+parts=
+  zbar
+
+[zbar]
+recipe = slapos.recipe.cmmi
+url = https://jaist.dl.sourceforge.net/project/zbar/zbar/0.10/zbar-0.10.tar.bz2
+md5sum = 0fd61eb590ac1bab62a77913c8b086a5
+configure-options = 
+  --disable-video
+  --without-imagemagick
+  --without-gtk
+  --without-xv
+  --without-qt
+  --without-python 
+  --without-x
+  --without-jpg
+environment =
+  CFLAGS=

component/zbarlight/buildout.cfg

+[buildout]
+extends =
+  ../pillow/buildout.cfg
+  ../zbar/buildout.cfg
+parts =
+  zbarlight
+
+[zbarlight]
+recipe = zc.recipe.egg:custom
+egg = zbarlight
+include-dirs =
+  ${zbar:location}/include
+library-dirs =
+  ${zbar:location}/lib
+rpath =
+  ${zbar:location}/lib
+depend-eggs = ${pillow-python:egg}

software/agent/software.cfg

@@ -50,8 +50,8 @@ gitdb = 0.6.4
 pycrypto = 2.6.1
 pycurl = 7.43.0
 slapos.recipe.download = 1.0
-slapos.recipe.template = 4.2
-slapos.toolbox = 0.73
+slapos.recipe.template = 4.3
+slapos.toolbox = 0.74
 smmap = 0.9.0
 
 # Required by:

software/apache-frontend/software.cfg

@@ -10,8 +10,8 @@ gitdb = 0.6.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
 rdiff-backup = 1.0.5+SlapOSPatched001
-slapos.recipe.template = 4.2
-slapos.toolbox = 0.73
+slapos.recipe.template = 4.3
+slapos.toolbox = 0.74
 smmap = 0.9.0
 numpy = 1.11.2
 pyasn1 = 0.2.3

software/cdn-me/software.cfg

@@ -40,4 +40,4 @@ cns.recipe.symlink = 0.2.3
 collective.recipe.environment = 0.2.0
 erp5.util = 0.4.49
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3

software/dream/software.cfg

@@ -53,7 +53,7 @@ mysqlclient = 1.3.12
 # indirect dependancies
 cp.recipe.cmd = 0.5
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3
 zope.exceptions = 4.0.7
 zope.testing = 4.1.3
 zc.recipe.testrunner = 2.0.0

software/erp5/instance-erp5-input-schema.json

@@ -117,7 +117,7 @@
             "timerserver-interval": {
               "description": "Timerserver tick perdiod, in seconds, or 0 to disable",
               "default": 5,
-              "type": "integer"
+              "type": "number"
             },
             "ssl-authentication": {
               "title": "Enable SSL Client authentication on this zope instance.",

software/erp5testnode/software.cfg

@@ -59,7 +59,7 @@ mode = 0644
 [versions]
 PyXML = 0.8.5
 erp5.util = 0.4.50
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3
 ipython = 5.3.0
 apache-libcloud = 2.1.0
 gitdb2 = 2.0.2

software/erp5testnode/testsuite/deploy-test/runTestSuite.py

@@ -106,6 +106,7 @@ def waitForSite(partition_path):
       date=strftime("%Y/%m/%d %H:%M:%S", gmtime(end)),
       duration=end - start,
   )
+  print(try_info + 'status_dict %r' % (status_dict,))
   return status_dict
 
 

software/erp5testnode/testsuite/dummy/software.cfg

@@ -15,4 +15,4 @@ md5sum = efd3b712a2294207f265a9c45648d5cf
 mode = 0644
 
 [versions]
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3

software/gitlab/software.cfg

@@ -381,5 +381,5 @@ cns.recipe.symlink = 0.2.3
 docutils = 0.12
 plone.recipe.command = 1.1
 rubygemsrecipe  = 0.2.2+slapos001
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3
 z3c.recipe.scripts = 1.0.1

software/html5as/software.cfg

@@ -72,7 +72,7 @@ async = 0.6.1
 gitdb = 0.5.4
 pycrypto = 2.6
 rdiff-backup = 1.0.5+SlapOSPatched001
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3
 slapos.toolbox = 0.40.4
 smmap = 0.8.2
 plone.recipe.command = 1.1

software/html5ide/software.cfg

@@ -48,4 +48,4 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3

software/jstestnode/software.cfg

@@ -113,5 +113,5 @@ mode = 0644
 
 [versions]
 erp5.util = 0.4.50
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3
 selenium = 3.8.0

software/jupyter/software.cfg

@@ -85,7 +85,7 @@ pyzmq = 16.0.2
 scikit-learn = 0.18.1
 seaborn = 0.7.1
 simplegeneric = 0.8.1
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3
 statsmodels = 0.8.0
 terminado = 0.6
 tornado = 4.4.2

software/kvm/common.cfg

@@ -41,6 +41,7 @@ parts = ${:common-parts}
 
 [eggs]
 recipe = zc.recipe.egg
+interpreter = python.eggs
 eggs =
   ${python-cffi:egg}
   ${python-cryptography:egg}
@@ -89,7 +90,7 @@ command =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = bf5ef731c0d8da0267a4939882b4eeee 
+md5sum = 5a17fc127190bbc19361c5ffb10711b3 
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
@@ -98,7 +99,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
 mode = 644
-md5sum = e2b8f86bdc12c86e7d959b55c6d54f6d
+md5sum = e3cc9ffe857da1078e321cab65173fb1
 download-only = true
 on-update = true
 
@@ -107,7 +108,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
 mode = 644
-md5sum = 05b6004e8c7a94de14f247affcef4971
+md5sum = 26d931a0279d49eadb3881f440b623bc
 download-only = true
 on-update = true
 
@@ -185,7 +186,7 @@ ignore-existing = true
 url = ${:_profile_base_location_}/template/template-kvm-run.in
 mode = 644
 filename = template-kvm-run.in
-md5sum = 178a24cdad77cb6c2e519ac629dd0e74
+md5sum = 122bf5e8c7a12bceef1bdd6d6b54f4d7
 download-only = true
 on-update = true
 
@@ -195,7 +196,7 @@ ignore-existing = true
 url = ${:_profile_base_location_}/template/kvm-controller-run.in
 mode = 644
 filename = kvm-controller-run.in
-md5sum = 71afd2d13f6e56993ae413a168e012d7
+md5sum = 7e6c79232cc88c15ed21c112ff801b76
 download-only = true
 on-update = true
 
@@ -225,7 +226,7 @@ ignore-existing = true
 url = ${:_profile_base_location_}/template/qemu-is-ready.in
 mode = 644
 filename = qemu-is-ready.in
-md5sum = 0066fa0f5f3dd47bded5e5924df2550d
+md5sum = b304eec8e2cb71f10ea83cac22f6db12
 download-only = true
 on-update = true
 

software/kvm/instance-kvm-cluster-input-schema.json

@@ -189,13 +189,35 @@
               "default": "started",
               "enum": ["started", "stopped"]
             },
+            "enable-device-hotplug": {
+              "title": "Enable device hotplug mode",
+              "description": "If yes, this will allow to Create devices like CPU and Memory in hotplug mode without restart the VM. Operatin System should be configured to Online new created devices.",
+              "type": "boolean",
+              "default": false
+            },
             "ram-size": {
               "title": "RAM size",
               "description": "RAM size, in MB.",
               "type": "integer",
               "default": 1024,
-              "minimum": 128,
-              "multipleOf": 128
+              "minimum": 1024,
+              "multipleOf": 512
+            },
+            "ram-max-size": {
+              "title": "Maximum RAM size, in MB",
+              "description": "Define the maximum size of the memory. The size is in MB and should be a multiple of 512.",
+              "type": "integer",
+              "default": 51200,
+              "minimum": 1024,
+              "multipleOf": 512
+            },
+            "ram-hotplug-slot-size": {
+              "title": "Size of Hotpluggable RAM slot, in MB",
+              "description": "Define the RAM size to plug on one hotpluggable slot in MB, understand the size of one RAM bar. The RAM hotplugged on each slot will always have the same RAM size.",
+              "type": "integer",
+              "default": 512,
+              "minimum": 512,
+              "multipleOf": 512
             },
             "auto-ballooning": {
               "title": "Enable qemu auto ballooning.",
@@ -245,10 +267,13 @@
               "type": "integer",
               "minimum": 1
             },
-            "cpu-options": {
-              "title": "CPU Additional options: cores, threads, sockets, maxcpus.",
-              "description": "Additional options to use with cpu-count. Options are separated by coma: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]. Set this option if you know what you're doing.",
-              "type": "string"
+            "cpu-max-count": {
+              "title": "Maximum CPU amount",
+              "description": "Specifies the maximum number of CPUs.",
+              "type": "integer",
+              "default": 24,
+              "minimum": 1,
+              "maximum": 64
             },
             "numa": {
               "title": "Simulate a multi node NUMA system.",

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -8,7 +8,7 @@
 {% set slave_frontend_iguid = slave_frontend_dict.get('instance-guid', '') -%}
 {% set kvm_instance_dict = {} -%}
 {% set kvm_hostname_list = [] -%}
-{% set monitor_url_list = [] -%}
+{% set monitor_base_url_dict = {} -%}
 
 {% macro setconfig(name, value) -%}
 {# will set a config-name = value if value is not empty -#}
@@ -51,12 +51,15 @@ config-authorized-key = {{ dumps(slapparameter_dict.get('authorized-keys') | joi
 config-nbd-port = {{ dumps(kvm_parameter_dict.get('nbd-port', 1024)) }}
 config-nbd2-port = {{ dumps(kvm_parameter_dict.get('nbd-port2', 1024)) }}
 config-ram-size = {{ dumps(kvm_parameter_dict.get('ram-size', 1024)) }}
+config-ram-max-size = {{ dumps(kvm_parameter_dict.get('ram-max-size', '50G')) }}
+config-enable-device-hotplug = {{ dumps(kvm_parameter_dict.get('enable-device-hotplug', False)) }}
+config-ram-hotplug-slot-size = {{ dumps(kvm_parameter_dict.get('ram-hotplug-slot-size', 512)) }}
 config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 10)) }}
 config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }}
 config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 1)) }}
+config-cpu-max-count = {{ dumps(kvm_parameter_dict.get('cpu-max-count', 24)) }}
 {{ setconfig('numa', kvm_parameter_dict.get('numa', '')) }}
 {{ setconfig('machine-options', kvm_parameter_dict.get('machine-options', '')) }}
-{{ setconfig('cpu-options', kvm_parameter_dict.get('cpu-options', '')) }}
 {{ setconfig('nbd-host', kvm_parameter_dict.get('nbd-host', '')) }}
 {{ setconfig('host2', kvm_parameter_dict.get('host2', '')) }}
 
@@ -67,7 +70,7 @@ config-auto-ballooning = {{ dumps(kvm_parameter_dict.get('auto-ballooning', True
 {{ setconfig('disk-cache', kvm_parameter_dict.get('disk-cache', '')) }}
 
 {% set nat_rules_list = kvm_parameter_dict.get('nat-rules', []) -%}
-{{ setconfig('nat-rules', ' '.join(nat_rules_list)) }}
+{{ setconfig('nat-rules', nat_rules_list | join(' ')) }}
 config-publish-nat-url = True
 config-use-nat = {{ use_nat }}
 config-use-tap = {{ dumps(kvm_parameter_dict.get('use-tap', True)) }}
@@ -124,10 +127,10 @@ return =
 {% if str(kvm_parameter_dict.get('use-tap', 'True')).lower() == 'true' -%}
 {{ '  ' }}tap-ipv4
 
-{% do monitor_url_list.append('${' ~ section ~ ':connection-monitor-base-url}') -%}
 {% do publish_dict.__setitem__('lan-' ~ instance_name, '${' ~ section ~ ':connection-tap-ipv4}') -%}
 {% do kvm_hostname_list.append(instance_name ~ ' ' ~ '${' ~ section ~ ':connection-tap-ipv4}') -%}
 {% endif -%}
+{% do monitor_base_url_dict.__setitem__(instance_name, '${' ~ section ~ ':connection-monitor-base-url}') -%}
 {% do publish_dict.__setitem__(instance_name ~ '-backend-url', '${' ~ section ~ ':connection-backend-url}') -%}
 {% do publish_dict.__setitem__(instance_name ~ '-url', '${' ~ section ~ ':connection-url}') -%}
 {% do kvm_instance_dict.__setitem__(instance_name, (use_nat, nat_rules_list)) -%}
@@ -241,11 +244,12 @@ cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.of
 username = admin
 password = ${monitor-htpasswd:passwd}
 
-[monitor-conf-parameters]
-monitor-url-list +=
-{% for url in monitor_url_list -%}
-{{ '  ' ~ url }}
+[monitor-base-url-dict]
+{% for key, value in monitor_base_url_dict.items() -%}
+{{ key }} = {{ value }}
 {% endfor %}
+
+[monitor-conf-parameters]
 private-path-list += 
   ${directory:webroot}/
 

software/kvm/instance-kvm-input-schema.json

@@ -4,13 +4,35 @@
 
   "title": "Input Parameters",
   "properties": {
+    "enable-device-hotplug": {
+      "title": "Enable device hotplug mode",
+      "description": "If yes, this will allow to Create devices like CPU and Memory in hotplug mode without restart the VM. Operatin System should be configured to Online new created devices.",
+      "type": "boolean",
+      "default": false
+    },
     "ram-size": {
       "title": "RAM size",
       "description": "RAM size, in MB.",
       "type": "integer",
       "default": 1024,
-      "minimum": 128,
-      "multipleOf": 128
+      "minimum": 1024,
+      "multipleOf": 512
+    },
+    "ram-max-size": {
+      "title": "Maximum RAM size, in MB",
+      "description": "Define the maximum size of the memory. The size is in MB and should be a multiple of 512.",
+      "type": "integer",
+      "default": 51200,
+      "minimum": 1024,
+      "multipleOf": 512
+    },
+    "ram-hotplug-slot-size": {
+      "title": "Size of Hotpluggable RAM slot, in MB",
+      "description": "Define the RAM size to plug on one hotpluggable slot in MB, understand the size of one RAM bar. The RAM hotplugged on each slot will always have the same RAM size.",
+      "type": "integer",
+      "default": 512,
+      "minimum": 512,
+      "multipleOf": 512
     },
     "auto-ballooning": {
       "title": "Enable qemu auto ballooning.",
@@ -54,10 +76,25 @@
       "type": "integer",
       "minimum": 1
     },
-    "cpu-options": {
-      "title": "CPU Additional options: cores, threads, sockets, maxcpus.",
-      "description": "Additional options to use with cpu-count. Options are separated by coma: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]. Set this option if you know what you're doing.",
-      "type": "string"
+    "cpu-max-count": {
+      "title": "Maximum CPU amount",
+      "description": "Specifies the maximum number of CPUs.",
+      "type": "integer",
+      "default": 24,
+      "minimum": 1,
+      "maximum": 64
+    },
+    "max-cpu-hotplug-count": {
+      "title": "Maximum hotpluggable CPU amount",
+      "description": "Specifies the maximum number of hotpluggable CPUs.",
+      "type": "integer",
+      "default": 1
+    },
+    "cpu-hotplug-amount": {
+      "title": "CPU hotplug amount",
+      "description": "Specifies the number of CPUs to hotplug",
+      "type": "integer",
+      "default": 0
     },
     "numa": {
       "title": "Simulate a multi node NUMA system.",

software/kvm/instance-kvm.cfg.jinja2

@@ -5,6 +5,7 @@
 {% set nat_restrict = slapparameter_dict.get('nat-restrict-mode', 'False').lower() -%}
 {% set name = slapparameter_dict.get('name', 'localhost') -%}
 {% set disable_ansible_promise = slapparameter_dict.get('disable-ansible-promise', 'True').lower() -%}
+{% set enable_device_hotplug = slapparameter_dict.get('enable-device-hotplug', 'false').lower() -%}
 {% set instance_type = slapparameter_dict.get('type', 'standalone') -%}
 {% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
 {% set frontend_software_type = 'default' -%}
@@ -62,10 +63,15 @@ storage-path = ${directory:srv}/passwd
 bytes = 8
 
 [kvm-controller-parameter-dict]
-python-path = {{ python_executable }}
+python-path = {{ python_eggs_executable }}
 vnc-passwd = ${gen-passwd:passwd}
 socket-path = ${directory:var}/qmp_socket
+pid-file = ${directory:run}/pid_file
 kvm-status-path = ${directory:var}/qemu-vm-is-ready
+cpu-count = ${slap-parameter:cpu-count}
+ram-hotplug-slot-size = ${slap-parameter:ram-hotplug-slot-size}
+ram-size = ${slap-parameter:ram-size}
+enable-device-hotplug = ${slap-parameter:enable-device-hotplug}
 
 [kvm-parameter-dict]
 python-path = {{ python_executable }}
@@ -87,13 +93,16 @@ disk-size = ${slap-parameter:disk-size}
 disk-type = ${slap-parameter:disk-type}
 
 
-pid-file-path = ${directory:run}/pid_file
+pid-file-path = ${kvm-controller-parameter-dict:pid-file}
 socket-path = ${kvm-controller-parameter-dict:socket-path}
 
-smp-count = ${slap-parameter:cpu-count}
-smp-options = ${slap-parameter:cpu-options}
-ram-size = ${slap-parameter:ram-size}
-numa = ${slap-parameter:numa}
+enable-device-hotplug = ${kvm-controller-parameter-dict:enable-device-hotplug}
+smp-count = ${kvm-controller-parameter-dict:cpu-count}
+smp-max-count = ${slap-parameter:cpu-max-count}
+
+ram-size = ${kvm-controller-parameter-dict:ram-size}
+ram-max-size = ${slap-parameter:ram-max-size}
+init-ram-size = 1024
 mac-address = ${create-mac:mac-address}
 tap-mac-address = ${create-tap-mac:mac-address}
 
@@ -143,7 +152,7 @@ disk-cache = ${slap-parameter:disk-cache}
 disk-aio = ${slap-parameter:disk-aio}
 auto-ballooning = ${slap-parameter:auto-ballooning}
 machine-options = ${slap-parameter:machine-options}
-cpu-options = ${slap-parameter:cpu-model}
+cpu-hotplug-slot-size = ${slap-parameter:cpu-model}
 
 log-file = ${directory:log}/qemu.log
 
@@ -550,15 +559,15 @@ nbd-host =
 nbd2-port = 1024
 nbd2-host =
 
+enable-device-hotplug = False
 ram-size = 1024
+ram-max-size = 50G
+ram-hotplug-slot-size = 512
 disk-size = 10
 disk-type = virtio
 
 cpu-count = 1
-# cpu-option is a string: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]
-cpu-options = 
-# list of numa options separate by space: node,nodeid=1,cpus=9-15 node,nodeid=2,cpus=1,3,7
-numa = 
+cpu-max-count = 24
 disk-cache = writeback
 disk-aio = native
 auto-ballooning = True

software/kvm/instance.cfg.in

@@ -93,6 +93,7 @@ context =
     raw netcat_bin ${netcat:location}/bin/netcat
     raw openssl_executable_location ${openssl:location}/bin/openssl
     raw python_executable ${buildout:executable}
+    raw python_eggs_executable ${buildout:bin-directory}/${eggs:interpreter}
     raw qemu_executable_location ${kvm:location}/bin/qemu-system-x86_64
     raw qemu_img_executable_location ${kvm:location}/bin/qemu-img
     raw qemu_start_promise_tpl ${template-qemu-ready:location}/${template-qemu-ready:filename}

software/kvm/software.cfg

@@ -5,13 +5,13 @@ extends = common.cfg
 # XXX - use websockify = 0.5.1 for compatibility with kvm frontend
 websockify = 0.5.1
 
-slapos.toolbox = 0.73
+slapos.toolbox = 0.74
 erp5.util = 0.4.49
 apache-libcloud = 1.1.0
 collective.recipe.environment = 0.2.0
 gitdb = 0.6.4
 pycurl = 7.43.0
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3
 smmap = 0.9.0
 # websockify = 0.8.0
 
@@ -45,4 +45,4 @@ paramiko = 2.0.2
 
 # Required by:
 # slapos.toolbox==0.71
-passlib = 1.6.5
\ No newline at end of file
+passlib = 1.6.5

software/kvm/template/kvm-controller-run.in

@@ -6,40 +6,58 @@
 import socket
 import time
 import os
+from slapos.qemuqmpclient import QemuQMPWrapper, getInitialQemuResourceDict
 
 # XXX: to be factored with slapos.toolbox qemu qmp wrapper.
 
 socket_path = '{{ parameter_dict.get("socket-path") }}'
+pid_file = '{{ parameter_dict.get("pid-file") }}'
 vnc_password = '{{ parameter_dict.get("vnc-passwd") }}'
 status_path = '{{ parameter_dict.get("kvm-status-path") }}'
+cpu_amount = {{ parameter_dict.get("cpu-count") }}
+cpu_model = '{{ parameter_dict.get("cpu-model", "qemu64-x86_64-cpu") }}'
+slot_hotplug_size = {{ parameter_dict.get("ram-hotplug-slot-size", 512) }}
+ram_size = {{ parameter_dict.get("ram-size") }}
+enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lower()
 
-if os.path.exists(status_path):
-  os.unlink(status_path)
+def write(message):
+  with open(status_path, 'w') as status_file:
+    status_file.write(message)
+
+def update():
 
-# Connect to KVM qmp socket
-so = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-connected = False
-while not connected:
   try:
-    so.connect(socket_path)
-  except socket.error:
-    time.sleep(1)
-  else:
-    connected = True
-data = so.recv(1024)
-
-# Enable qmp
-so.send('{ "execute": "qmp_capabilities" }')
-data = so.recv(1024)
-
-# Set VNC password
-so.send('{ "execute": "change", ' \
-        '"arguments": { "device": "vnc", "target": "password", ' \
-        '               "arg": "' + vnc_password + '" } }')
-data = so.recv(1024)
-
-# Finish
-so.close()
-
-with open(status_path, 'w') as status_file:
-  status_file.write("OK")
+    init_dict = getInitialQemuResourceDict(pid_file)
+    init_ram_size = int(init_dict['ram'].split('M')[0])
+    if cpu_amount < 1:
+      raise ValueError("CPU should be at least equal to 1.")
+    hotplug_ram = ram_size - init_ram_size
+    if hotplug_ram < 0:
+      raise ValueError("RAM size cannot be less than the initial value %s MB" % init_ram_size)
+    if os.path.exists(status_path):
+      os.unlink(status_path)
+
+    qemu_wrapper = QemuQMPWrapper(socket_path)
+    qemu_wrapper.setVNCPassword(vnc_password)
+    if enable_device_hotplug == 'true':
+      write("Qemu Controller is still running...")
+      qemu_wrapper.updateDevice({
+        'device': 'cpu',
+        'amount': cpu_amount,
+        'model': cpu_model
+      })
+      qemu_wrapper.updateDevice({
+        'device': 'memory',
+        'mem': hotplug_ram,
+        'slot': slot_hotplug_size,
+        'nslot': 128,
+        'canreboot': 1
+      })
+  except Exception, e:
+    write(str(e))
+    raise
+
+  write("")
+
+if __name__ == "__main__":
+  update()

software/kvm/template/qemu-is-ready.in

 #!{{ dash }}
 
-if [ -f "{{ qemu_ready_path }}" ]; then
-  echo "VM correctly started."
+FILE="{{ qemu_ready_path }}"
+# don't start checks too fast
+sleep 2
+if [ -f "$FILE" ]; then
+  if [ "$(cat $FILE)" = "" ]; then
+    echo "VM correctly started."
+  else
+    >&2 echo "Qemu Controller failed"
+    >&2 cat $FILE
+    exit 1
+  fi
 else
   log_file="{{ qemu_service_log_file }}"
   >&2 echo "Qemu process is not correctly started."

software/kvm/template/template-kvm-run.in

@@ -39,13 +39,13 @@ tap_interface = '{{ parameter_dict.get("tap-interface") }}'
 listen_ip = '{{ parameter_dict.get("ipv4") }}'
 mac_address = '{{ parameter_dict.get("mac-address") }}'
 tap_mac_address = '{{ parameter_dict.get("tap-mac-address") }}'
-smp_count = '{{ parameter_dict.get("smp-count") }}'
-smp_options = '{{ parameter_dict.get("smp-options") }}'.strip()
-numa_list = '{{ parameter_dict.get("numa") }}'.split()
-ram_size = '{{ parameter_dict.get("ram-size") }}'
+numa_list = '{{ parameter_dict.get("numa", "") }}'.split()
+ram_size = {{ parameter_dict.get("ram-size") }}
+ram_max_size = '{{ parameter_dict.get("ram-max-size") }}'
+init_ram_size = {{ parameter_dict.get("init-ram-size") }}
 pid_file_path = '{{ parameter_dict.get("pid-file-path") }}'
 external_disk_number = {{ parameter_dict.get("external-disk-number") }}
-external_disk_size = '{{ parameter_dict.get("external-disk-size") }}'
+external_disk_size = {{ parameter_dict.get("external-disk-size") }}
 external_disk_format = '{{ parameter_dict.get("external-disk-format") }}'
 disk_storage_dict = {}
 disk_storage_list = """{{ parameter_dict.get("disk-storage-list") }}""".split('\n')
@@ -69,9 +69,13 @@ disk_cache = disk_cache if disk_cache in ["none", "writeback", "unsafe",
 disk_aio = '{{ parameter_dict.get("disk-aio") }}'.strip()
 disk_aio = disk_aio if disk_aio in ["threads", "native"] and \
               disk_cache == "directsync" else "threads"
+smp_count = {{ parameter_dict.get("smp-count") }}
+smp_max_count = {{ parameter_dict.get("smp-max-count") }}
 machine_options = '{{ parameter_dict.get("machine-options", "") }}'.strip()
 cpu_model = '{{ parameter_dict.get("cpu-model", "") }}'.strip()
 
+enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lower()
+
 logfile = '{{ parameter_dict.get("log-file") }}'
 
 if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false':
@@ -255,15 +259,14 @@ if use_tap == 'true':
             tap_interface, vhost),
           '-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, tap_mac_address)]
 
-smp = smp_count
-if smp_options:
-  for option in smp_options.split(','):
-    key, val = option.split('=')
-    if key in ('cores', 'threads', 'sockets', 'maxcpus') and val.isdigit():
-      smp += ',%s=%s' % (key, val)
+if enable_device_hotplug != 'true':
+  smp = '%s,maxcpus=%s' % (smp_count, smp_max_count)
+  ram = '%sM,slots=128,maxmem=%s' % (ram_size, ram_max_size)
+else:
+  smp = '1,maxcpus=%s' % smp_max_count
+  ram = '%sM,slots=128,maxmem=%s' % (init_ram_size, ram_max_size)
 kvm_argument_list = [qemu_path,
-  '-enable-kvm', '-smp', smp, '-name', vm_name,
-  '-m', ram_size, '-vga', 'std',
+  '-enable-kvm', '-smp', smp, '-name', vm_name, '-m', ram, '-vga', 'std',
   '-drive', 'file=%s,if=%s,cache=%s,aio=%s' % (disk_path, disk_type, disk_cache, disk_aio),
   '-vnc', '%s:1,ipv4,password' % listen_ip,
   '-boot', 'order=cd,menu=on',

software/nayuos/software.cfg

@@ -17,7 +17,7 @@ parts +=
 versions = versions
 
 [versions]
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3
 
 
 [template-instance]

software/neoppod/software-common.cfg

@@ -116,8 +116,8 @@ mysqlclient = 1.3.12
 persistent = 4.2.3
 pycrypto = 2.6.1
 pycurl = 7.43.0
-slapos.recipe.template = 4.2
-slapos.toolbox = 0.73
+slapos.recipe.template = 4.3
+slapos.toolbox = 0.74
 smmap2 = 2.0.1
 transaction = 1.7.0
 zodbpickle = 0.6.0

software/neotest/instance.cfg.in

-# TODO instance which runs this test periodically automatically and ingests results to ERP5
+# NEO test instance: run neotest under Nexedi testing infrastructure
+[buildout]
+parts = runTestSuite
+
+# std stuff for slapos instance
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
+offline = true
+
+# software release we instantiate was supplied here
+[software]
+dir     = {{ buildout['directory'] }}
+bin     = {{ buildout['bin-directory'] }}
+
+# instance directories
+[directory]
+recipe  = slapos.cookbook:mkdirectory
+home    = ${buildout:directory}
+bin     = ${:home}/bin
+neotest = ${:home}/neotest
+
+# script to run the testsuite from inside <instance>/neotest/
+# located @ <instance>/bin/runTestSuite so testnode can see this as run tests entrypoint.
+[runTestSuite]
+recipe  = slapos.cookbook:wrapper
+wrapper-path = ${directory:bin}/${:_buildout_section_name_}
+command-line = /bin/bash -c 'cd ${directory:neotest} && ${software:bin}/neotest-runTestSuite "$@"' runTestSuite
+# vvv appends "$@" to argv ^^^ without shell-escaping
+parameters-extra = true

software/neotest/neotest-env.sh.in

@@ -2,6 +2,8 @@
 
 . ${gowork:env.sh}
 
+PATH="${coreutils:location}/bin:$PATH"
+PATH="${util-linux:location}/bin:$PATH"
 PATH="${ethtool:location}/sbin:$PATH"
 PATH="${ioping:location}/bin:$PATH"
 PATH="${lmbench:location}/bin:$PATH"

software/neotest/neotest-runTestSuite.in

+#!/bin/bash -e
+# neotest's runTestSuite wraper so it could be run without any environment preset
+
+. ${buildout:directory}/neotest-env.sh
+exec ${gowork:src}/lab.nexedi.com/kirr/neo/go/neo/t/nxd/runTestSuite "$@"

software/neotest/software.cfg

@@ -12,6 +12,8 @@ extends =
     ../../component/ethtool/buildout.cfg
     ../../component/ioping/buildout.cfg
     ../../component/lmbench/buildout.cfg
+    ../../component/coreutils/buildout.cfg
+    ../../component/util-linux/buildout.cfg
 
 parts =
     gowork
@@ -19,6 +21,11 @@ parts =
     ioping
     ethtool
 
+#   neotest uses realpath & friends
+    coreutils
+#   ----//---- lsblk
+    util-linux
+
     neoppod-develop
     neoppod
     wendelin.core-dev
@@ -26,11 +33,12 @@ parts =
 
     neotest-env.sh
     neotest
+    neotest-runTestSuite
 
 #   for instance
     slapos-deps-eggs
     slapos-cookbook
-#   instance.cfg
+    instance.cfg
 
 
 # go packages to install (+ automatically their dependencies)
@@ -53,7 +61,7 @@ output  = ${buildout:directory}/${:_buildout_section_name_}
 
 [neotest-env.sh]
 <= buildout-template
-md5sum  = 6e2203c7a86e8a3e8e56b8086115f5d4
+md5sum  = 595eebbbcb56b6a8464d48833cfae57b
 
 [neotest]
 <= buildout-template
@@ -61,11 +69,25 @@ output  = ${buildout:bin-directory}/${:_buildout_section_name_}
 mode    = 0755
 md5sum  = fb3b4109128c1db1739ef5bb6abd1d94
 
+[neotest-runTestSuite]
+<= buildout-template
+output  = ${buildout:bin-directory}/${:_buildout_section_name_}
+mode    = 0755
+md5sum  = 6a4281730b68cdba5c873817a6754428
+
+
+# instance
+[jinja2-template]
+recipe  = slapos.recipe.template:jinja2
+template= ${:_profile_base_location_}/${:_buildout_section_name_}.in
+rendered= ${buildout:directory}/${:_buildout_section_name_}
+mode    = 0644
+context =
+  section buildout  buildout
 
-# instance (TODO)
 [instance.cfg]
-<= buildout-template
-md5sum  = 410e1b2d72829824b28cc0299adb472e
+<= jinja2-template
+md5sum  = 7bd68ed8842cd25301bf04bcdcef88f6
 
 
 # eggs:
@@ -80,6 +102,8 @@ eggs    =
     wendelin.core
 # for ZEO scripts (runzeo)
     ZEO
+# for nxd/runTestSuite
+    erp5.util
 
 
 # wendelin.core: latest not yet released
@@ -92,6 +116,7 @@ pyasn1 = 0.3.7
 ZODB3 = 3.11.0
 numpy = 1.13.3
 zope.testing = 4.6.2
+erp5.util = 0.4.50
 
 # Required by:
 # ZEO==4.3.1
@@ -103,7 +128,3 @@ ZConfig = 3.2.0
 # ZEO==4.3.1
 # ZODB==4.4.5
 zc.lockfile = 1.2.1
-
-# Required by:
-# slapos.toolbox==0.73
-erp5.util = 0.4.49

software/nginx-push-stream/software.cfg

@@ -45,5 +45,18 @@ output = ${buildout:directory}/instance-nginx.cfg.in
 mode = 0644
 
 [versions]
-slapos.recipe.template = 4.2
-slapos.toolbox = 0.73
+slapos.recipe.template = 4.3
+slapos.toolbox = 0.74
+dnspython = 1.15.0
+PyRSS2Gen = 1.1
+erp5.util = 0.4.50
+passlib = 1.7.1
+GitPython = 2.1.8
+lockfile = 0.12.2
+apache-libcloud = 2.2.1
+feedparser = 5.2.1
+atomize = 0.2.0
+inotifyx = 0.2.2
+gitdb2 = 2.0.3
+pyasn1 = 0.4.2
+smmap2 = 2.0.3

software/powerdns/software.cfg

@@ -65,4 +65,4 @@ mode = 0644
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3

software/re6stnet/software.cfg

@@ -110,8 +110,8 @@ gitdb = 0.6.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
 pycurl = 7.43.0
-slapos.recipe.template = 4.2
-slapos.toolbox = 0.73
+slapos.recipe.template = 4.3
+slapos.toolbox = 0.74
 smmap = 0.9.0
 
 # Required by:

software/seleniumrunner/software.cfg

@@ -47,4 +47,4 @@ mode = 0644
 
 [versions]
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3

software/slapos-in-partition/software.cfg

@@ -11,4 +11,4 @@ extends = common.cfg
 Pygments = 1.6
 collective.recipe.environment = 0.2.0
 collective.recipe.template = 1.10
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3

software/slapos-testing/software.cfg

@@ -97,4 +97,4 @@ mode = 640
 Pygments = 2.1.3
 collective.recipe.template = 1.10
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3

software/slaprunner/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = 713db528880282d568278f09458d2aab
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = 7df86928aeef0fbada832b0283b9a454
+md5sum = 8b1caca52ab1307343ebada59f5a6c66
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2
@@ -35,7 +35,7 @@ md5sum = 98ce179badc6af5979a64a7c3d0a2ceb
 
 [instance-runner-export]
 filename = instance-runner-export.cfg.in
-md5sum = 00ddcaa6e571505d1930612d12de8fde
+md5sum = d1122988652127af5e252f1503eeacab
 
 [template-resilient]
 filename = instance-resilient.cfg.jinja2

software/slaprunner/instance-runner-export.cfg.in

@@ -7,11 +7,13 @@ parts +=
   nginx-launcher
   certificate-authority
   ca-nginx
+  logrotate-entry-nginx
   gunicorn-launcher
   gunicorn-graceful
   publish-connection-information
   slaprunner-promise
   apache-httpd-promise
+  logrotate-entry-apache-httpd
   slaprunner-supervisord-wrapper
   runner-sshd-add-authorized-key
   runner-sshd-graceful
@@ -30,6 +32,7 @@ parts +=
   bash-profile
   supervisord-wrapper
   supervisord-promise
+  logrotate-entry-supervisord
   httpd-graceful-wrapper
 ## Monitoring part
 ## Monitor for runner

software/slaprunner/instance-runner.cfg

@@ -4,10 +4,12 @@ parts =
   nginx-launcher
   certificate-authority
   ca-nginx
+  logrotate-entry-nginx
   gunicorn-launcher
   gunicorn-graceful
   publish-connection-information
   slaprunner-promise
+  logrotate-entry-apache-httpd
   apache-httpd-promise
   slaprunner-supervisord-wrapper
   runner-sshd-add-authorized-key
@@ -27,6 +29,7 @@ parts =
   bash-profile
   supervisord-wrapper
   supervisord-promise
+  logrotate-entry-supervisord
   httpd-graceful-wrapper
 {% if slapparameter_dict.get('no-ipv4-frontend', 'false') == 'false' %}
   slaprunner-frontend-promise
@@ -347,6 +350,12 @@ mode = 700
 context =
     section param_nginx_frontend nginx-frontend
 
+[logrotate-entry-nginx]
+<= logrotate-entry-base
+name = nginx
+log = $${directory:log}/nginx.access.log $${directory:log}/nginx.error.log
+post = kill -SIGUSR1 $(cat $${buildout:directory}/var/run/nginx.pid)
+
 [httpd-parameters]
 path_pid = $${directory:run}/httpd.pid
 path_error_log = $${directory:log}/httpd-error.log
@@ -387,6 +396,12 @@ wait-for-files =
   $${ca-nginx:cert-file}
   $${ca-nginx:key-file}
 
+[logrotate-entry-apache-httpd]
+<= logrotate-entry-base
+name = apache
+log = $${directory:log}/httpd-access.log $${directory:log}/httpd-error.log
+post = test ! -s $${buildout:directory}/var/run/httpd.pid || $${buildout:directory}/bin/slapos-kill --pidfile $${buildout:directory}/var/run/httpd.pid -s USR1
+
 [httpd-graceful-wrapper]
 recipe = collective.recipe.template
 input = inline:
@@ -777,6 +792,12 @@ recipe = slapos.cookbook:wrapper
 command-line = $${buildout:directory}/bin/supervisord -c $${supervisord-conf:rendered} --nodaemon
 wrapper-path = $${directory:services}/supervisord
 
+[logrotate-entry-supervisord]
+<= logrotate-entry-base
+name = supervisord
+log = $${directory:log}/slapproxy.log $${directory:log}/supervisord.log $${directory:log}/supervisord-errors.log
+post = kill -SIGUSR2 $(cat $${buildout:directory}/var/run/supervisord.pid)
+
 [supervisord-promise]
 recipe = slapos.cookbook:check_port_listening
 path = $${directory:promises}/supervisord

software/slaprunner/software.cfg

@@ -15,8 +15,8 @@ gitdb = 0.6.4
 gunicorn = 19.7.1
 prettytable = 0.7.2
 pycurl = 7.43.0
-slapos.recipe.template = 4.2
-slapos.toolbox = 0.73
+slapos.recipe.template = 4.3
+slapos.toolbox = 0.74
 smmap = 0.9.0
 
 # Required by:

software/test-slave-instance-deployment/software.cfg

@@ -59,7 +59,7 @@ eggs = collective.recipe.template
 collective.recipe.template = 1.11
 plone.recipe.command = 1.1
 slapos.recipe.build = 0.28
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3
 
 # Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.
 [buildout]

software/varnish/software.cfg

@@ -62,4 +62,4 @@ md5sum = 0ea12a4ad2d2e3d406476e35b8d3e3fb
 mode = 640
 
 [versions]
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3

stack/caucase/buildout.cfg

@@ -94,8 +94,8 @@ caucase = 0.1.4
 futures = 3.1.1
 gitdb2 = 2.0.2
 gunicorn = 19.7.1
-slapos.recipe.template = 4.2
-slapos.toolbox = 0.73
+slapos.recipe.template = 4.3
+slapos.toolbox = 0.74
 smmap2 = 2.0.3
 
 # Required by:

stack/cloudooo.cfg

@@ -89,7 +89,7 @@ PasteScript = 2.0.2
 WSGIUtils = 0.7
 python-magic = 0.4.6
 rdiff-backup = 1.0.5+SlapOSPatched001
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3
 
 # Required by:
 # PasteScript==2.0

stack/erp5/buildout.cfg

@@ -63,6 +63,7 @@ extends =
   ../../component/findutils/buildout.cfg
   ../../component/userhosts/buildout.cfg
   ../../component/postfix/buildout.cfg
+  ../../component/zbarlight/buildout.cfg
   ../monitor/buildout.cfg
   ../../software/caucase/software.cfg
   ../../software/jupyter/software.cfg
@@ -465,6 +466,7 @@ eggs = ${neoppod:eggs}
   ${h5py:egg}
   openpyxl
   ${statsmodels:egg}
+  ${zbarlight:egg}
   lock_file
   astor
   APacheDEX
@@ -730,7 +732,7 @@ spyne = 2.12.14
 suds = 0.4
 facebook-sdk = 2.0.0
 threadframe = 0.2
-timerserver = 2.0.2
+timerserver = 2.0.4
 urlnorm = 1.1.4
 uuid = 1.30
 validictory = 1.1.0

stack/monitor/buildout.cfg

@@ -61,7 +61,7 @@ eggs =
 # Monitor templates files
 [monitor-httpd-conf]
 <= monitor-template-base
-md5sum = f2d6951670733de3b37c0ebe9eee343b
+md5sum = b5f42503799e7e770afce4097d3b75ae
 filename = monitor-httpd.conf.in
 
 [monitor-template-wrapper]
@@ -72,7 +72,7 @@ md5sum = 1695c9a06a2b11ccfe893d7a224e489d
 [monitor-conf]
 <= monitor-template-base
 filename = monitor.conf.in
-md5sum = fc65084dd1c2fe2487b58a003b576f61
+md5sum = 888e2845d09bfaa59c25f56f5bcf76b1
 
 [monitor-instance-info]
 <= monitor-template-base
@@ -131,6 +131,6 @@ depends =
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 pycurl = 7.43.0
-slapos.toolbox = 0.73
+slapos.toolbox = 0.74
 pyasn1 = 0.3.7
 

stack/monitor/buildout.hash.cfg

@@ -15,4 +15,4 @@
 # not need these here).
 [monitor2-template]
 filename = instance-monitor.cfg.jinja2.in
-md5sum = 5027f0b1abcc28ce3817cd70fb667a3b
+md5sum = 03254b14a2ff242f7588a307d8c27f23

stack/monitor/instance-monitor.cfg.jinja2.in

@@ -42,6 +42,7 @@ services = ${directory:services}
 services-conf = ${directory:etc}/monitor.conf.d
 log = ${directory:log}/monitor
 monitor-var = ${directory:var}/monitor
+monitor-log = ${directory:monitor}/private/monitor-log
 
 [ca-directory]
 recipe = slapos.cookbook:mkdirectory
@@ -88,14 +89,11 @@ promise-folder = ${directory:promises}
 monitor-promise-folder = ${monitor-directory:promises}
 promises-timeout-file = ${monitor-promise-timeout-file:file}
 pid-file = ${monitor-directory:pids}/monitor-bootstrap.pid
+randomsleep = {{ bin_directory }}/randomsleep
 
 public-path-list = 
-private-path-list = 
-    ${directory:log}
-# 
-monitor-url-list = 
-  ${monitor-instance-parameter:monitor-url-list}
-
+private-path-list = ${directory:log}
+monitor-url-list = ${monitor-instance-parameter:monitor-url-list}
 parameter-file-path = ${monitor-instance-parameter:configuration-file-path}
 
 parameter-list = 
@@ -234,6 +232,12 @@ context =
     raw dash_binary {{ dash_executable_location }}
 command = kill -USR1 $(cat ${monitor-httpd-conf-parameter:pid-file})
 
+[logrotate-entry-monitor-httpd]
+<= logrotate-entry-base
+name = monitor-apache
+log = $${basedirectory:log}/monitor/monitor-httpd-error.log
+post = test ! -s $${buildout:directory}/var/run/monitor-httpd.pid || $${buildout:directory}/bin/slapos-kill --pidfile $${buildout:directory}/var/run/monitor-httpd.pid -s USR1
+
 [xnice-bin]
 recipe = collective.recipe.template
 input = inline:#!/bin/sh
@@ -264,14 +268,14 @@ recipe = slapos.cookbook:cron.d
 cron-entries = ${cron:cron-entries}
 name = monitor-globalstate
 frequency = * * * * *
-command = ${monitor-globalstate-wrapper:wrapper-path}
+command = {{ bin_directory }}/randomsleep 60 && ${monitor-globalstate-wrapper:wrapper-path}
 
 [monitor-configurator-cron-entry]
 recipe = slapos.cookbook:cron.d
 cron-entries = ${cron:cron-entries}
 name = monitor-configurator
 frequency = * * * * *
-command = ${monitor-configurator-wrapper:wrapper-path}
+command = {{ bin_directory }}/randomsleep 60 && ${monitor-configurator-wrapper:wrapper-path}
 
 [monitor-httpd-promise]
 recipe = slapos.cookbook:check_url_available
@@ -407,6 +411,7 @@ depends =
   ${monitor-bootstrap-promise:file}
   ${promise-check-slapgrid:output}
   ${promise-monitor-httpd-is-process-older-than-dependency-set:wrapper-path}
+  ${logrotate-entry-monitor-httpd:name}
 
 [monitor-publish]
 monitor-base-url = ${monitor-publish-parameters:monitor-base-url}

stack/monitor/templates/monitor-httpd.conf.in

@@ -2,6 +2,7 @@ PidFile "{{ parameter_dict.get('pid-file') }}"
 
 StartServers 1
 ServerLimit 1
+MaxRequestWorkers 4
 ThreadLimit 4
 ThreadsPerChild 4
 

stack/monitor/templates/monitor.conf.in

 [monitor]
+{% set monitor_url_list = parameter_dict.pop("monitor-url-list", "").strip() -%}
 {% for key, value in parameter_dict.items() -%}
-{% if key == "monitor-url-list" and monitor_base_urls is defined -%}
-{{ key }} =
+{{ key }} = {{ value.strip().replace("\n", "\n  ") }}
+{% endfor -%}
+
+monitor-url-list =
+{% if monitor_url_list -%}
+{{ '  ' ~ monitor_url_list.replace("\n", "\n  ") }}
+{% else -%}
 {% for key, value in monitor_base_urls.items() -%}
 {{ '  ' ~ value }}
 {% endfor -%}
-{% else -%}
-{{ key }} = {{ value.strip().replace("\n", "\n  ") }}
 {% endif -%}
-{% endfor -%}

stack/slapos.cfg

@@ -93,7 +93,7 @@ eggs =
 [versions]
 setuptools = 33.1.1
 # Use SlapOS patched zc.buildout
-zc.buildout = 2.5.2+slapos011
+zc.buildout = 2.5.2+slapos012
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
 zc.recipe.egg = 2.0.3+slapos003
 # Use own version of h.r.download to be able to open .xz and .lz archives
@@ -121,14 +121,14 @@ netaddr = 0.7.19
 pbr = 2.0.0
 plone.recipe.command = 1.1
 prettytable = 0.7.2
-psutil = 5.2.0
+psutil = 5.4.3
 pyOpenSSL = 17.2.0
 pyparsing = 2.2.0
 pytz = 2016.10
 requests = 2.13.0
 six = 1.10.0
 slapos.cookbook = 1.0.53
-slapos.core = 1.4.3
+slapos.core = 1.4.4
 slapos.extension.strip = 0.4
 slapos.libnetworkcache = 0.15
 slapos.rebootstrap = 4.1
@@ -140,7 +140,7 @@ xml-marshaller = 0.9.7
 paramiko = 2.1.3
 
 # Required by:
-# slapos.core==1.4.3
+# slapos.core==1.4.4
 Flask = 0.12
 
 # Required by:
@@ -160,7 +160,7 @@ ipaddress = 1.0.18
 jsonschema = 2.6.0
 
 # Required by:
-# slapos.core==1.4.3
+# slapos.core==1.4.4
 # XXX 'slapos node format' raises an exception with netifaces 0.10.5.
 netifaces = 0.10.4
 
@@ -173,15 +173,15 @@ packaging = 16.8
 pycparser = 2.17
 
 # Required by:
-# slapos.core==1.4.3
+# slapos.core==1.4.4
 supervisor = 3.3.3
 
 # Required by:
-# slapos.core==1.4.3
+# slapos.core==1.4.4
 uritemplate = 3.0.0
 
 # Required by:
-# slapos.core==1.4.3
+# slapos.core==1.4.4
 zope.interface = 4.3.3
 
 [networkcache]