Update Release Candidate

component/PyWavelets/buildout.cfg

+[buildout]
+extends =
+  ../numpy/openblas.cfg
+  ../cython/buildout.cfg
+  ../scipy/buildout.cfg
+  
+parts =
+  PyWavelets
+
+[PyWavelets-env]
+<= numpy-env
+
+[PyWavelets]
+recipe = zc.recipe.egg:custom
+egg = PyWavelets
+environment = PyWavelets-env
+setup-eggs =
+  ${numpy:egg}
+  ${scipy:egg}
+  ${cython:egg}
+rpath =
+  ${gcc:location}/lib
+  ${gcc:location}/lib64
+  ${openblas:location}/lib
+
+[PyWavelets-repository]
+recipe  = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+repository = https://github.com/PyWavelets/pywt.git
+location = ${buildout:parts-directory}/PyWavelets
+
+[PyWavelets-develop]
+# This only work if we use zc.recipe.egg from zc.buildout 2 patched
+# by nexedi (zc.recipe.egg = 2.0.3+slapos001 and zc.buildout = 2.5.2+slapos003)
+recipe = zc.recipe.egg:develop
+egg = PyWavelets
+setup = ${PyWavelets-repository:location}
+environment = PyWavelets-env
+setup-eggs =
+  ${cython:egg}
+  ${numpy:egg}
+  ${scipy:egg}
+rpath =
+  ${gcc-fortran:location}/lib
+  ${gcc-fortran:location}/lib64
+  ${openblas:location}/lib

component/leptonica/buildout.cfg

@@ -11,14 +11,12 @@ extends =
 
 [leptonica]
 recipe = slapos.recipe.cmmi
-url = http://leptonica.googlecode.com/files/leptonica-1.68.tar.gz
-md5sum = 5cd7092f9ff2ca7e3f3e73bfcd556403
+url = http://www.leptonica.com/source/leptonica-1.76.0.tar.gz
+md5sum = a263a5e4f7e8f8a661fb121a265d2d20
 configure-options =
   --disable-static
-patch-options = -p1
-patches =
-  ${:_profile_base_location_}/leptonica-1.69-zlib-include.patch#cff3dc942075190939b407c38e0d3201
+
 environment =
   CPPFLAGS=-I${zlib:location}/include -I${libjpeg:location}/include -I${libpng:location}/include -I${libtiff:location}/include -I${webp:location}/include -I${giflib:location}/include
   LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${libjpeg:location}/lib -Wl,-rpath=${libjpeg:location}/lib -L${libpng:location}/lib -Wl,-rpath=${libpng:location}/lib -L${libtiff:location}/lib -Wl,-rpath=${libtiff:location}/lib -L${webp:location}/lib -Wl,-rpath=${webp:location}/lib -L${giflib:location}/lib -Wl,-rpath=${giflib:location}/lib
-  PATH=${patch:location}/bin:%(PATH)s
+  PATH=${patch:location}/bin:%(PATH)s
\ No newline at end of file

component/scikit-image/buildout.cfg

+[buildout]
+extends =
+  ../numpy/openblas.cfg
+  ../cython/buildout.cfg
+  ../scipy/buildout.cfg
+  ../PyWavelets/buildout.cfg
+  ../pillow/buildout.cfg
+
+parts =
+  scikit-image
+
+[scikit-image-env]
+<= numpy-env
+
+[scikit-image]
+recipe = zc.recipe.egg:custom
+egg = scikit-image
+environment = scikit-image-env
+setup-eggs =
+  ${numpy:egg}
+  ${scipy:egg}
+  ${cython:egg}
+  ${PyWavelets:egg}
+  ${pillow-python:egg}
+  networkx
+rpath =
+  ${gcc:location}/lib
+  ${gcc:location}/lib64
+  ${openblas:location}/lib
+
+[scikit-image-repository]
+recipe  = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+repository = https://github.com/scikit-image/scikit-image.git
+location = ${buildout:parts-directory}/scikit-image
+
+[scikit-image-develop]
+# This only work if we use zc.recipe.egg from zc.buildout 2 patched
+# by nexedi (zc.recipe.egg = 2.0.3+slapos001 and zc.buildout = 2.5.2+slapos003)
+recipe = zc.recipe.egg:develop
+egg = scikit-image
+setup = ${scikit-image-repository:location}
+environment = scikit-image-env
+setup-eggs =
+  ${cython:egg}
+  ${numpy:egg}
+  ${scipy:egg}
+  ${PyWavelets:egg}
+  ${pillow-python:egg}
+  networkx
+rpath =
+  ${gcc-fortran:location}/lib
+  ${gcc-fortran:location}/lib64
+  ${openblas:location}/lib

component/tesseract/buildout.cfg

 [buildout]
+
 extends =
+  ../pkgconfig/buildout.cfg
   ../autoconf/buildout.cfg
   ../automake/buildout.cfg
   ../jbigkit/buildout.cfg
@@ -8,45 +10,56 @@ extends =
   ../libtool/buildout.cfg
   ../patch/buildout.cfg
   ../webp/buildout.cfg
-
+  
 parts =
   tesseract
+  tesseract-traineddata
+  tesseract-eng-traineddata
+  tesseract-osd-traineddata
 
 [tesseract]
 recipe = slapos.recipe.cmmi
-url = http://tesseract-ocr.googlecode.com/files/tesseract-3.01.tar.gz
-md5sum = 1ba496e51a42358fb9d3ffe781b2d20a
-patch-options =
-  -p1
-patches =
-  ${:_profile_base_location_}/tesseract-3.00-gcc-4.7-build.patch#ca80db3ec489c547b03f3ee48879c1b1
-  ${:_profile_base_location_}/tesseract-3.01-remove-bom.patch#2e691858cb492b7c17d23bf0912b3d24
-  ${:_profile_base_location_}/tesseract-3.01-gcc6-ftbs.patch#f7a6140c0fe390b96fe753a70e9d59fd
+url = https://github.com/tesseract-ocr/tesseract/archive/6b250b58121a9858d3e3019a78a6f7d421bd0fc7.tar.gz
+md5sum = fdc38148ad8eb1bd0485a217503dd6d5
+pkg_config_depends = ${leptonica:location}/lib/pkgconfig:${fontconfig:location}/lib/pkgconfig:${fontconfig:pkg_config_depends}:${lcms2:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig
 pre-configure =
-  libtoolize -f -c
-  aclocal -I ${libtool:location}/share/aclocal -I config
-  autoheader -f
-  automake -c -a -f
+  aclocal -I${pkgconfig:location}/share/aclocal -I${libtool:location}/share/aclocal
+  ./autogen.sh
+  autoreconf -ivf
   autoconf -Wno-portability
+
 configure-options =
   --disable-static
-  --datarootdir=${tesseract-eng-traineddata:location}
+  --datarootdir=${tesseract-traineddata:location}
 # XXX: tesseract seems not easily configurable at runtime about where to find
 # its trained data, so we set its datarootdir above to a controlled location
 
 # tesseract has a non-standard way of testing for leptonica, hence the
 # LIBLEPT_HEADERSDIR entry below:
 environment =
-  PATH=${autoconf:location}/bin:${automake:location}/bin:${libtool:location}/bin:${m4:location}/bin:${patch:location}/bin:%(PATH)s
+  PATH=${pkgconfig:location}/bin:${autoconf:location}/bin:${automake:location}/bin:${libtool:location}/bin:${m4:location}/bin:${patch:location}/bin:%(PATH)s
+  PKG_CONFIG_PATH=${:pkg_config_depends}
   ACLOCAL_ARGS=-I${libtool:location}/share/aclocal
+  ACLOCAL_PATH=${pkgconfig:location}/share/aclocal:${libtool:location}/share/aclocal
   LIBLEPT_HEADERSDIR=${leptonica:location}/include
   CPPFLAGS=-I${leptonica:location}/include
   LDFLAGS =-L${leptonica:location}/lib -Wl,-rpath=${leptonica:location}/lib -L${jbigkit:location}/lib -Wl,-rpath=${jbigkit:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+
 make-options =
   LIBTOOL=${libtool:location}/bin/libtool
 
+[tesseract-traineddata]
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+
 [tesseract-eng-traineddata]
-recipe = slapos.recipe.build:download-unpacked
-strip-top-level-dir = true
-url = http://tesseract-ocr.googlecode.com/files/tesseract-ocr-3.01.eng.tar.gz
-md5sum = 89c139a73e0e7b1225809fc7b226b6c9
+recipe = slapos.recipe.build:download
+destination = ${tesseract-traineddata:location}/tessdata/eng.traineddata
+url = https://github.com/tesseract-ocr/tessdata/raw/590567f20dc044f6948a8e2c61afc714c360ad0e/eng.traineddata
+md5sum = 57e0df3d84fed9fbf8c7a8e589f8f012
+
+[tesseract-osd-traineddata]
+recipe = slapos.recipe.build:download
+destination = ${tesseract-traineddata:location}/tessdata/osd.traineddata
+url = https://github.com/tesseract-ocr/tessdata/raw/590567f20dc044f6948a8e2c61afc714c360ad0e/osd.traineddata
+md5sum = 7611737524efd1ce2dde67eff629bbcf
+

component/tesseract/tesseract-3.00-gcc-4.7-build.patch

---- tesseract-3.00/viewer/svutil.cpp.old	2012-04-11 09:34:26.168608940 +0200
-+++ tesseract-3.00/viewer/svutil.cpp	2012-04-11 09:34:46.108565692 +0200
-@@ -21,6 +21,7 @@
- // thread/process creation & synchronization and network connection.
- 
- #include <stdio.h>
-+#include <unistd.h>
- #ifdef WIN32
- #include <windows.h>
- #include <winsock.h>

component/tesseract/tesseract-3.01-gcc6-ftbs.patch

-commit 58e79a222e12280984ed19ab4d3bcac654e121fa
-Author: Arnaud Fontaine <arnaud.fontaine@nexedi.com>
-Date:   Fri Nov 4 19:10:13 2016 +0900
-
-    Fix compilation errors.
-
-diff --git a/cube/char_set.cpp b/cube/char_set.cpp
-index 3cf4798..6b29883 100644
---- a/cube/char_set.cpp
-+++ b/cube/char_set.cpp
-@@ -65,13 +65,13 @@ CharSet *CharSet::Create(TessdataManager *tessdata_manager,
-       !tessdata_manager->SeekToStart(TESSDATA_UNICHARSET)) {
-     fprintf(stderr, "Cube ERROR (CharSet::Create): could not find "
-             "either cube or tesseract unicharset\n");
--    return false;
-+    return NULL;
-   }
-   FILE *charset_fp = tessdata_manager->GetDataFilePtr();
-   if (!charset_fp) {
-     fprintf(stderr, "Cube ERROR (CharSet::Create): could not load "
-             "a unicharset\n");
--    return false;
-+    return NULL;
-   }
- 
-   // If we found a cube unicharset separate from tesseract's, load it and
-@@ -84,7 +84,7 @@ CharSet *CharSet::Create(TessdataManager *tessdata_manager,
-     loaded = char_set->LoadSupportedCharList(charset_fp, NULL);
-   if (!loaded) {
-     delete char_set;
--    return false;
-+    return NULL;
-   }
- 
-   char_set->init_ = true;
-diff --git a/cube/cube_line_segmenter.cpp b/cube/cube_line_segmenter.cpp
-index deee573..3f0b762 100644
---- a/cube/cube_line_segmenter.cpp
-+++ b/cube/cube_line_segmenter.cpp
-@@ -124,7 +124,7 @@ Pixa *CubeLineSegmenter::CrackLine(Pix *cracked_line_pix,
- 
-   if (line_con_comps == NULL) {
-     delete []lines_pixa;
--    return false;
-+    return NULL;
-   }
- 
-   // assign each conn comp to the a line based on its centroid
-@@ -142,7 +142,7 @@ Pixa *CubeLineSegmenter::CrackLine(Pix *cracked_line_pix,
-         delete []lines_pixa;
-         boxaDestroy(&line_con_comps);
-         pixaDestroy(&line_con_comps_pix);
--        return false;
-+        return NULL;
-       }
-     }
- 
-@@ -413,14 +413,14 @@ Pix *CubeLineSegmenter::Pixa2Pix(Pixa *pixa, Box **dest_box,
- 
-   (*dest_box) = boxCreate(min_x, min_y, max_x - min_x, max_y - min_y);
-   if ((*dest_box) == NULL) {
--    return false;
-+    return NULL;
-   }
- 
-   // create the union pix
-   Pix *union_pix = pixCreate((*dest_box)->w, (*dest_box)->h, img_->d);
-   if (union_pix == NULL) {
-     boxDestroy(dest_box);
--    return false;
-+    return NULL;
-   }
- 
-   // create a pix corresponding to the union of all pixs
-diff --git a/cube/cube_object.cpp b/cube/cube_object.cpp
-index 48bce64..b9a7113 100644
---- a/cube/cube_object.cpp
-+++ b/cube/cube_object.cpp
-@@ -165,7 +165,7 @@ WordAltList *CubeObject::Recognize(LangModel *lang_mod, bool word_mode) {
-       if (deslanted_beam_obj_ == NULL) {
-         fprintf(stderr, "Cube ERROR (CubeObject::Recognize): could not "
-                 "construct deslanted BeamSearch\n");
--        return false;
-+        return NULL;
-       }
-     }
- 
-diff --git a/cube/word_list_lang_model.cpp b/cube/word_list_lang_model.cpp
-index 18f85c1..0f7f562 100644
---- a/cube/word_list_lang_model.cpp
-+++ b/cube/word_list_lang_model.cpp
-@@ -74,7 +74,7 @@ LangModEdge **WordListLangModel::GetEdges(CharAltList *alt_list,
-   // initialize if necessary
-   if (init_ == false) {
-     if (Init() == false) {
--      return false;
-+      return NULL;
-     }
-   }
- 

component/tesseract/tesseract-3.01-remove-bom.patch

-The patch below removes a utf-8 BOM mark.
-
-Avoid touching it as the BOM is invisible, and copy/pasting might not work.
-
-It is needed because old compilers treat the BOM as garbage instead of
-whitespace.
-
---- tesseract-3.01/ccutil/strngs.h.orig	2012-05-24 15:13:22.743808379 +0200
-+++ tesseract-3.01/ccutil/strngs.h	2012-05-24 15:16:54.468858282 +0200
-@@ -1,4 +1,4 @@
--/**********************************************************************
-+/**********************************************************************
-  * File:        strngs.h  (Formerly strings.h)
-  * Description: STRING class definition.
-  * Author:					Ray Smith

slapos/recipe/random.py

@@ -134,14 +134,14 @@ class Password(object):
     if self.storage_path:
       try:
         with open(self.storage_path) as f:
-          passwd = f.read()
+          passwd = f.read().strip('\n')
       except IOError as e:
         if e.errno != errno.ENOENT:
           raise
     if not passwd:
       passwd = self.generatePassword(int(options.get('bytes', '8')))
       self.update = self.install
-    self.passwd = passwd.strip('\n')
+    self.passwd = passwd
     # Password must not go into .installed file, for 2 reasons:
     # security of course but also to prevent buildout to always reinstall.
     def get(option, *args, **kw):

software/agent/software.cfg

@@ -51,7 +51,7 @@ pycrypto = 2.6.1
 pycurl = 7.43.0
 slapos.recipe.download = 1.0
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.76
+slapos.toolbox = 0.77
 smmap = 0.9.0
 
 # Required by:

software/apache-frontend/software.cfg

@@ -11,7 +11,7 @@ plone.recipe.command = 1.1
 pycrypto = 2.6.1
 rdiff-backup = 1.0.5+SlapOSPatched001
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.76
+slapos.toolbox = 0.77
 smmap = 0.9.0
 numpy = 1.11.2
 pyasn1 = 0.2.3

software/build-rina/software.cfg

@@ -82,3 +82,7 @@ packages +=
   dh-autoreconf pkg-config doxygen maven xmlto
 # hellorina (shouldn't parts like lxml-python depend on the python of the SR?)
   python-dev
+
+[versions]
+erp5.util = 0.4.51
+slapos.recipe.template = 4.3

software/caddy-frontend/TODO.rst

 Generally things to be done with ``caddy-frontend``:
 
+ * tests: add assertion with results of promises in etc/promise for each partition
+ * generated files: ``| trim`` values (like ``slave_password[slave]`` in ``templates/template-log-access.conf.in``) in generated configuration files to have them renfered correctly
+ * check the whole frontend slave snippet with ``caddy -validate`` during buildout run, and reject if does not pass validation
  * ``apache-ca-certificate`` shall be merged with ``apache-certificate``
 
    * ``apache-ca-certificate`` shall be appended to ``apache-certificate`` if not already there

software/caddy-frontend/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = 906e5bd66b1265b8109a86b6ab46e91f
+md5sum = b73505ae80d6325a244f5094f8edc0ae
 
 [template-apache-frontend]
 filename = instance-apache-frontend.cfg
@@ -22,7 +22,7 @@ md5sum = b170d0987563b481eb71cf705c3658ab
 
 [template-apache-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = 7f15b5745eda8e1f02d4bf7d886dcdad
+md5sum = 27e98547061bd81e5f84cb7dd21b683b
 
 [template-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
@@ -44,6 +44,10 @@ md5sum = 6689d96fc18d9aad78d77fe87770d4da
 filename = templates/apache-custom-slave-list.cfg.in
 md5sum = fb6c93f42f232e381174a5951c3fc222
 
+[caddy-backend-url-validator]
+filename = templates/caddy-backend-url-validator.in
+md5sum = 0979a03476e86bf038516c9565dadc17
+
 [template-not-found-html]
 filename = templates/notfound.html
 md5sum = f20d6c3d2d94fb685f8d26dfca1e822b

software/caddy-frontend/common.cfg

@@ -46,6 +46,13 @@ url = ${:_profile_base_location_}/instance-apache-frontend.cfg
 output = ${buildout:directory}/template-caddy-frontend.cfg
 mode = 0644
 
+[caddy-backend-url-validator]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/templates/${:filename}
+filename = caddy-backend-url-validator.in
+output = ${buildout:directory}/caddy-backend-url-validator
+mode = 0750
+
 [template-caddy-replicate]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -72,7 +72,22 @@ context =
 {% for slave in slave_instance_list %}
 {# BBB: apache_custom_https AND apache_custom_http #}
 {%   if not ((slave.has_key('caddy_custom_http') or slave.has_key('apache_custom_http') or slave.has_key('caddy_custom_https') or slave.has_key('apache_custom_https')) and not slave.get('slave_reference') in authorized_slave_string) %}
-{%     do authorized_slave_list.append(slave) %}
+{%     set slave_ok = True %}
+{%     if slave.get('url') %}
+{%       if subprocess_module.call([caddy_backend_url_validator, slave['url']]) == 1 %}
+{%         set slave_ok = False %}
+{%       endif %}
+{%     endif %}
+{%     if slave.get('https-url') %}
+{%       if subprocess_module.call([caddy_backend_url_validator, slave['https-url']]) == 1 %}
+{%         set slave_ok = False %}
+{%       endif %}
+{%     endif %}
+{%     if slave_ok %}
+{%       do authorized_slave_list.append(slave) %}
+{%     else %}
+{%       do rejected_slave_list.append(slave.get('slave_reference')) %}
+{%     endif %}
 {%   else %}
 {%     do rejected_slave_list.append(slave.get('slave_reference')) %}
 {%   endif %}

software/caddy-frontend/instance.cfg

@@ -43,6 +43,8 @@ template = ${template-caddy-replicate:target}
 filename = instance-caddy-replicate.cfg
 extensions = jinja2.ext.do
 extra-context =
+    import subprocess_module subprocess
+    raw caddy_backend_url_validator ${caddy-backend-url-validator:output}
     raw template_publish_slave_information ${template-replicate-publish-slave-information:target}
 # Must match the key id in [switch-softwaretype] which uses this section.
     raw software_type RootSoftwareInstance-default-custom-personal-replicate

software/caddy-frontend/software.cfg

@@ -11,7 +11,7 @@ plone.recipe.command = 1.1
 pycrypto = 2.6.1
 rdiff-backup = 1.0.5+SlapOSPatched001
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.76
+slapos.toolbox = 0.77
 smmap = 0.9.0
 numpy = 1.11.2
 pyasn1 = 0.2.3

software/caddy-frontend/templates/caddy-backend-url-validator.in

+#!${dash:location}/bin/dash
+config="https://example.com {\n  proxy / $1 {\n  }\n}"
+
+echo -e $config | ${caddy:output} -conf stdin -validate > /dev/null 2>&1

software/caddy-frontend/test/test.py

@@ -2692,3 +2692,91 @@ class TestRe6stVerificationUrlSlave(SlaveHttpFrontendTestCase,
       'URL="some-re6st-verification-url"' in
       open(re6st_connectivity_promise_list[0]).read()
     )
+
+
+class TestMalformedBackenUrlSlave(SlaveHttpFrontendTestCase,
+                                  TestDataMixin):
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'domain': 'example.com',
+      'nginx-domain': 'nginx.example.com',
+      'public-ipv4': utils.LOCAL_IPV4,
+      'apache-certificate': open('wildcard.example.com.crt').read(),
+      'apache-key': open('wildcard.example.com.key').read(),
+      'port': HTTPS_PORT,
+      'plain_http_port': HTTP_PORT,
+      'nginx_port': NGINX_HTTPS_PORT,
+      'plain_nginx_port': NGINX_HTTP_PORT,
+      'monitor-httpd-port': MONITOR_HTTPD_PORT,
+      '-frontend-config-1-monitor-httpd-port': MONITOR_F1_HTTPD_PORT,
+    }
+
+  @classmethod
+  def getSlaveParameterDictDict(cls):
+    return {
+      'empty': {
+      },
+      'url': {
+        'url': "https://[fd46::c2ae]:!py!u'123123'",
+      },
+      'https-url': {
+        'https-url': "https://[fd46::c2ae]:!py!u'123123'",
+      }
+    }
+
+  def test_master_partition_state(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    self.assertKeyWithPop('monitor-setup-url', parameter_dict)
+
+    expected_parameter_dict = {
+      'monitor-base-url': None,
+      'domain': 'example.com',
+      'accepted-slave-amount': '1',
+      'rejected-slave-amount': '2',
+      'slave-amount': '3',
+      'rejected-slave-list': '["_url", "_https-url"]'}
+
+    self.assertEqual(
+      expected_parameter_dict,
+      parameter_dict
+    )
+
+  def test_empty(self):
+    parameter_dict = self.slave_connection_parameter_dict_dict[
+      'empty']
+    self.assertLogAccessUrlWithPop(parameter_dict, 'empty')
+    self.assertEqual(
+      parameter_dict,
+      {
+        'domain': 'empty.example.com',
+        'replication_number': '1',
+        'url': 'http://empty.example.com',
+        'site_url': 'http://empty.example.com',
+        'secure_access': 'https://empty.example.com',
+        'public-ipv4': utils.LOCAL_IPV4,
+      }
+    )
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
+
+    self.assertEqual(
+      utils.der2pem(result.peercert),
+      open('wildcard.example.com.crt').read())
+
+    self.assertEqual(result.status_code, no_backend_response_code)
+
+  def test_url(self):
+    parameter_dict = self.slave_connection_parameter_dict_dict[
+      'url'].copy()
+    self.assertEqual(
+      parameter_dict, {}
+    )
+
+  def test_https_url(self):
+    parameter_dict = self.slave_connection_parameter_dict_dict[
+      'https-url'].copy()
+    self.assertEqual(
+      parameter_dict, {}
+    )

software/caddy-frontend/test/test_data/test.TestMalformedBackenUrlSlave.test_file_list_log-CADDY.txt

+TestMalformedBackenUrlSlave-0/var/log/monitor-httpd-error.log
+TestMalformedBackenUrlSlave-1/var/log/frontend-access.log
+TestMalformedBackenUrlSlave-1/var/log/frontend-error.log
+TestMalformedBackenUrlSlave-1/var/log/httpd/_empty_access_log
+TestMalformedBackenUrlSlave-1/var/log/httpd/_empty_error_log
+TestMalformedBackenUrlSlave-1/var/log/monitor-httpd-error.log
+TestMalformedBackenUrlSlave-1/var/log/nginx-access.log
+TestMalformedBackenUrlSlave-1/var/log/nginx-error.log
+TestMalformedBackenUrlSlave-1/var/log/trafficserver/manager.log
+TestMalformedBackenUrlSlave-1/var/log/trafficserver/traffic.out
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestMalformedBackenUrlSlave.test_file_list_run-CADDY.txt

+TestMalformedBackenUrlSlave-0/var/run/monitor-httpd.pid
+TestMalformedBackenUrlSlave-0/var/run/monitor/monitor-bootstrap.pid
+TestMalformedBackenUrlSlave-1/var/run/caddy_configuration.signature
+TestMalformedBackenUrlSlave-1/var/run/httpd.pid
+TestMalformedBackenUrlSlave-1/var/run/monitor-httpd.pid
+TestMalformedBackenUrlSlave-1/var/run/monitor/monitor-bootstrap.pid
+TestMalformedBackenUrlSlave-1/var/run/ncaddy_configuration.signature
+TestMalformedBackenUrlSlave-1/var/run/nginx.pid

software/caddy-frontend/test/test_data/test.TestMalformedBackenUrlSlave.test_monitor_promise_list-CADDY.txt

+TestMalformedBackenUrlSlave-1/etc/monitor-promise/check-_empty-error-log-last-day
+TestMalformedBackenUrlSlave-1/etc/monitor-promise/check-_empty-error-log-last-hour
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestMalformedBackenUrlSlave.test_promise_list-CADDY.txt

+TestMalformedBackenUrlSlave-0/etc/promise/check-free-disk-space
+TestMalformedBackenUrlSlave-0/etc/promise/monitor-http-frontend
+TestMalformedBackenUrlSlave-0/etc/promise/monitor-httpd-listening-on-tcp
+TestMalformedBackenUrlSlave-0/etc/promise/promise-monitor-httpd-is-process-older-than-dependency-set
+TestMalformedBackenUrlSlave-1/etc/promise/caddy-frontend-is-running-actual-software-release
+TestMalformedBackenUrlSlave-1/etc/promise/caddy_cached
+TestMalformedBackenUrlSlave-1/etc/promise/caddy_frontend_ipv4_http
+TestMalformedBackenUrlSlave-1/etc/promise/caddy_frontend_ipv4_https
+TestMalformedBackenUrlSlave-1/etc/promise/caddy_frontend_ipv6_http
+TestMalformedBackenUrlSlave-1/etc/promise/caddy_frontend_ipv6_https
+TestMalformedBackenUrlSlave-1/etc/promise/caddy_ssl_cached
+TestMalformedBackenUrlSlave-1/etc/promise/check-free-disk-space
+TestMalformedBackenUrlSlave-1/etc/promise/frontend-caddy-configuration-promise
+TestMalformedBackenUrlSlave-1/etc/promise/monitor-http-frontend
+TestMalformedBackenUrlSlave-1/etc/promise/monitor-httpd-listening-on-tcp
+TestMalformedBackenUrlSlave-1/etc/promise/nginx-configuration-promise
+TestMalformedBackenUrlSlave-1/etc/promise/nginx_frontend_ipv4_http
+TestMalformedBackenUrlSlave-1/etc/promise/nginx_frontend_ipv4_https
+TestMalformedBackenUrlSlave-1/etc/promise/nginx_frontend_ipv6_http
+TestMalformedBackenUrlSlave-1/etc/promise/nginx_frontend_ipv6_https
+TestMalformedBackenUrlSlave-1/etc/promise/promise-monitor-httpd-is-process-older-than-dependency-set
+TestMalformedBackenUrlSlave-1/etc/promise/promise-nginx-is-process-older-than-dependency-set
+TestMalformedBackenUrlSlave-1/etc/promise/re6st-connectivity
+TestMalformedBackenUrlSlave-1/etc/promise/trafficserver-cache-availability
+TestMalformedBackenUrlSlave-1/etc/promise/trafficserver-port-listening
\ No newline at end of file

software/caddy-frontend/test/test_data/test.TestMalformedBackenUrlSlave.test_supervisor_state-CADDY.txt

+TestMalformedBackenUrlSlave-0:bootstrap-monitor EXITED
+TestMalformedBackenUrlSlave-0:certificate_authority-on-watch RUNNING
+TestMalformedBackenUrlSlave-0:crond RUNNING
+TestMalformedBackenUrlSlave-0:monitor-httpd-graceful EXITED
+TestMalformedBackenUrlSlave-0:monitor-httpd-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:6tunnel-11080-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:6tunnel-11443-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:6tunnel-12080-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:6tunnel-12443-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:6tunnel-26011-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:6tunnel-26012-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:bootstrap-monitor EXITED
+TestMalformedBackenUrlSlave-1:certificate_authority-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:crond-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:frontend-caddy-safe-graceful EXITED
+TestMalformedBackenUrlSlave-1:frontend-nginx-safe-graceful EXITED
+TestMalformedBackenUrlSlave-1:frontend_caddy-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:frontend_nginx-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:monitor-httpd-graceful EXITED
+TestMalformedBackenUrlSlave-1:monitor-httpd-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:trafficserver-on-watch RUNNING
+TestMalformedBackenUrlSlave-1:trafficserver-reload EXITED
+watchdog:watchdog RUNNING

software/hellorina/software.cfg

@@ -40,3 +40,6 @@ recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:_buildout_section_name_}
 mode = 755
 md5sum = 78b77a6bda9958f547f7d89b747731e3
+
+[versions]
+slapos.recipe.template = 4.3

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -273,6 +273,7 @@ parts =
   httpd-promise
   publish-connection-information
   directory-doc
+  monitor-htpasswd
 
 # Complete parts with sections
   {{ part_list | join('\n  ') }}

software/kvm/instance-kvm-resilient.cfg.jinja2

@@ -21,6 +21,7 @@ offline = true
 
 # += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
 parts +=
+  monitor-htpasswd
   {{ parts.replicate("kvm", backup_amount) }}
   publish-connection-information
   kvm-frontend-url-promise

software/kvm/software.cfg

@@ -108,7 +108,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
 mode = 644
-md5sum = ba3337b3678ed9d3578cc88749c5cd13
+md5sum = d9fe920d31f1ef0e377aa768ccd24f4c
 download-only = true
 on-update = true
 
@@ -117,7 +117,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
 mode = 644
-md5sum = 93e7143b46c6136b7cafe888fac90aba
+md5sum = 1095968487282784a735735aa1b37d35
 download-only = true
 on-update = true
 
@@ -255,7 +255,7 @@ context =
 # XXX - use websockify = 0.5.1 for compatibility with kvm frontend
 websockify = 0.5.1
 
-slapos.toolbox = 0.76
+slapos.toolbox = 0.77
 erp5.util = 0.4.51
 apache-libcloud = 1.1.0
 collective.recipe.environment = 0.2.0

software/monitor/instance-monitor.cfg.jinja2

@@ -89,11 +89,6 @@ command-line =
   ${monitor-directory:bin}/python {{ monitor_collect_csv_dump }} --output_folder ${monitor-directory:consumption}
 wrapper-path = ${monitor-directory:reports}/monitor-collect-csv-dump  
 
-[monitor-check-cpu-usage]
-recipe = slapos.cookbook:wrapper
-command-line = ${monitor-directory:bin}/python {{ monitor_check_system_health }} cpu ${init-monitor-parameters:cpu-load-file}
-wrapper-path = ${directory:promises}/system-CPU-load-check
-
 [monitor-check-memory-usage]
 recipe = slapos.cookbook:wrapper
 command-line = {{ buildout_bin}}/check-computer-memory
@@ -102,20 +97,22 @@ command-line = {{ buildout_bin}}/check-computer-memory
                --unit percent
 wrapper-path = ${directory:promises}/check-computer-memory-usage
 
+[monitor-check-cpu-usage]
+recipe = slapos.cookbook:promise.plugin
+eggs =
+  slapos.toolbox
+file = ${monitor-conf-parameters:promise-output-file}
+content = 
+  from slapos.promise.plugin.check_server_cpu_load import RunPromise
+output = ${directory:plugins}/system-CPU-load-check.py
+mode = 600
+config-cpu-load-threshold = ${slap-parameter:cpu-load-threshold}
+
 [publish-connection-information]
 recipe = slapos.cookbook:publish
 monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
 server_log_url = ${monitor-publish-parameters:monitor-base-url}/${slap-configuration:private-hash}/ 
 
-[init-monitor-parameters]
-recipe = plone.recipe.command
-cpu-load-file = ${directory:monitor}/cpu-load-tolerance
-mem-free-file = ${directory:monitor}/mem-free-limit
-command = 
-  if [ ! -s "${:cpu-load-file}" ]; then
-    echo ${slap-parameter:cpu-load-threshold} > ${:cpu-load-file}
-  fi
-
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration.serialised
 computer = ${slap-connection:computer-id}
@@ -127,5 +124,6 @@ private-hash = ${pwgen:passwd}${pwgen32:passwd}
 frontend-domain = 
 
 [slap-parameter]
-cpu-load-threshold = 2.0
+# Max cpu load for one core on server
+cpu-load-threshold = 3.0
 memory-percent-threshold = 96

software/monitor/instance.cfg

@@ -22,7 +22,6 @@ context = key develop_eggs_directory buildout:develop-eggs-directory
           raw monitor_template_output ${monitor-template:output}
           raw network_benck_cfg_output ${network-bench-cfg:output}
           raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
-          raw monitor_check_system_health ${monitor-system-health:output}
 mode = 0644
 
 [instance-base-distributor]

software/monitor/script/check_system_health.py

-#!/usr/bin/env python
-
-import subprocess
-import os
-import re
-import json
-
-cpu_command_list = ['top', '-n', '1', '-b']
-mem_command_list = ['free', '-m']
-head_command_list = ['head', '-n', '5']
-cpu_core_cmd_list = ['nproc']
-
-def cpu_usage(tolerance=1.5):
-  # tolerance=1.5 => accept up to 1.5 =150% CPU load
-  uptime_result = subprocess.check_output(['uptime'])
-  line = uptime_result.strip().split(' ')
-  load, load5, long_load = line[-3:]
-  core_count = int(subprocess.check_output(cpu_core_cmd_list).strip())
-  threshold = core_count * tolerance
-  if float(long_load) > threshold:
-    # display top statistics
-    top = subprocess.Popen(cpu_command_list, stdout=subprocess.PIPE)
-    result = subprocess.check_output(head_command_list, stdin=top.stdout)
-    message = "CPU load is high: %s %s %s\n\n" % (load, load5, long_load)
-    message += result
-    return message
-
-def check_last_result(file, last_value, threshold=7.0, elt_count=5):
-  mem_average = 0.0
-  value_list = []
-  if os.path.exists(file):
-    with open(file) as f:
-      values = f.read()
-      value_list = values.split(' ')
-      size = len(value_list)
-      value_list.append(str(last_value))
-      if size >= elt_count:
-        while len(value_list) > elt_count:
-          value_list.pop(0)
-        # calculate average
-        average = sum([float(l) for l in value_list])/(size * 1.0)
-        if average < threshold:
-          mem_average = round(average, 2)
-  else:
-    value_list.append(str(last_value))
-
-  with open(file, 'w') as f:
-    f.write(' '.join(value_list))
-  return mem_average
-
-def memory_usage(storage_file, threshold=7.0, elt_count=5):
-  mem_stats = subprocess.check_output(mem_command_list)
-  result_list = mem_stats.split('\n')
-  usage = re.sub('\s+', ' ', result_list[1])
-  usage_real = re.sub('\s+', ' ', result_list[2])
-  usage_list = usage.split(' ')
-  mem_total = float(usage_list[1])
-  mem_free = float(usage_real.split(' ')[-1])
-  if mem_free == 0.0:
-    mem_available = 0.0
-  else:
-    mem_available = round(mem_free * 100 / (mem_total * 1.0), 2)
-  average = check_last_result(
-                              storage_file,
-                              mem_available,
-                              threshold=threshold,
-                              elt_count=elt_count)
-  if average != 0.0 and average < threshold:
-    # mem used at (threshold)% at least
-    message = "Memory usage is high. %s%% is available (%s%% for last %s minutes).\n\n" % (
-      mem_available, average, elt_count)
-    message += mem_stats
-    return message
-  swap_usage = re.sub('\s+', ' ', result_list[3])
-  swap_usage_list = swap_usage.split(' ')
-  swap_total = float(swap_usage_list[1])
-  swap_free = float(swap_usage_list[3])
-  if swap_total > 1:
-    if swap_free == 0.0:
-      swap_available = 0.0
-    else:
-      swap_available = round(swap_free * 100 / (swap_total * 1.0), 2) * 100
-    if swap_available < threshold*1.7:
-      message = "Memory SWAP usage is high. %s%% is available.\n\n" % swap_available
-      message += mem_stats
-      return message
-
-if __name__ == '__main__':
-  if len(sys.argv) < 2:
-    print "Usage: %s [cpu | mem] CONFIG_FILE [BASE_DIR]" % os.path.basename(sys.argv[0])
-    exit(2)
-  check_type = sys.argv[1]
-  threshold = None
-  if len(sys.argv) >= 3:
-    config_file = sys.argv[2]
-    if os.path.exists(config_file):
-      with open(config_file) as f:
-        try:
-          threshold = float(f.read())
-          if not threshold > 0:
-            threshold = None
-        except ValueError:
-          pass
-
-  if check_type == "cpu":
-    result = cpu_usage(threshold or 1.5)
-    if result:
-      print result
-      exit(1)
-  elif check_type == "mem":
-    directory = ""
-    if len(sys.argv) >= 4:
-      directory = sys.argv[3]
-    if not os.path.exists(directory) or not os.path.isdir(directory):
-      directory = os.getcwd()
-    storage_file = os.path.join(directory, 'mem-usage.mo')
-    result = memory_usage(storage_file, threshold=(threshold or 4.0), elt_count=10)
-    if result:
-      print result
-      exit(1)
-  else:
-    exit(3)
-
-  exit(0)
\ No newline at end of file

software/monitor/software.cfg

@@ -21,14 +21,14 @@ parts =
 recipe = slapos.recipe.template 
 url = ${:_profile_base_location_}/instance.cfg
 output = ${buildout:directory}/template.cfg
-md5sum = 641c5916739f78171c616af00fe974a2
+md5sum = 1b7d2d097f208f6641bf98a17df079c8
 mode = 0644
 
 [template-monitor]
 recipe = slapos.recipe.build:download 
 url = ${:_profile_base_location_}/instance-monitor.cfg.jinja2
 destination = ${buildout:directory}/template-base-monitor.cfg
-md5sum = 79125819f20f4f18a301b806daed2ceb
+md5sum = ef3297619e1fc2a5a8d1b0546c1a0db2
 mode = 0644
 
 [template-monitor-distributor]
@@ -59,13 +59,6 @@ filename = collect_csv_dump.py
 output = ${:destination}/${:filename}
 md5sum = cad2402bbd21907cfed6bc5af8c5d3ab
 
-[monitor-system-health]
-<= monitor-template-script
-url = ${:_profile_base_location_}/script/${:filename}
-filename = check_system_health.py
-output = ${:destination}/${:filename}
-md5sum = 7eb74a0be4995c6a1015a9a1eb6874c6
-
 [extra-eggs]
 <= monitor-eggs
 interpreter = pythonwitheggs

software/neoppod/software-common.cfg

@@ -55,6 +55,8 @@ eggs = neoppod[admin, ctl, master, storage-mysqldb]
   ZODB
   zope.testing
   zodbtools
+  coverage
+  setproctitle
 patch-binary = ${patch:location}/bin/patch
 ZEO-patch-options = -p1
 ZEO-patches =
@@ -121,15 +123,18 @@ md5sum = 87d18c7021e4d43756813a83c9da5e97
 BTrees = 4.4.1
 ZODB = 4.4.5
 apache-libcloud = 1.5.0
+coverage = 4.5.1
 ecdsa = 0.13
 gitdb2 = 2.0.0
+msgpack = 0.5.6
 msgpack-python = 0.5.6
 mysqlclient = 1.3.12
 persistent = 4.2.3
 pycrypto = 2.6.1
 pycurl = 7.43.0
+setproctitle = 1.1.10
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.76
+slapos.toolbox = 0.77
 smmap2 = 2.0.1
 transaction = 1.7.0
 zodbpickle = 0.6.0

software/nginx-push-stream/software.cfg

@@ -46,7 +46,7 @@ mode = 0644
 
 [versions]
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.76
+slapos.toolbox = 0.77
 dnspython = 1.15.0
 PyRSS2Gen = 1.1
 erp5.util = 0.4.51

software/re6stnet/software.cfg

@@ -110,7 +110,7 @@ filename = registry-run.in
 md5sum = 0bf4f2c03e06b55c6c6cc55fa33e65d6
 
 [versions]
-re6stnet = 0.494
+re6stnet = 0.501
 apache-libcloud = 0.17.0
 ecdsa = 0.13
 gitdb = 0.6.4
@@ -118,7 +118,7 @@ plone.recipe.command = 1.1
 pycrypto = 2.6.1
 pycurl = 7.43.0
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.76
+slapos.toolbox = 0.77
 smmap = 0.9.0
 dnspython = 1.15.0
 erp5.util = 0.4.51

software/slapos-testing/software.cfg

@@ -175,7 +175,7 @@ Pygments = 2.1.3
 slapos.recipe.build = 0.36
 slapos.recipe.cmmi = 0.7
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.76 
+slapos.toolbox = 0.77
 
 # All depencies should be pinned.
 apache-libcloud = 2.3.0

software/slaprunner/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = e033845c9c24e4bb20caeedf19f9628a
 
 [instance-runner-import]
 filename = instance-runner-import.cfg.in
-md5sum = 5cfa49bcf20612844e1c50a85740d0b3
+md5sum = 7a879739afe55320ee96409bcc8a52ab
 
 [template-runner-export-script]
 filename = template/runner-export.sh.jinja2
@@ -34,11 +34,11 @@ md5sum = 98ce179badc6af5979a64a7c3d0a2ceb
 
 [instance-runner-export]
 filename = instance-runner-export.cfg.in
-md5sum = e0e62859fc00207e40af6acb402665fb
+md5sum = 1a812a06cc02bb11636009f4ec043d54
 
 [template-resilient]
 filename = instance-resilient.cfg.jinja2
-md5sum = a902b84ac7d1e29a7fdb06cbc7dec150
+md5sum = 8ed180de711d207a540d0acb539b2536
 
 [template_nginx_conf]
 filename = nginx_conf.in

software/slaprunner/instance-resilient.cfg.jinja2

@@ -28,6 +28,7 @@ offline = true
 
 # += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
 parts +=
+  monitor-htpasswd
   {{ parts.replicate("runner", number_of_instances + 1) }}
   publish-connection-information
 

software/slaprunner/instance-runner-export.cfg.in

@@ -41,6 +41,11 @@ parts +=
   monitor-check-resilient-feed-file
   monitor-check-webrunner-internal-instance
 
+[directory]
+recipe = slapos.cookbook:mkdirectory
+# XXX - keep srv path with slash at the end.
+srv = ${:home}/srv/
+
 [proxy-free-port]
 recipe = slapos.cookbook:free_port
 minimum = 49980

software/slaprunner/instance-runner-import.cfg.in

@@ -34,6 +34,12 @@ parts +=
 
   monitor-base
 
+[directory]
+recipe = slapos.cookbook:mkdirectory
+# XXX - keep srv path with slash at the end.
+srv = ${:home}/srv/
+
+
 # For the needs of importer, we run the full slaprunner
 # In case both exporter and importer (aka main instance and clone instance)
 # run with the same IP (usually for testing purposes),

software/slaprunner/software.cfg

@@ -171,7 +171,7 @@ prettytable = 0.7.2
 pycurl = 7.43.0
 slapos.recipe.template = 4.3
 collective.recipe.environment = 0.2.0
-slapos.toolbox = 0.76
+slapos.toolbox = 0.77
 smmap = 0.9.0
 lockfile = 0.12.2
 

stack/caucase/buildout.cfg

@@ -95,7 +95,7 @@ futures = 3.1.1
 gitdb2 = 2.0.2
 gunicorn = 19.7.1
 slapos.recipe.template = 4.3
-slapos.toolbox = 0.76
+slapos.toolbox = 0.77
 smmap2 = 2.0.3
 
 # Required by:

stack/erp5/buildout.cfg

@@ -39,6 +39,8 @@ extends =
   ../../component/python-ldap-python/buildout.cfg
   ../../component/rdiff-backup/buildout.cfg
   ../../component/scikit-learn/buildout.cfg
+  ../../component/scikit-image/buildout.cfg
+  ../../component/PyWavelets/buildout.cfg
   ../../component/stunnel/buildout.cfg
   ../../component/subversion/buildout.cfg
   ../../component/tesseract/buildout.cfg
@@ -92,6 +94,8 @@ parts +=
   kumo
   w3-validator
   tesseract
+  tesseract-eng-traineddata
+  tesseract-osd-traineddata
   scipy
   ocropy
   hookbox
@@ -448,6 +452,7 @@ eggs = ${neoppod:eggs}
   ${pycrypto-python:egg}
   ${scipy:egg}
   ${scikit-learn:egg}
+  ${scikit-image:egg}
   sympy
   ${h5py:egg}
   openpyxl
@@ -497,6 +502,9 @@ eggs = ${neoppod:eggs}
   xfw
   jsonschema
   selenium
+  pytesseract
+  decorator
+  networkx
 # Needed for checking ZODB Components source code
   pylint
   pytracemalloc
@@ -627,7 +635,6 @@ python-magic = 0.4.12+SlapOSPatched001
 # use newer version than specified in ZTK
 PasteDeploy = 1.5.2
 argparse = 1.4.0
-coverage = 4.3.4
 zope.dottedname = 4.1.0
 
 # test_UserManagerInterfaces in testERP5Security fails with 1.10.0.
@@ -657,7 +664,7 @@ zope.app.testing = 3.8.1
 
 # Pinned versions
 APacheDEX = 1.6.2
-Pillow = 4.0.0
+Pillow = 5.2.0
 Products.CMFActionIcons = 2.1.3
 Products.DCWorkflowGraph = 0.4.1
 # Products.ExternalEditor 2.0.0's dtml is not based on Zope2 OFS's one.
@@ -725,7 +732,14 @@ validictory = 1.1.0
 xfw = 0.10
 xupdate-processor = 0.4
 selenium = 3.8.0
+scikit-image = 0.14.0
+PyWavelets = 0.5.2
+networkx = 2.1
+pytesseract = 0.2.2
 zbarlight = 2.0
+cloudpickle = 0.5.3
+dask = 0.18.1
+toolz = 0.9.0
 
 # Required by:
 # Products.CMFCore==2.2.10
@@ -782,3 +796,4 @@ unidiff = 0.5.5
 # Required by:
 # deepdiff = 3.3.0
 jsonpickle = 0.9.6
+decorator = 4.3.0

stack/monitor/buildout.cfg

@@ -121,6 +121,6 @@ depends =
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 pycurl = 7.43.0
-slapos.toolbox = 0.76
+slapos.toolbox = 0.77
 pyasn1 = 0.3.7
 

stack/resilient/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = c6c11db5372150019debb1ce519b907d
 
 [template-pull-backup]
 filename = instance-pull-backup.cfg.in
-md5sum = 85e777bd349a5c881d36e747882aee8a
+md5sum = cda4bbedb3ec014ba0311629dd003b3a
 
 [template-replicated]
 filename = template-replicated.cfg.in

stack/resilient/instance-pull-backup.cfg.in

@@ -277,11 +277,23 @@ monitor-username = admin
 recipe = slapos.recipe.template:jinja2
 template = inline:
   #!${dash:location}/bin/dash
+  # only check integrity if pull is not running
+  latest_item=$(ls -t $${pbs:status-item-directory} | head -n1)
+  if [ ! -z "$latest_item" ]; then
+    pbs_result=$(cat "$${pbs:status-item-directory}/$latest_item" | python -c "import sys, json; print json.load(sys.stdin)['title']" 2>/dev/null)
+    if [ "$?" -eq 0 ]; then
+      echo $pbs_result | egrep "pull_raw\s*:\s*STARTED" > /dev/null
+      if [ "$?" -eq 0 ]; then
+        echo "Skipped, PBS pull is running.";
+        exit 0;
+      fi
+    fi
+  fi
   # Raise an error if signatures are different
   # Error cannot be deduced if files do not exist
   cd $${directory:pbs-backup}
   if [ ! -f "proof.signature" ]; then exit 0; fi
-  backup_signature=$(find . -maxdepth 2 -name backup.signature)
+  backup_signature=$(find . -maxdepth 2 -name backup.signature -not -path "./tmp/*")
   if [ -z "$backup_signature" ]; then
     exit 0;
   else

stack/slapos.cfg

@@ -134,7 +134,7 @@ pytz = 2016.10
 requests = 2.13.0
 six = 1.10.0
 slapos.cookbook = 1.0.66
-slapos.core = 1.4.8
+slapos.core = 1.4.9
 slapos.extension.strip = 0.4
 slapos.libnetworkcache = 0.15
 slapos.rebootstrap = 4.1