Update Release Candidate

Conflicts:
	software/slapos-master/software.cfg

component/apache/apache-backend.conf.in

@@ -78,7 +78,7 @@ LoadModule headers_module modules/mod_headers.so
 LoadModule deflate_module modules/mod_deflate.so
 LoadModule filter_module modules/mod_filter.so
 
-AddOutputFilterByType DEFLATE text/cache-manifest text/html text/plain text/css application/hal+json application/json application/x-javascript text/xml application/xml application/rss+xml text/javascript image/svg+xml
+AddOutputFilterByType DEFLATE text/cache-manifest text/html text/plain text/css application/hal+json application/json application/x-javascript text/xml application/xml application/rss+xml text/javascript image/svg+xml application/x-font-ttf application/font-woff application/font-woff2 application/x-font-opentype
 
 PidFile "{{ parameter_dict['pid-file'] }}"
 ServerAdmin admin@

component/apache/buildout.cfg

@@ -187,5 +187,5 @@ make-targets =
 [template-apache-backend-conf]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/apache-backend.conf.in
-md5sum = e376fdbe8b19551ec08604e64e9a53b9
+md5sum = feef079241bda3407b7ceed5876cb61f
 mode = 640

component/curl/buildout.cfg

@@ -12,8 +12,8 @@ parts =
 
 [curl]
 recipe = slapos.recipe.cmmi
-url = http://curl.haxx.se/download/curl-7.49.0.tar.bz2
-md5sum = 7416aaff4a9210b43edda7615ffa4169
+url = http://curl.haxx.se/download/curl-7.50.1.tar.bz2
+md5sum = 015f6a0217ca6f2c5442ca406476920b
 configure-options =
   --disable-static
   --disable-ldap

component/dropbear/buildout.cfg

@@ -20,11 +20,9 @@ md5sum = 0284ea239083f04c8b874e08e1aca243
 #      in order have all patches working.
 url = http://matt.ucc.asn.au/dropbear/releases/dropbear-0.53.1.tar.bz2
 
-# NOTE DEFAULT_RECV_WINDOW and RECV_MAX_PAYLOAD_LEN are tweaked to support
-#      faster network throughput compared to dropbear defaults.
 configure-options =
   --with-zlib=${zlib:location}
-  CFLAGS="-DENABLE_SINGLEUSER -D__DIRTY_NO_SHELL_CHECKING -DDEFAULT_RECV_WINDOW=1048576 -DRECV_MAX_PAYLOAD_LEN=524288"
+  CFLAGS="-DENABLE_SINGLEUSER -D__DIRTY_NO_SHELL_CHECKING"
 
 environment =
   CPPFLAGS =-I${zlib:location}/include

component/egg-patch/ZODB3-persistent-ghostify-slots.patch

+From d387a425941b37b99355077657edf7a2f117cf47 Mon Sep 17 00:00:00 2001
+From: Kirill Smelkov <kirr@nexedi.com>
+Date: Thu, 21 Jul 2016 22:34:55 +0300
+Subject: [PATCH] persistent: On deactivate release in-slots objects too
+
+( This is backport of https://github.com/zopefoundation/persistent/pull/44
+  to ZODB-3.10 )
+
+On ._p_deactivate() and ._p_invalidate(), when an object goes to ghost
+state, objects referenced by all its attributes, except related to
+persistence machinery, are released, this way freeing memory (if they
+were referenced only from going-to-ghost object).
+
+That's the idea - an object in ghost state is simply a stub, which loads
+its content on first access (via hooking into get/set attr) while
+occupying minimal memory in not-yet-loaded state.
+
+However the above is not completely true right now, as currently on
+ghostification only object's .__dict__ is released, while in-slots objects
+are retained attached to ghost object staying in RAM:
+
+    ---- 8< ----
+    from ZODB import DB
+    from persistent import Persistent
+    import gc
+
+    db = DB(None)
+    jar = db.open()
+
+    class C:
+        def __init__(self, v):
+            self.v = v
+        def __del__(self):
+            print 'released (%s)' % self.v
+
+    class P1(Persistent):
+        pass
+
+    class P2(Persistent):
+        __slots__ = ('aaa')
+
+    p1 = P1()
+    jar.add(p1)
+    p1.aaa = C(1)
+
+    p2 = P2()
+    jar.add(p2)
+    p2.aaa = C(2)
+
+    p1._p_invalidate()
+    # "released (1)" is printed
+
+    p2._p_invalidate()
+    gc.collect()
+    # "released (2)" is NOT printed     <--
+    ---- 8< ----
+
+So teach ghostify() & friends to release objects in slots to free-up
+memory when an object goes to ghost state.
+
+NOTE PyErr_Occurred() added after ghostify() calls because
+pickle_slotnames() can raise an error, but we do not want to change
+ghostify() prototype for backward compatibility reason - as it is used
+in cPersistenceCAPIstruct.
+
+( I hit this bug with wendelin.core which uses proxies to load
+  data from DB to virtual memory manager and then deactivate proxy right
+  after load has been completed:
+
+  https://lab.nexedi.com/nexedi/wendelin.core/blob/f7803634/bigfile/file_zodb.py#L239
+  https://lab.nexedi.com/nexedi/wendelin.core/blob/f7803634/bigfile/file_zodb.py#L295 )
+---
+ src/persistent/cPersistence.c          | 41 +++++++++++++++++++++++++++++++++-
+ src/persistent/tests/testPersistent.py | 24 ++++++++++++++++++++
+ 2 files changed, 64 insertions(+), 1 deletion(-)
+
+diff --git a/src/persistent/cPersistence.c b/src/persistent/cPersistence.c
+index b4a185c..28d1f9a 100644
+--- a/src/persistent/cPersistence.c
++++ b/src/persistent/cPersistence.c
+@@ -75,6 +75,7 @@ fatal_1350(cPersistentObject *self, const char *caller, const char *detail)
+ #endif
+ 
+ static void ghostify(cPersistentObject*);
++static PyObject * pickle_slotnames(PyTypeObject *cls);
+ 
+ /* Load the state of the object, unghostifying it.  Upon success, return 1.
+  * If an error occurred, re-ghostify the object and return -1.
+@@ -141,7 +142,7 @@ accessed(cPersistentObject *self)
+ static void
+ ghostify(cPersistentObject *self)
+ {
+-  PyObject **dictptr;
++  PyObject **dictptr, *slotnames;
+ 
+   /* are we already a ghost? */
+   if (self->state == cPersistent_GHOST_STATE)
+@@ -171,6 +172,8 @@ ghostify(cPersistentObject *self)
+     _estimated_size_in_bytes(self->estimated_size);
+   ring_del(&self->ring);
+   self->state = cPersistent_GHOST_STATE;
++
++  /* clear __dict__ */
+   dictptr = _PyObject_GetDictPtr((PyObject *)self);
+   if (dictptr && *dictptr)
+     {
+@@ -178,6 +181,38 @@ ghostify(cPersistentObject *self)
+       *dictptr = NULL;
+     }
+ 
++  /* clear all slots besides _p_* */
++  slotnames = pickle_slotnames(Py_TYPE(self));
++  if (slotnames && slotnames != Py_None)
++  {
++    int i;
++
++    for (i = 0; i < PyList_GET_SIZE(slotnames); i++)
++    {
++        PyObject *name;
++        char *cname;
++        int is_special;
++
++        name = PyList_GET_ITEM(slotnames, i);
++        if (PyBytes_Check(name))
++        {
++            cname = PyBytes_AS_STRING(name);
++            is_special = !strncmp(cname, "_p_", 3);
++            if (is_special) /* skip persistent */
++            {
++                continue;
++            }
++        }
++
++        /* NOTE: this skips our delattr hook */
++        if (PyObject_GenericSetAttr((PyObject *)self, name, NULL) < 0)
++            /* delattr of non-set slot will raise AttributeError - we
++             * simply ignore. */
++            PyErr_Clear();
++    }
++  }
++  Py_XDECREF(slotnames);
++
+   /* We remove the reference to the just ghosted object that the ring
+    * holds.  Note that the dictionary of oids->objects has an uncounted
+    * reference, so if the ring's reference was the only one, this frees
+@@ -261,6 +296,8 @@ Per__p_deactivate(cPersistentObject *self)
+          called directly. Methods that override this need to
+          do the same! */
+       ghostify(self);
++      if (PyErr_Occurred())
++        return NULL;
+     }
+ 
+   Py_INCREF(Py_None);
+@@ -289,6 +326,8 @@ Per__p_invalidate(cPersistentObject *self)
+       if (Per_set_changed(self, NULL) < 0)
+         return NULL;
+       ghostify(self);
++      if (PyErr_Occurred())
++        return NULL;
+     }
+   Py_INCREF(Py_None);
+   return Py_None;
+diff --git a/src/persistent/tests/testPersistent.py b/src/persistent/tests/testPersistent.py
+index 51e0382..fdb8b67 100644
+--- a/src/persistent/tests/testPersistent.py
++++ b/src/persistent/tests/testPersistent.py
+@@ -180,6 +180,30 @@ class PersistenceTest(unittest.TestCase):
+         self.assertEqual(obj._p_changed, None)
+         self.assertEqual(obj._p_state, GHOST)
+ 
++    def test__p_invalidate_from_changed_w_slots(self):
++        from persistent import Persistent
++        class Derived(Persistent):
++            __slots__ = ('myattr1', 'myattr2')
++            def __init__(self):
++                self.myattr1 = 'value1'
++                self.myattr2 = 'value2'
++        obj = Derived()
++        jar = self._makeJar()
++        jar.add(obj)
++        obj._p_activate()
++        obj._p_changed = True
++        jar._loaded = []
++        jar._registered = []
++        self.assertEqual(Derived.myattr1.__get__(obj), 'value1')
++        self.assertEqual(Derived.myattr2.__get__(obj), 'value2')
++        obj._p_invalidate()
++        self.assertIs(obj._p_changed, None)
++        self.assertEqual(list(jar._loaded), [])
++        self.assertRaises(AttributeError, lambda: Derived.myattr1.__get__(obj))
++        self.assertRaises(AttributeError, lambda: Derived.myattr2.__get__(obj))
++        self.assertEqual(list(jar._loaded), [])
++        self.assertEqual(list(jar._registered), [])
++
+     def test_initial_serial(self):
+         NOSERIAL = "\000" * 8
+         obj = self._makeOne()
+-- 
+2.9.2.701.gf965a18.dirty
+

component/flex/buildout.cfg

 [buildout]
 extends =
   ../m4/buildout.cfg
+  ../xz-utils/buildout.cfg
 parts =
   flex
 
 [flex]
 recipe = slapos.recipe.cmmi
-url = http://downloads.sourceforge.net/project/flex/flex/flex-2.5.35/flex-2.5.35.tar.gz
-md5sum = 201d3f38758d95436cbc64903386de0b
+url = http://downloads.sourceforge.net/project/flex/flex-2.6.0.tar.xz
+md5sum = 3cbbfa1554d0b75fad9f8100732454de
 environment =
   M4=${m4:location}/bin/m4
+  PATH=${xz-utils:location}/bin:%(PATH)s

component/fontconfig/buildout.cfg

@@ -13,8 +13,8 @@ parts =
 
 [fontconfig]
 recipe = slapos.recipe.cmmi
-url = http://fontconfig.org/release/fontconfig-2.11.1.tar.bz2
-md5sum = 824d000eb737af6e16c826dd3b2d6c90
+url = http://fontconfig.org/release/fontconfig-2.12.1.tar.bz2
+md5sum = b5af5a423ee3b5cfc34846838963c058
 pkg_config_depends = ${freetype:pkg_config_depends}:${freetype:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig
 # XXX-Cedric : should we use --with-add-fonts={somefont:location}/share,{someotherfont:location}/share?
 configure-options =

component/gcc/buildout.cfg

@@ -44,8 +44,8 @@ environment =
 
 [gcc-common]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/gcc/gcc-5.3.0/gcc-5.3.0.tar.bz2
-md5sum = c9616fd448f980259c31de613e575719
+url = http://ftp.gnu.org/gnu/gcc/gcc-5.4.0/gcc-5.4.0.tar.bz2
+md5sum = 4c626ac2a83ef30dfb9260e6f59c2b30
 # make install does not work when several core are used
 make-targets = install -j1
 

component/gettext/buildout.cfg

@@ -10,8 +10,8 @@ extends =
 
 [gettext]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/pub/gnu/gettext/gettext-0.19.6.tar.lz
-md5sum = 45b2a123cdc7cef54df98152a0da3fcc
+url = http://ftp.gnu.org/pub/gnu/gettext/gettext-0.19.8.1.tar.lz
+md5sum = d838d2c4144261d0c5fbab4a0aceb5c1
 
 configure-options =
   --disable-static

component/gmp/buildout.cfg

@@ -8,9 +8,9 @@ parts =
 
 [gmp]
 recipe = slapos.recipe.cmmi
-version = 6.1.0
+version = 6.1.1
 url = https://gmplib.org/download/gmp/gmp-${:version}.tar.xz
-md5sum = a9868ef2556ad6a2909babcd1428f3c7
+md5sum = e70e183609244a332d80529e7e155a35
 configure-options =
   --enable-cxx
   --disable-static

component/grep/buildout.cfg

@@ -8,8 +8,8 @@ parts =
 
 [grep]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/grep/grep-2.21.tar.xz
-md5sum = 43c48064d6409862b8a850db83c8038a
+url = http://ftp.gnu.org/gnu/grep/grep-2.25.tar.xz
+md5sum = 04e96b0e6f0fe6a180ae62c88fcd0af6
 environment =
   PATH=${xz-utils:location}/bin:%(PATH)s
   CPPFLAGS=-I${pcre:location}/include

component/groonga/buildout.cfg

@@ -14,11 +14,12 @@ extends =
 
 [groonga]
 recipe = slapos.recipe.cmmi
-url = http://packages.groonga.org/source/groonga/groonga-6.0.2.tar.gz
-md5sum = 10fde26350296a479953fd28679f2566
+url = http://packages.groonga.org/source/groonga/groonga-6.0.8.tar.gz
+md5sum = 42c78baddfca42d3a19fe8b06fac821d
 # temporary patch to respect more tokens in natural language mode.
 patches =
   ${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b
+  ${:_profile_base_location_}/groonga-6.0.8-bugfix04107.patch#7618361b006abdbcceb1fa1df0bd2136
 patch-options = -p1
 configure-options =
   --disable-static

component/groonga/groonga-6.0.8-bugfix04107.patch

+diff --git a/lib/ii.c b/lib/ii.c
+index 97b34ec..9523cf7 100644
+--- a/lib/ii.c
++++ b/lib/ii.c
+@@ -5110,14 +5110,18 @@ grn_ii_cursor_next_internal(grn_ctx *ctx, grn_ii_cursor *c,
+               c->pb.rid = 0;
+               if (br->jump > 0 && !BUFFER_REC_DELETED(br)) {
+                 buffer_rec *jump_br = BUFFER_REC_AT(c->buf, br->jump);
+-                uint8_t *jump_bp;
+-                uint32_t jump_rid;
+-                jump_bp = GRN_NEXT_ADDR(jump_br);
+-                GRN_B_DEC(jump_rid, jump_bp);
+-                if (jump_rid < c->min) {
+-                  c->nextb = br->jump;
+-                } else {
++                if (BUFFER_REC_DELETED(jump_br)) {
+                   c->nextb = br->step;
++                } else {
++                  uint8_t *jump_bp;
++                  uint32_t jump_rid;
++                  jump_bp = GRN_NEXT_ADDR(jump_br);
++                  GRN_B_DEC(jump_rid, jump_bp);
++                  if (jump_rid < c->min) {
++                    c->nextb = br->jump;
++                  } else {
++                    c->nextb = br->step;
++                  }
+                 }
+               } else {
+                 c->nextb = br->step;

component/gzip/buildout.cfg

 [buildout]
 extends =
-  ../patch/buildout.cfg
   ../xz-utils/buildout.cfg
 parts =
   gzip
 
 [gzip]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.gnu.org/pub/gnu/gzip/gzip-1.6.tar.xz
-md5sum = da981f86677d58a106496e68de6f8995
+url = ftp://ftp.gnu.org/pub/gnu/gzip/gzip-1.8.tar.xz
+md5sum = f7caabb65cddc1a4165b398009bd05b9
 environment =
-  PATH=${patch:location}/bin:${xz-utils:location}/bin:%(PATH)s
-patch-options = -p1
-# The --rsyncable patch is from debian/ubuntu,
-# specifically https://launchpad.net/ubuntu/+source/gzip/1.6-3ubuntu1
-# It is required to minimize the bandwidth used by rsync.
-# For an explanation, see http://beeznest.wordpress.com/2005/02/03/rsyncable-gzip/
-# Hunks for .texi files have been removed to avoid a dependency on makeinfo.
-patches =
-  ${:_profile_base_location_}/rsyncable.diff#0587af03a5580e2b7b4007469ee2b601
-
+  PATH=${xz-utils:location}/bin:%(PATH)s

component/imagemagick/buildout.cfg

@@ -25,9 +25,9 @@ extends =
 
 [imagemagick]
 recipe = slapos.recipe.cmmi
-version = 7.0.1-1
+version = 7.0.2-10
 url = https://www.imagemagick.org/download/releases/ImageMagick-${:version}.tar.xz
-md5sum = 24673d00fcc8aa00313eeca6aa20fd3c
+md5sum = e1cb23d9c10a8eff228ef30ee281711a
 pkg_config_depends = ${fontconfig:location}/lib/pkgconfig:${fontconfig:pkg_config_depends}:${lcms2:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig
 configure-options =
   --disable-static
@@ -58,8 +58,8 @@ configure-options =
   --with-frozenpaths
 patch-options = -p1
 patches =
-  ${:_profile_base_location_}/imagemagick-6.6.6-1-no-gsx-gsc-probe.patch#3f28ecd9f6722cf2c3238ce6ec3d7a68
-  ${:_profile_base_location_}/safe_policy.patch#07889fefbd9b55f19b9136ed6c17aa8c
+  ${:_profile_base_location_}/imagemagick-7.0.2-10-no-gsx-gsc-probe.patch#64898455d5175efedd1a7bef9f1f18b5
+  ${:_profile_base_location_}/safe_policy.patch#383c0392de7257c9dff7270973342914
 environment =
   PATH=${freetype:location}/bin:${ghostscript:location}/bin:${inkscape:location}/bin:${libxml2:location}/bin:${patch:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${:pkg_config_depends}

component/imagemagick/imagemagick-6.6.6-1-no-gsx-gsc-probe.patch → component/imagemagick/imagemagick-7.0.2-10-no-gsx-gsc-probe.patch

---- ImageMagick-6.6.6-1/configure.ac~	2010-11-28 21:51:05.000000000 +0100
-+++ ImageMagick-6.6.6-1/configure.ac	2010-12-03 14:13:14.000000000 +0100
-@@ -2791,7 +2791,7 @@
+--- ImageMagick-7.0.2-10/configure.ac.orig      2016-08-30 11:33:39.160279386 +0200
++++ ImageMagick-7.0.2-10/configure.ac   2016-08-30 11:35:34.753290590 +0200
+@@ -3110,7 +3110,7 @@
+ AC_PATH_PROG(MrSIDDecodeDelegate, "$MrSIDDecodeDelegateDefault", "$MrSIDDecodeDelegateDefault")
+ AC_PATH_PROG(MVDelegate, "$MVDelegateDefault", "$MVDelegateDefault")
  AC_PATH_PROG(PCLDelegate, "$PCLDelegateDefault", "$PCLDelegateDefault")
- AC_PATH_PROG(PGPDecodeDelegate, "$PGPDecodeDelegateDefault", "$PGPDecodeDelegateDefault")
- AC_PATH_PROG(POVDelegate, "$POVDelegateDefault", "$POVDelegateDefault")
 -AC_PATH_PROGS(PSDelegate, gsx gsc "$PSDelegateDefault", "$PSDelegateDefault")
 +AC_PATH_PROGS(PSDelegate, "$PSDelegateDefault", "$PSDelegateDefault")
- AC_PATH_PROG(RLEEncodeDelegate, "$RLEEncodeDelegateDefault", "$RLEEncodeDelegateDefault")
  AC_PATH_PROG(RMDelegate, "$RMDelegateDefault", "$RMDelegateDefault")
  AC_PATH_PROG(RSVGDecodeDelegate, "$RSVGDecodeDelegateDefault", "$RSVGDecodeDelegateDefault")
---- ImageMagick-6.6.6-1/configure~	2010-11-28 23:27:16.000000000 +0100
-+++ ImageMagick-6.6.6-1/configure	2010-12-03 14:13:57.000000000 +0100
-@@ -30931,7 +30931,7 @@
- fi
- 
- 
--for ac_prog in gsx gsc "$PSDelegateDefault"
-+for ac_prog in "$PSDelegateDefault"
- do
-   # Extract the first word of "$ac_prog", so it can be a program name with args.
- set dummy $ac_prog; ac_word=$2
+ AC_PATH_PROG(SVGDecodeDelegate, "$SVGDecodeDelegateDefault", "$SVGDecodeDelegateDefault")

component/imagemagick/safe_policy.patch

---- ImageMagick-7.0.1-1/config/policy.xml.orig	2016-05-04 00:18:54.000000000 +0200
-+++ ImageMagick-7.0.1-1/config/policy.xml	2016-05-06 08:22:30.002045742 +0200
-@@ -50,16 +50,25 @@
+--- ImageMagick-7.0.2-10/config/policy.xml.orig	2016-08-30 11:37:29.110253211 +0200
++++ ImageMagick-7.0.2-10/config/policy.xml	2016-08-30 11:40:09.719555899 +0200
+@@ -50,19 +50,28 @@
  -->
  <policymap>
    <!-- <policy domain="resource" name="temporary-path" value="/tmp"/> -->
@@ -15,6 +15,9 @@
 -  <!-- <policy domain="resource" name="throttle" value="0"/> -->
 -  <!-- <policy domain="resource" name="time" value="3600"/> -->
 -  <!-- <policy domain="system" name="precision" value="6"/> -->
+-  <!-- <policy domain="coder" rights="none" pattern="MVG" /> -->
+-  <!-- <policy domain="delegate" rights="none" pattern="HTTPS" /> -->
+-  <!-- <policy domain="path" rights="none" pattern="@*" /> -->
 +  <policy domain="resource" name="memory" value="2GiB"/>
 +  <policy domain="resource" name="map" value="4GiB"/>
 +  <policy domain="resource" name="width" value="10MP"/>
@@ -26,7 +29,10 @@
 +  <policy domain="resource" name="throttle" value="0"/>
 +  <policy domain="resource" name="time" value="3600"/>
 +  <policy domain="system" name="precision" value="6"/>
-   <policy domain="cache" name="shared-secret" value="passphrase"/>
++  <policy domain="coder" rights="none" pattern="MVG" />
++  <policy domain="delegate" rights="none" pattern="HTTPS" />
++  <policy domain="path" rights="none" pattern="@*" />
+   <policy domain="cache" name="shared-secret" value="passphrase" stealth="true"/>
 +  <policy domain="coder" rights="none" pattern="EPHEMERAL" />
 +  <policy domain="coder" rights="none" pattern="HTTPS" />
 +  <policy domain="coder" rights="none" pattern="MSL" />

component/inkscape/buildout.cfg

@@ -14,6 +14,7 @@ extends =
   ../libsigc/buildout.cfg
   ../libxml2/buildout.cfg
   ../libxslt/buildout.cfg
+  ../patch/buildout.cfg
   ../perl/buildout.cfg
   ../pkgconfig/buildout.cfg
   ../popt/buildout.cfg
@@ -35,6 +36,11 @@ environment =
 recipe = slapos.recipe.cmmi
 url = https://inkscape.org/en/gallery/item/3860/inkscape-0.91.tar.bz2
 md5sum = 278dfa4514adcde23546370ec2c84581
+patch-options = -p0
+# based on
+# http://bazaar.launchpad.net/~inkscape.dev/inkscape/trunk/revision/15017
+patches =
+  ${:_profile_base_location_}/inkscape-0.91_gcc6.patch#ce3ddbb90f7e5e81fd322469194b6c3c
 pkg_config_depends = ${gtkmm:location}/lib/pkgconfig:${gtkmm:pkg_config_depends}:${gsl:location}/lib/pkgconfig:${popt:location}/lib/pkgconfig:${garbage-collector:location}/lib/pkgconfig:${libxslt:location}/lib/pkgconfig
 configure-options =
   --disable-static
@@ -45,7 +51,7 @@ configure-options =
   --disable-cdr
   --without-gnome-vfs
 environment =
-  PATH=${freetype:location}/bin:${gdk-pixbuf:location}/bin:${gettext:location}/bin:${glib:location}/bin:${intltool:location}/bin:${libxml2:location}/bin:${pkgconfig:location}/bin:${pango:location}/bin:${perl:location}/bin:%(PATH)s
+  PATH=${freetype:location}/bin:${gdk-pixbuf:location}/bin:${gettext:location}/bin:${glib:location}/bin:${intltool:location}/bin:${libxml2:location}/bin:${pango:location}/bin:${patch:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${:pkg_config_depends}
   CPPFLAGS=-I${boost-lib:location}/include -I${cairo:location}/include -I${garbage-collector:location}/include -I${libpng:location}/include -I${popt:location}/include  -I${zlib:location}/include
 # rpath seems not taken from pkgconfig...

component/inkscape/inkscape-0.91_gcc6.patch

+--- src/ui/dialog/layer-properties.cpp	2014-12-21 21:58:32 +0000
++++ src/ui/dialog/layer-properties.cpp	2016-07-18 17:19:25 +0000
+@@ -146,7 +146,7 @@
+     destroy_();
+     Glib::signal_idle().connect(
+         sigc::bind_return(
+-            sigc::bind(sigc::ptr_fun(&::operator delete), this),
++            sigc::bind(sigc::ptr_fun<void*, void>(&::operator delete), this),
+             false 
+         )
+     );

component/kumo/buildout.cfg

 [buildout]
 extends =
+  ../gcc/buildout.cfg
   ../tokyocabinet/buildout.cfg
   ../messagepack/buildout.cfg
   ../openssl/buildout.cfg
@@ -24,5 +25,6 @@ configure-options =
 
 environment =
   CPPFLAGS=-I${zlib:location}/include -I${openssl:location}/include
-  LDFLAGS=-L${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${tokyocabinet:location}/lib -Wl,-rpath=${messagepack:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib
-  PATH=${patch:location}/bin:%(PATH)s
+  LD_LIBRARY_PATH=${gcc-fortran:location}/lib:${gcc-fortran:location}/lib64
+  LDFLAGS=-Wl,-rpath=${gcc-fortran:location}/lib -Wl,-rpath=${gcc-fortran:location}/lib64 -L${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${tokyocabinet:location}/lib -Wl,-rpath=${messagepack:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib
+  PATH=${gcc-fortran:location}/bin:${patch:location}/bin:%(PATH)s

component/mariadb/buildout.cfg

@@ -22,9 +22,9 @@ parts =
 
 [mariadb]
 recipe = slapos.recipe.cmmi
-version = 10.1.14
+version = 10.1.17
 url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http:/ftp.osuosl.org/pub/mariadb/?serve
-md5sum = 294925531e0fd2f0461e3894496a5adc
+md5sum = 036aca95257cb2948dd100605ec6d5a1
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 patch-options = -p0
 patches =
@@ -70,8 +70,8 @@ post-install =
 # mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
 # http://mroonga.github.com/
 recipe = slapos.recipe.cmmi
-url = http://packages.groonga.org/source/mroonga/mroonga-6.02.tar.gz
-md5sum = a7c6320e273eaa407860cac11970dae8
+url = http://packages.groonga.org/source/mroonga/mroonga-6.08.tar.gz
+md5sum = 10c13aa9f7e520d93799be893aa44d3c
 pre-configure =
   mkdir fake_mariadb_source &&
   ln -s ${mariadb:location}/include/mysql/private fake_mariadb_source/sql

component/postgresql/buildout.cfg

@@ -35,5 +35,5 @@ environment =
 
 [postgresql92]
 <= postgresql-common
-url = http://ftp.postgresql.org/pub/source/v9.2.17/postgresql-9.2.17.tar.bz2
-md5sum = a75d4a82eae1edda04eda2e60656e74c
+url = http://ftp.postgresql.org/pub/source/v9.2.18/postgresql-9.2.18.tar.bz2
+md5sum = fd175eb5f29557c6ef2eeaf340330f9a

component/tar/buildout.cfg

 [buildout]
 extends =
-  ../patch/buildout.cfg
+  ../xz-utils/buildout.cfg
 
 parts = tar
 
 [tar]
-patch-options = -p1
-patches =
-  ${:_profile_base_location_}/tar-drop.gets.patch#9352820566aa3534a04bd269c9f89f48
-
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/tar/tar-1.26.tar.gz
-md5sum = 00d1e769c6af702c542cca54b728920d
+url = http://ftp.gnu.org/gnu/tar/tar-1.29.tar.xz
+md5sum = a1802fec550baaeecff6c381629653ef
 environment =
   FORCE_UNSAFE_CONFIGURE=1
-  PATH=${patch:location}/bin:%(PATH)s
+  PATH=${xz-utils:location}/bin:%(PATH)s

component/tar/tar-drop.gets.patch

-diff -ur tar-1.26.orig/gnu/stdio.in.h tar-1.26/gnu/stdio.in.h
---- tar-1.26.orig/gnu/stdio.in.h	2011-03-12 10:14:33.000000000 +0100
-+++ tar-1.26/gnu/stdio.in.h	2012-08-24 15:35:22.299190847 +0200
-@@ -164,7 +164,10 @@
-    so any use of gets warrants an unconditional warning.  Assume it is
-    always declared, since it is required by C89.  */
- #undef gets
-+#if defined(__GLIBC__) && !defined(__UCLIBC__) && !__GLIBC_PREREQ(2, 16)
- _GL_WARN_ON_USE (gets, "gets is a security hole - use fgets instead");
-+#endif
-+
- 
- #if @GNULIB_FOPEN@
- # if @REPLACE_FOPEN@

component/wget/buildout.cfg

@@ -11,8 +11,8 @@ parts =
 
 [wget]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/wget/wget-1.16.3.tar.xz
-md5sum = d2e4455781a70140ae83b54ca594ce21
+url = http://ftp.gnu.org/gnu/wget/wget-1.18.tar.xz
+md5sum = af9ca95a4bb8ac4a9bf10aeae66fa5ec
 configure-options =
   --enable-ipv6
   --enable-opie

software/agent/software.cfg

@@ -43,34 +43,34 @@ eggs =
 [versions]
 apache-libcloud = 0.18.0
 ecdsa = 0.13
-erp5.util = 0.4.44
+erp5.util = 0.4.45
 gitdb = 0.6.4
 pycrypto = 2.6.1
 slapos.recipe.download = 1.0
 slapos.recipe.template = 2.8
-slapos.toolbox = 0.56
+slapos.toolbox = 0.58
 smmap = 0.9.0
 
 # Required by:
-# slapos.toolbox==0.56
-GitPython = 2.0.6
+# slapos.toolbox = 0.58
+GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 feedparser = 5.2.1
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 lockfile = 0.12.2
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 paramiko = 2.0.1
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 rpdb = 0.1.5

software/apache-frontend/common.cfg

@@ -167,7 +167,7 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
 recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/templates/trafficserver/${:filename}
-md5sum = 65afeef0229430ad8a6fbc57298b787b
+md5sum = 59287d2de3d948b135619edd211a5e84
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 filename = records.config.jinja2
 download-only = true

software/apache-frontend/software.cfg

@@ -11,29 +11,29 @@ plone.recipe.command = 1.1
 pycrypto = 2.6.1
 rdiff-backup = 1.0.5
 slapos.recipe.template = 2.8
-slapos.toolbox = 0.56
+slapos.toolbox = 0.58
 smmap = 0.9.0
 
 # Required by:
-# slapos.toolbox==0.56
-GitPython = 2.0.6
+# slapos.toolbox = 0.58
+GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 feedparser = 5.2.1
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 lockfile = 0.12.2
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 paramiko = 2.0.1
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 rpdb = 0.1.5

software/apache-frontend/templates/trafficserver/records.config.jinja2

@@ -184,10 +184,10 @@ CONFIG proxy.config.http.background_fill_completed_threshold FLOAT 0.5
    ##################################
    # origin server connect attempts #
    ##################################
-CONFIG proxy.config.http.connect_attempts_max_retries INT 6
+CONFIG proxy.config.http.connect_attempts_max_retries INT 1
 CONFIG proxy.config.http.connect_attempts_max_retries_dead_server INT 3
 CONFIG proxy.config.http.connect_attempts_rr_retries INT 3
-CONFIG proxy.config.http.connect_attempts_timeout INT 30
+CONFIG proxy.config.http.connect_attempts_timeout INT 160
 CONFIG proxy.config.http.post_connect_attempts_timeout INT 1800
 CONFIG proxy.config.http.down_server.cache_time INT 300
 CONFIG proxy.config.http.down_server.abort_threshold INT 10

software/cdn-me/software.cfg

@@ -38,6 +38,6 @@ eggs =
 [versions]
 cns.recipe.symlink = 0.2.3
 collective.recipe.environment = 0.2.0
-erp5.util = 0.4.44
+erp5.util = 0.4.45
 plone.recipe.command = 1.1
 slapos.recipe.template = 2.8

software/erp5/instance-erp5-input-schema.json

@@ -38,7 +38,7 @@
     "developer-list": {
       "description": "List of logins which should get the Developper role (required to modify portal_components' content), defaulting to inituser-login's value",
       "items": {
-        "pattern": "/^\\S+$/",
+        "pattern": "^\\S+$",
         "type": "string"
       },
       "uniqueItems": true,
@@ -143,28 +143,28 @@
     "kumofs": {
       "description": "Persistent memcached service",
       "additionalProperties": {
-        "$ref": "./instance-kumofs-schema.json#properties"
+        "$ref": "./instance-kumofs-schema.json#/properties"
       },
       "type": "object"
     },
     "memcached": {
       "description": "Volatile memcached service",
       "additionalProperties": {
-        "$ref": "./instance-kumofs-schema.json#properties"
+        "$ref": "./instance-kumofs-schema.json#/properties"
       },
       "type": "object"
     },
     "cloudooo": {
       "description": "Format conversion service",
       "additionalProperties": {
-        "$ref": "./instance-cloudooo-schema.json#properties"
+        "$ref": "./instance-cloudooo-schema.json#/properties"
       },
       "type": "object"
     },
     "mariadb": {
       "description": "Relational database service",
       "additionalProperties": {
-        "$ref": "./instance-mariadb-schema.json#properties"
+        "$ref": "./instance-mariadb-schema.json#/properties"
       },
       "type": "object"
     },

software/erp5/instance-erp5-output-schema.json

 {
   "$schema": "http://json-schema.org/draft-04/schema#",
   "description": "Values returned by ERP5 instantiation",
+  "additionalProperties": false,
   "properties": {
     "hosts-dict": {
       "description": "Hosts mapping, including auto-generated entries",
       "patternProperties": {
         ".*": {
-          "description": "IP or domain names current entry resolves to",
+          "description": "IP current entry resolves to",
           "type": "string"
         }
       },
@@ -24,31 +25,59 @@
       "description": "Initial user password",
       "type": "string"
     },
-    "kumofs-url": {
+    "deadlock-debugger-password": {
+      "description": "Deadlock debugger password",
+      "type": "string"
+    },
+    "memcached-persistent-url": {
       "description": "Persistent memcached access information",
+      "pattern": "^memcached://",
       "type": "string"
     },
-    "memcached-url": {
+    "memcached-volatile-url": {
       "description": "Volatile memcached access information",
+      "pattern": "^memcached://",
       "type": "string"
     },
     "cloudooo-url": {
-      "description": "Conversion service access information",
+      "description": "Conversion service access information - DEPRECATED",
+      "pattern": "^cloudooo://",
       "type": "string"
     },
-    "mariadb-url": {
+    "mariadb-database-list": {
       "description": "Relational database access information",
-      "type": "string"
+      "items": {
+        "pattern": "^mysql://",
+        "type": "string"
+      },
+      "uniqueItems": true,
+      "type": "array"
+    },
+    "mariadb-test-database-list": {
+      "description": "Relational database access information",
+      "items": {
+	"pattern": "^mysql://",
+        "type": "string"
+      },
+      "uniqueItems": true,
+      "type": "array"
+    },
+    "neo-masters": {
+      "$ref": "../neoppod/instance-neo-output-schema.json#/properties/masters"
+    },
+    "neo-admins": {
+      "$ref": "../neoppod/instance-neo-output-schema.json#/properties/admins"
     },
     "jupyter-url": {
       "description": "Jupyter notebook web UI access information",
-      "type": "string",
-      "optional": true
+      "pattern": "^https://",
+      "type": "string"
     }
   },
   "patternProperties": {
     "family-.*": {
       "description": "Zope family access information",
+      "pattern": "^https://",
       "type": "string"
     }
   },

software/erp5/software.cfg.json

@@ -5,14 +5,14 @@
   "software-type": {
     "default": {
       "title": "Default",
-      "description": "Default deployment of ERP5 with auto-create instance.",
+      "description": "No automated database modification (ERP5Site is not automatically created).",
       "request": "instance-erp5-input-schema.json",
       "response": "instance-erp5-output-schema.json",
       "index": 0
     },
     "create-erp5-site": {
       "title": "Create ERP5 Site", 
-      "description": "Default deployment of ERP5 with automatic creation of the instance.",
+      "description": "Automated ERP5Site creation on instanciation when ZODB is found empty.",
       "request": "instance-erp5-input-schema.json",
       "response": "instance-erp5-output-schema.json",
       "index": 1

software/html5as/software.cfg

@@ -79,7 +79,7 @@ plone.recipe.command = 1.1
 
 # Required by:
 # slapos.toolbox==0.40.2
-GitPython = 2.0.6
+GitPython = 2.0.8
 
 # Required by:
 # slapos.toolbox==0.40.2

software/jstestnode/software.cfg

@@ -111,6 +111,6 @@ output = ${buildout:directory}/runTestSuite.in
 mode = 0644
 
 [versions]
-erp5.util = 0.4.44
+erp5.util = 0.4.45
 slapos.recipe.template = 2.9
 selenium = 2.53.1

software/kvm/software.cfg

@@ -15,12 +15,12 @@ plone.recipe.command = 1.1
 pycrypto = 2.6.1
 slapos.recipe.template = 2.7
 smmap = 0.9.0
-erp5.util = 0.4.44
+erp5.util = 0.4.45
 pycurl = 7.19.5.1
 
 # Required by:
 # slapos.toolbox==0.48
-GitPython = 2.0.6
+GitPython = 2.0.8
 
 # Required by:
 # slapos.toolbox==0.48

software/neoppod/instance-neo-storage-mysql.cfg.in

@@ -14,6 +14,7 @@ mysql-base-directory = {{ mariadb_location }}
 [my-cnf-parameters]
 socket = ${directory:var_run}/mariadb.sock
 data-directory = ${mysqld:data-directory}
+tmp-directory = ${directory:tmp}
 pid-file = ${directory:var_run}/mariadb.pid
 error-log = ${directory:log}/mariadb_error.log
 slow-query-log = ${directory:log}/mariadb_slowquery.log

software/neoppod/my.cnf.in

@@ -5,6 +5,7 @@
 skip_networking
 socket = {{ socket }}
 datadir = {{ parameter_dict['data-directory'] }}
+tmpdir = {{ parameter_dict['tmp-directory'] }}
 pid_file = {{ parameter_dict['pid-file'] }}
 log_error = {{ parameter_dict['error-log'] }}
 slow_query_log

software/neoppod/software-common.cfg

@@ -45,6 +45,7 @@ eggs = neoppod[admin, ctl, master, storage-importer, storage-mysqldb, tests]
 patch-binary = ${patch:location}/bin/patch
 ZODB3-patches =
   ${:_profile_base_location_}/../../component/egg-patch/ZODB3-3.10.5.patch#c5fe331b1e3a930446f93ab4f6e97c6e
+  ${:_profile_base_location_}/../../component/egg-patch/ZODB3-persistent-ghostify-slots.patch#3a66e9c018d7269bd522d5b0a746f510
 ZODB3-patch-options = -p1
 
 [slapos-deps-eggs]
@@ -95,27 +96,28 @@ md5sum = 82f3f76f54ee9db355966a7ada61f56e
 
 [instance-neo-storage-mysql]
 <= download-base-neo
-md5sum = 4572f8d3f92f1b1639600d0eb7119ab5
+md5sum = cd2a978a09c5686205592923866f6584
 
 [template-neo-my-cnf]
 <= download-base-neo
 url = ${:_profile_base_location_}/my.cnf.in
-md5sum = febd3ed58043ce1367b86cf6e4e69700
+md5sum = 81ab5e842ecf8385b12d735585497cc8
 
 [versions]
 slapos.recipe.template = 2.9
 # patched egg
-ZODB3 = 3.10.5+SlapOSPatched001
-# Required by slapos.toolbox==0.56
-slapos.toolbox = 0.56
+ZODB3 = 3.10.5+SlapOSPatched002
+# Required by slapos.toolbox = 0.58
+slapos.toolbox = 0.58
 apache-libcloud = 0.20.1
 atomize = 0.2.0
 ecdsa = 0.13
 feedparser = 5.2.1
-GitPython = 2.0.6
+GitPython = 2.0.8
 gitdb = 0.6.4
 lockfile = 0.12.2
 mysqlclient = 1.3.7
-paramiko = 2.0.1
+paramiko = 2.0.2
+passlib = 1.6.5
 pycrypto = 2.6.1
 smmap = 0.9.0

software/neoppod/software.cfg

@@ -38,8 +38,8 @@ ZODB3-patches +=
   ${neoppod-repository:location}/ZODB3.patch
 
 [versions]
-ZODB3 = 3.10.5+SlapOSPatched002
-erp5.util = 0.4.44
+ZODB3 = 3.10.5+SlapOSPatched003
+erp5.util = 0.4.45
 # To match ERP5
 transaction = 1.1.1
 ZConfig = 2.9.3

software/re6stnet/software.cfg

@@ -112,15 +112,15 @@ gitdb = 0.6.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
 slapos.recipe.template = 2.7
-slapos.toolbox = 0.56
+slapos.toolbox = 0.58
 smmap = 0.9.0
 
 # Required by:
-# slapos.toolbox==0.56
-GitPython = 2.0.6
+# slapos.toolbox = 0.58
+GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 atomize = 0.2.0
 
 # Required by:
@@ -128,11 +128,11 @@ atomize = 0.2.0
 backports.ssl-match-hostname = 3.4.0.2
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 feedparser = 5.1.3
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 lockfile = 0.12.2
 
 # Required by:
@@ -140,10 +140,10 @@ lockfile = 0.12.2
 miniupnpc = 1.9
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 paramiko = 2.0.1
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 rpdb = 0.1.5
 

software/slapos-master/software.cfg

 [buildout]
 extends =
-  ../../stack/erp5/buildout.cfg
+  ../../software/erp5/sofware.cfg
 
 parts +=
   vifib-fix-products-paths
@@ -10,29 +10,17 @@ parts +=
 # Used to generate bt5lists.
 list = ${erp5:location}/bt5 ${erp5:location}/product/ERP5/bootstrap ${vifib:location}/master/bt5
 
-[genbt5list]
-recipe = plone.recipe.command
-stop-on-error = true
-genbt5list = ${erp5:location}/product/ERP5/bin/genbt5list
-command =
-  ${buildout:executable} ${:genbt5list} ${local-bt5-repository:list}
-update-command = ${:command}
-
 [erp5_repository_list]
 repository_id_list = erp5 vifib/master
 
 [erp5]
-recipe = slapos.recipe.build:gitclone
-repository = https://lab.nexedi.com/nexedi/erp5.git
 branch = erp5-vifib
-git-executable = ${git:location}/bin/git
 revision = b22077622b356aade6981957ddc6ff271c228bbb 
 
 [vifib]
-recipe = slapos.recipe.build:gitclone
-branch = master
+<= erp5
 repository = https://lab.nexedi.com/nexedi/slapos.core.git
-git-executable = ${git:location}/bin/git
+branch = master
 revision = b3b9da6158f0dd3956e72a804c57f18b6f966fc1 
 
 [vifib-fix-products-paths]

software/slaprunner/common.cfg

@@ -101,7 +101,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/instance-resilient-test.cfg.jinja2
 download-only = true
-md5sum = fb8c45e5c35548331fb06c4633ec592a
+md5sum = c31de887459dfb45fe1213f2837cadcb
 filename = instance-resilient-test.cfg.jinja2
 mode = 0644
 
@@ -110,7 +110,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/nginx_conf.in
 download-only = true
-md5sum = 94d83ef3eb89c2d75c8c079ab12b4518
+md5sum = 2b06f7eb9a1d45d250d4b92a944db925
 filename = nginx_conf.in
 mode = 0644
 

software/slaprunner/instance-resilient-test.cfg.jinja2

@@ -61,6 +61,6 @@ config-ignore-known-hosts-file = true
 #config-frontend-domain = google.com
 # XXX Hack to deploy Root Instance on the same computer as the type-test Instance
 sla-computer_guid = ${slap-connection:computer-id}
-return = backend_url
+return = backend-url
 
 [slap-parameter]

software/slaprunner/nginx_conf.in

@@ -35,21 +35,6 @@ http {
         scgi_temp_path {{ param_tempdir['scgi_temp_path'] }};
 
         location / {
-            # When no .htpasswd exist, redirect the user to account creation page
-            if ( !-f {{ param_nginx_frontend['etc_dir'] }}/.htpasswd ) {
-                # redirect URL is different wether nginx is accessed directly or behind apache.
-                # nginx does not support nested if or multiple conditions, so we use this well known hack.
-                set $test no_htpasswd;
-            }
-            if ( $host = [{{ param_nginx_frontend['global-ip'] }}] ) {
-                set $test "${test}_backend_access";
-            }
-            if ( $test = no_htpasswd) {
-                return 301 $scheme://$host/setAccount ;
-            }
-            if ( $test = no_htpasswd_backend_access) {
-                return 301 /setAccount ;
-            }
             auth_basic "Restricted";
             auth_basic_user_file {{ param_nginx_frontend['etc_dir'] }}/.htpasswd;
             proxy_redirect off;

software/slaprunner/software.cfg

@@ -13,36 +13,36 @@ apache-libcloud = 0.20.1
 cns.recipe.symlink = 0.2.3
 collective.recipe.environment = 0.2.0
 ecdsa = 0.13
-erp5.util = 0.4.44
+erp5.util = 0.4.45
 futures = 3.0.5
 gitdb = 0.6.4
 gunicorn = 19.5.0
 prettytable = 0.7.2
 pycrypto = 2.6.1
 slapos.recipe.template = 2.9
-slapos.toolbox = 0.56
+slapos.toolbox = 0.58
 smmap = 0.9.0
 
 # Required by:
-# slapos.toolbox==0.56
-GitPython = 2.0.6
+# slapos.toolbox = 0.58
+GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 feedparser = 5.2.1
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 lockfile = 0.12.2
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 paramiko = 2.0.1
 
 # Required by:
-# slapos.toolbox==0.55
+# slapos.toolbox = 0.58
 passlib = 1.6.5
\ No newline at end of file

software/wendelin/software.cfg

@@ -52,6 +52,5 @@ revision = 9c1384dc94905f05b8bac807c59f38990668b648
 revision = 4dfdf05a89445a9286dbdf364d495bedf10f9394 
 
 [versions]
-scipy = 0.15.1
-msgpack-python = 0.4.6
+msgpack-python = 0.4.8
 wendelin.core = 0.7

stack/boinc/buildout.cfg

@@ -75,7 +75,7 @@ smmap = 0.8.2
 
 # Required by:
 # slapos.toolbox==0.40.2
-GitPython = 2.0.6
+GitPython = 2.0.8
 
 # Required by:
 # slapos.toolbox==0.40.2

stack/cloudooo.cfg

@@ -102,4 +102,4 @@ PasteDeploy = 1.5.2
 
 # Required by:
 # cloudooo==1.2.5.dev0
-erp5.util = 0.4.44
+erp5.util = 0.4.45

stack/erp5/buildout.cfg

@@ -156,7 +156,7 @@ mode = 755
 [template-mariadb]
 <= download-base
 filename = instance-mariadb.cfg.in
-md5sum = ede2481d6ce60a335bde920d8cec5318
+md5sum = 9312af2f9d9faf06d2f26f073ad60180
 link-binary =
   ${coreutils:location}/bin/basename
   ${coreutils:location}/bin/cat
@@ -192,7 +192,7 @@ md5sum = bc6048b85b410693e60e5a77399dd1b7
 [template-my-cnf]
 <= download-base
 filename = my.cnf.in
-md5sum = ac5c87991d95907f62bf2129ecfd42c4
+md5sum = d50920c942b8ee98402f8551fef38383
 
 [template-mariadb-initial-setup]
 <= download-base
@@ -622,8 +622,8 @@ scripts +=
 Acquisition = 2.13.9+SlapOSPatched001
 Products.DCWorkflow = 2.2.4+SlapOSPatched001
 pysvn = 1.7.10+SlapOSPatched002
-python-ldap = 2.4.25+SlapOSPatched001
-python-magic = 0.4.11+SlapOSPatched001
+python-ldap = 2.4.27+SlapOSPatched001
+python-magic = 0.4.12+SlapOSPatched001
 
 # specify dev version to be sure that an old released version is not used
 cloudooo = 1.2.5-dev
@@ -632,7 +632,7 @@ cloudooo = 1.2.5-dev
 PasteDeploy = 1.5.2
 Pygments = 2.1.3
 argparse = 1.4.0
-coverage = 4.0.3
+coverage = 4.2
 zope.dottedname = 4.1.0
 
 # test_UserManagerInterfaces in testERP5Security fails with 1.10.0.
@@ -664,14 +664,14 @@ zope.app.publication = 3.14.0
 zope.app.testing = 3.8.1
 
 # Pinned versions
-Pillow = 3.2.0
+Pillow = 3.3.1
 Products.CMFActionIcons = 2.1.3
 Products.DCWorkflowGraph = 0.4.1
 # Products.ExternalEditor 2.0.0's dtml is not based on Zope2 OFS's one.
 Products.ExternalEditor = 1.1.1
 Products.GenericSetup = 1.8.3
 Products.LongRequestLogger = 2.0.0
-Products.MimetypesRegistry = 2.0.9
+Products.MimetypesRegistry = 2.0.10
 Products.PluginRegistry = 1.4
 Products.TIDStorage = 5.4.9
 PyPDF2 = 1.26.0
@@ -696,19 +696,20 @@ httplib2 = 0.9.2
 huBarcode = 1.0.0
 interval = 1.0.0
 ipdb = 0.10.1
-ipykernel = 4.3.1
-ipython = 4.2.1
+ipykernel = 4.4.1
+ipython = 5.1.0
 ipywidgets = 5.1.5
 logilab-common = 1.2.2
-matplotlib = 1.5.1
+matplotlib = 1.5.2
 mistune = 0.7.3
-notebook = 4.2.1
+notebook = 4.2.2
 numpy = 1.11.1
 objgraph = 3.0.0
 pandas = 0.18.1
 ply = 3.8
 polib = 1.0.7
 pprofile = 1.9.1
+prompt-toolkit = 1.0.3
 ptyprocess = 0.5.1
 pycountry = 1.20
 pyflakes = 1.2.3
@@ -716,12 +717,12 @@ pyflakes = 1.2.3
 pylint = 1.4.4
 python-memcached = 1.58
 pytracemalloc = 1.2
-pyzmq = 15.2.0
+pyzmq = 15.4.0
 qrcode = 5.3
 restkit = 4.2.2
 rtjp-eventlet = 0.3.2
 scikit-learn = 0.17.1
-scipy = 0.17.1
+scipy = 0.18.0
 simplegeneric = 0.8.1
 socketpool = 0.5.3
 spyne = 2.12.11
@@ -729,8 +730,9 @@ suds = 0.4
 terminado = 0.6
 threadframe = 0.2
 timerserver = 2.0.2
-tornado = 4.3
-urlnorm = 1.1.2
+tornado = 4.4.1
+traitlets = 4.2.2
+urlnorm = 1.1.4
 uuid = 1.30
 validictory = 1.0.2
 widgetsnbextension = 1.2.3
@@ -742,23 +744,23 @@ xupdate-processor = 0.4
 Products.ZSQLMethods = 2.13.4
 
 # Required by:
-# ipython==4.2.1
+# ipython==5.1.0
 backports.shutil-get-terminal-size = 1.0.0
 
 # Required by:
-# tornado==4.3
+# tornado==4.4.1
 backports.ssl-match-hostname = 3.5.0.1
 
 # Required by:
-# tornado==4.3
-certifi = 2016.2.28
+# tornado==4.4.1
+certifi = 2016.8.8
 
 # Required by:
 # matplotlib==1.5.1
 cycler = 0.10.0
 
 # Required by:
-# ipython==4.2.1
+# ipython==5.1.0
 # traitlets==4.2.2
 decorator = 4.0.10
 
@@ -767,32 +769,32 @@ decorator = 4.0.10
 fpconst = 0.7.2
 
 # Required by:
-# nbformat==4.0.1
-# notebook==4.2.1
+# nbformat==4.1.0
+# notebook==4.2.2
 # traitlets==4.2.2
 ipython-genutils = 0.1.0
 
 # Required by:
-# notebook==4.2.1
+# notebook==4.2.2
 # nbconvert 4.2.0 depends on entrypoints egg that is not available as tar/zip source.
 nbconvert = 4.1.0
 
 # Required by:
 # nbconvert==4.1.0
-# notebook==4.2.1
-nbformat = 4.0.1
+# notebook==4.2.2
+nbformat = 4.1.0
 
 # Required by:
-# pickleshare==0.7.2
+# ipython==5.1.0
 pathlib2 = 2.1.0
 
 # Required by:
-# ipython==4.2.1
-pexpect = 4.1.0
+# ipython==5.1.0
+pexpect = 4.2.1
 
 # Required by:
-# ipython==4.2.1
-pickleshare = 0.7.2
+# ipython==5.1.0
+pickleshare = 0.7.4
 
 # Required by:
 # matplotlib==1.5.1
@@ -800,13 +802,12 @@ pickleshare = 0.7.2
 python-dateutil = 2.5.3
 
 # Required by:
-# tornado==4.3
+# tornado==4.4.1
 singledispatch = 3.4.0.3
 
 # Required by:
-# ipython==4.2.1
-# notebook==4.2.1
-traitlets = 4.2.2
+# prompt-toolkit==1.0.3
+wcwidth = 0.1.7
 
 # Required by:
 # zope.app.testing==3.8.1

stack/erp5/instance-mariadb.cfg.in

@@ -102,6 +102,7 @@ ip = {{ ip }}
 port = {{ port }}
 socket = ${directory:run}/mariadb.sock
 data-directory = ${directory:mariadb-data}
+tmp-directory = ${directory:tmp}
 pid-file = ${directory:run}/mariadb.pid
 error-log = ${directory:log}/mariadb_error.log
 slow-query-log = ${directory:log}/mariadb_slowquery.log
@@ -204,6 +205,7 @@ etc = ${buildout:directory}/etc
 services = ${:etc}/run
 promise = ${:etc}/promise
 srv = ${buildout:directory}/srv
+tmp = ${buildout:directory}/tmp
 backup = ${:srv}/backup
 mariadb-backup-full = ${:backup}/mariadb-full
 mariadb-backup-incremental = ${:backup}/mariadb-incremental

stack/erp5/my.cnf.in

@@ -19,6 +19,7 @@ skip_networking
 {% endif -%}
 socket = {{ socket }}
 datadir = {{ parameter_dict['data-directory'] }}
+tmpdir = {{ parameter_dict['tmp-directory'] }}
 pid_file = {{ parameter_dict['pid-file'] }}
 log_error = {{ parameter_dict['error-log'] }}
 slow_query_log

stack/lamp.cfg

@@ -50,7 +50,7 @@ smmap = 0.8.2
 
 # Required by:
 # slapos.toolbox==0.40.2
-GitPython = 2.0.6
+GitPython = 2.0.8
 
 # Required by:
 # slapos.toolbox==0.40.2

stack/lamp/buildout.cfg

@@ -194,7 +194,7 @@ smmap = 0.8.2
 
 # Required by:
 # slapos.toolbox==0.40.2
-GitPython = 2.0.6
+GitPython = 2.0.8
 
 # Required by:
 # slapos.toolbox==0.40.2

stack/lapp/buildout.cfg

@@ -191,7 +191,7 @@ smmap = 0.8.2
 
 # Required by:
 # slapos.toolbox==0.40.2
-GitPython = 2.0.6
+GitPython = 2.0.8
 
 # Required by:
 # slapos.toolbox==0.40.2

stack/monitor/buildout.cfg

 [buildout]
-# XXX THIS STACK IS A KIND OF FORK OF `stack/monitor`. THIS ONE WAS
-#     CREATED AS A REDESIGNED ONE TO REMOVE UNWANTED FEATURES AND
-#     TO GO FURTHER TO THE GOOD DESIGN DIRECTION. SEE THE README FOR
-#     MORE INFORMATION.
 
 extends =
   ../../component/apache/buildout.cfg
@@ -43,6 +39,7 @@ eggs =
   plone.recipe.command
   collective.recipe.template
   cns.recipe.symlink
+  slapos.toolbox
 
 [extra-eggs]
 <= monitor-eggs
@@ -90,17 +87,19 @@ recipe = slapos.recipe.template:jinja2
 filename = template-monitor.cfg
 template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in
 rendered = ${buildout:directory}/template-monitor.cfg
-md5sum = 7ac78495de73cadafea87f97428e487f
+md5sum = 84998b1ca3c29445dca70b495515c35b
 context =
     key apache_location apache:location
     key gzip_location gzip:location
-    raw monitor_bin ${monitor2-bin:location}/${monitor2-bin:filename}
-    raw monitor_collect ${monitor-collect:location}/${monitor-collect:filename}
+    raw monitor_bin ${buildout:directory}/bin/monitor.bootstrap
+    raw monitor_collect ${buildout:directory}/bin/monitor.collect
+    raw monitor_runpromise ${buildout:directory}/bin/monitor.runpromise
+    raw monitor_genstatus ${buildout:directory}/bin/monitor.genstatus
+    raw monitor_genrss ${buildout:directory}/bin/monitor.genrss
+    raw monitor_configwrite ${buildout:directory}/bin/monitor.configwrite
     raw monitor_conf_template ${monitor-conf:location}/${monitor-conf:filename}
-    raw monitor_document_edit ${monitor-document-edit:location}/${monitor-document-edit:filename}
     raw monitor_https_cors ${monitor-httpd-cors:location}/${monitor-httpd-cors:filename}
     raw monitor_instance_info ${monitor-instance-info:location}/${monitor-instance-info:filename}
-    raw monitor_globalstate ${monitor-globalstate:location}/${monitor-globalstate:filename}
     raw curl_executable_location ${curl:location}/bin/curl
     raw dash_executable_location ${dash:location}/bin/dash
     raw dcron_executable_location ${dcron:location}/sbin/crond
@@ -109,48 +108,12 @@ context =
     raw openssl_executable_location ${openssl:location}/bin/openssl
     raw python_executable ${buildout:executable}
     raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter}
-    raw promise_executor_py ${run-promise-py:rendered}
     raw template_wrapper ${monitor-template-wrapper:location}/${monitor-template-wrapper:filename}
-    raw status2rss_executable_path ${status2rss-executable:location}/${status2rss-executable:filename}
 depends = 
   ${monitor-eggs:eggs}
 
-
-[monitor2-bin]
-<= monitor-template-script
-filename = monitor.py
-md5sum = 5525e7445dab16fd03f4eeccf069b74b
-
-[run-promise-py]
-recipe = slapos.recipe.template:jinja2
-template = ${:_profile_base_location_}/scripts/run-promise.py
-rendered = ${buildout:parts-directory}/monitor-scripts/run-promise.py
-md5sum = 97148dfbb730cc4f55ed54513ce823e0
-mode = 0755
-context =
-  raw python ${buildout:directory}/bin/${extra-eggs:interpreter}
-
-[status2rss-executable]
-<= monitor-template-script
-filename = status2rss.py
-md5sum = 88e3bf955e1e4eac76a444d50fa4f020
-
-[monitor-globalstate]
-<= monitor-template-script
-filename = globalstate.py
-md5sum = 3377e325baa4ecfcd6eee06945fb69fc
-
-[monitor-collect]
-<= monitor-template-script
-filename = collect.py
-md5sum = 78fbcb56761315bde354fe7914d3c54f
-
-[monitor-document-edit]
-<= monitor-template-script
-filename = monitor-document.py
-md5sum = 399ff4939b55ff74e6d48bec5a495981
-
 [versions]
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
+slapos.toolbox = 0.58
 

stack/monitor/instance-monitor.cfg.jinja2.in

@@ -135,7 +135,7 @@ monitor-hal-json = ${monitor-directory:public}/monitor.hal.json
 service-pid-folder = ${monitor-directory:pids}
 crond-folder = ${logrotate-directory:cron-entries}
 logrotate-folder = ${logrotate:logrotate-entries}
-promise-runner = {{ promise_executor_py }}
+promise-runner = {{ monitor_runpromise }}
 promise-folder-list = 
   ${directory:promises}
   ${directory:monitor-promise}
@@ -159,6 +159,8 @@ collector-db = ${monitor-instance-parameter:collector-db}
 collect-script = {{ monitor_collect }}
 python = {{ python_with_eggs }}
 
+promise-output-file = ${directory:monitor}/monitor-bootstrap-status
+
 [monitor-conf]
 recipe = slapos.recipe.template:jinja2
 template = {{ monitor_conf_template }}
@@ -272,7 +274,7 @@ command = kill -USR1 $(cat ${monitor-httpd-conf-parameter:pid-file})
 [monitor-status2rss-wrapper]
 recipe = slapos.cookbook:wrapper
 # XXX - hard-coded Urls
-command-line = {{ python_with_eggs }} {{ status2rss_executable_path }} --output '${monitor-directory:public}/feed' --items_folder '${monitor-directory:public}'  --feed_url '${monitor-conf-parameters:base-url}/public/feed' --public_url '${monitor-conf-parameters:base-url}/share/jio_public/' --private_url '${monitor-conf-parameters:base-url}/share/jio_private/' --instance_name '${monitor-conf-parameters:title}' --hosting_name '${monitor-conf-parameters:root-title}'
+command-line = {{ monitor_genrss }} --output '${monitor-directory:public}/feed' --items_folder '${monitor-directory:public}'  --feed_url '${monitor-conf-parameters:base-url}/public/feed' --public_url '${monitor-conf-parameters:base-url}/share/jio_public/' --private_url '${monitor-conf-parameters:base-url}/share/jio_private/' --instance_name '${monitor-conf-parameters:title}' --hosting_name '${monitor-conf-parameters:root-title}'
 
 wrapper-path = ${directory:bin}/monitor-status2rss.py
 
@@ -285,13 +287,13 @@ command = ${monitor-status2rss-wrapper:wrapper-path}
 
 [monitor-globalstate-wrapper]
 recipe = slapos.cookbook:wrapper
-command-line = {{ python_with_eggs }} {{ monitor_globalstate }} '${monitor-conf:rendered}' '${monitor-instance-info:rendered}'
+command-line = {{ monitor_genstatus }} '${monitor-conf:rendered}' '${monitor-instance-info:rendered}'
 wrapper-path = ${directory:bin}/monitor-globalstate
 
 [monitor-configurator-wrapper]
 recipe = slapos.cookbook:wrapper
 # XXX - hard coded path
-command-line = {{ python_with_eggs }} {{ monitor_document_edit }} --config_folder '${monitor-conf-parameters:private-folder}/config/.jio_documents' --output_cfg_file '${monitor-instance-parameter:configuration-file-path}' --htpasswd_bin '{{ apache_location }}/bin/htpasswd'
+command-line = {{ monitor_configwrite }} --config_folder '${monitor-conf-parameters:private-folder}/config/.jio_documents' --output_cfg_file '${monitor-instance-parameter:configuration-file-path}' --htpasswd_bin '{{ apache_location }}/bin/htpasswd'
 wrapper-path = ${directory:bin}/monitor-configurator
 
 [monitor-globalstate-cron-entry]
@@ -330,7 +332,7 @@ monitor-httpd-ipv6 = ${slap-configuration:ipv6-random}
 monitor-httpd-port = 8196
 # XXX - Set monitor-base-url = ${monitor-httpd-conf-parameter:url} => https://[ipv6]:port
 monitor-base-url = ${monitor-frontend-promise:url}
-# monitor-base-url = ${monitor-httpd-conf-parameter:url}
+#monitor-base-url = ${monitor-httpd-conf-parameter:url}
 root-instance-title = ${slap-configuration:root-instance-title}
 monitor-url-list =
 cors-domains = monitor.app.officejs.com 
@@ -367,6 +369,17 @@ dash_path = {{ dash_executable_location }}
 curl_path = {{ curl_executable_location }}
 check-secure = 1
 
+[monitor-bootstrap-promise]
+recipe = slapos.recipe.template:jinja2
+template = {{ template_wrapper }}
+rendered = ${directory:promises}/monitor-bootstrap-status
+file = ${monitor-conf-parameters:promise-output-file}
+command = if [ ! -f "${:file}" ]; then echo "Monitor bootstrap exited with error." && exit 2; else echo "Bootstrap OK"; fi
+mode = 0700
+context =
+    key content :command
+    raw dash_binary {{ dash_executable_location }}
+
 [monitor-base]
 # create dependencies between required monitor parts
 recipe = plone.recipe.command
@@ -383,6 +396,7 @@ depends =
   ${ca-httpd:wrapper}
   ${monitor-httpd-promise:filename}
   ${monitor-status2rss-cron-entry:name}
+  ${monitor-bootstrap-promise:file}
 
 [monitor-publish]
 monitor-base-url = ${publish:monitor-base-url}

stack/monitor/scripts/globalstate.py

-#!/usr/bin/env python
-
-import sys
-import os
-import glob
-import json
-import ConfigParser
-import time
-from datetime import datetime
-
-def softConfigGet(config, *args, **kwargs):
-  try:
-    return config.get(*args, **kwargs)
-  except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
-    return ""
-
-def generateStatisticsData(stat_file_path, content):
-  # csv document for statictics
-  if not os.path.exists(stat_file_path):
-    with open(stat_file_path, 'w') as fstat:
-      data_dict = {
-        "date": time.time(),
-        "data": ["Date, Success, Error, Warning"]
-      }
-      fstat.write(json.dumps(data_dict))
-
-  current_state = ''
-  if content.has_key('state'):
-    current_state = '%s, %s, %s, %s' % (
-      content['date'],
-      content['state']['success'],
-      content['state']['error'],
-      content['state']['warning'])
-
-  # append to file
-  if current_state:
-    with open (stat_file_path, mode="r+") as fstat:
-      fstat.seek(0,2)
-      position = fstat.tell() -2
-      fstat.seek(position)
-      fstat.write('%s}' % ',"{}"]'.format(current_state))
-
-def main(args_list):
-  monitor_file, instance_file = args_list
-
-  monitor_config = ConfigParser.ConfigParser()
-  monitor_config.read(monitor_file)
-
-  base_folder = monitor_config.get('monitor', 'private-folder')
-  status_folder = monitor_config.get('monitor', 'public-folder')
-  base_url = monitor_config.get('monitor', 'base-url')
-  related_monitor_list = monitor_config.get("monitor", "monitor-url-list").split()
-  statistic_folder = os.path.join(base_folder, 'data', '.jio_documents')
-  parameter_file = os.path.join(base_folder, 'config', '.jio_documents', 'config.json')
-
-  report_date = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
-
-  if not os.path.exists(statistic_folder):
-    try:
-      os.makedirs(statistic_folder)
-    except OSError, e:
-      if e.errno == os.errno.EEXIST and os.path.isdir(statistic_folder):
-        pass
-      else: raise
-
-  # search for all status files
-  file_list = filter(os.path.isfile,
-      glob.glob("%s/*.status.json" % status_folder)
-    )
-  error = warning = success = 0
-  status = 'OK'
-  promise_list = []
-  global_state_file = os.path.join(base_folder, 'monitor.global.json')
-  public_state_file = os.path.join(status_folder, 'monitor.global.json')
-  for file in file_list:
-    try:
-      with open(file, 'r') as temp_file:
-        tmp_json = json.loads(temp_file.read())
-    except ValueError:
-      # bad json file ?
-      continue
-    if tmp_json['status'] == 'ERROR':
-      error  += 1
-    elif tmp_json['status'] == 'OK':
-      success += 1
-    elif tmp_json['status'] == 'WARNING':
-      warning += 1
-    tmp_json['time'] = tmp_json['start-date'].split(' ')[1]
-    promise_list.append(tmp_json)
-
-  if error:
-    status = 'ERROR'
-  elif warning:
-    status = 'WARNING'
-
-  global_state_dict = dict(
-      status=status,
-      state={
-        'error': error,
-        'success': success,
-        'warning': warning,
-      },
-      type='global',
-      date=report_date,
-      _links={"rss_url": {"href": "%s/public/feed" % base_url},
-              "public_url": {"href": "%s/share/jio_public/" % base_url},
-              "private_url": {"href": "%s/share/jio_private/" % base_url}
-            },
-      data={'state': 'monitor_state.data',
-            'process_state': 'monitor_process_resource.status',
-            'process_resource': 'monitor_resource_process.data',
-            'memory_resource': 'monitor_resource_memory.data',
-            'io_resource': 'monitor_resource_io.data',
-            'monitor_process_state': 'monitor_resource.status'}
-    )
-
-  global_state_dict['_embedded'] = {'promises': promise_list}
-
-  if os.path.exists(instance_file):
-    config = ConfigParser.ConfigParser()
-    config.read(instance_file)
-    if 'instance' in config.sections():
-      instance_dict = {}
-      global_state_dict['title'] = config.get('instance', 'name')
-      global_state_dict['hosting-title'] = config.get('instance', 'root-name')
-      if not global_state_dict['title']:
-        global_state_dict['title'] = 'Instance Monitoring'
-      
-      instance_dict['computer'] = config.get('instance', 'computer')
-      instance_dict['ipv4'] = config.get('instance', 'ipv4')
-      instance_dict['ipv6'] = config.get('instance', 'ipv6')
-      instance_dict['software-release'] = config.get('instance', 'software-release')
-      instance_dict['software-type'] = config.get('instance', 'software-type')
-      instance_dict['partition'] = config.get('instance', 'partition')
-
-      global_state_dict['_embedded'].update({'instance' : instance_dict})
-
-  if related_monitor_list:
-    global_state_dict['_links']['related_monitor'] = [{'href': "%s/share/jio_public" % url}
-                          for url in related_monitor_list]
-
-  if os.path.exists(parameter_file):
-    with open(parameter_file) as cfile:
-      global_state_dict['parameters'] = json.loads(cfile.read())
-
-  # Public information with the link to private folder
-  public_state_dict = dict(
-    status=status,
-    date=report_date,
-    _links={'monitor': {'href': '%s/share/jio_private/' % base_url}},
-    title=global_state_dict.get('title', '')
-  )
-  public_state_dict['hosting-title'] = global_state_dict.get('hosting-title', '')
-  public_state_dict['_links']['related_monitor'] = global_state_dict['_links'].get('related_monitor', [])
-
-  with open(global_state_file, 'w') as fglobal:
-    fglobal.write(json.dumps(global_state_dict))
-
-  with open(public_state_file, 'w') as fpglobal:
-    fpglobal.write(json.dumps(public_state_dict))
-
-  generateStatisticsData(
-    os.path.join(statistic_folder, 'monitor_state.data.json'),
-    global_state_dict)
-
-  return 0
-
-if __name__ == "__main__":
-  if len(sys.argv) < 3:
-    print("Usage: %s <monitor_conf_path> <instance_conf_path>" % sys.argv[0])
-    sys.exit(2)
-  sys.exit(main(sys.argv[1:]))

stack/monitor/scripts/monitor-document.py

-#!/usr/bin/env python
-
-import sys
-import os
-import re
-import json
-import argparse
-import subprocess
-from datetime import datetime
-import time
-
-
-def parseArguments():
-  """
-  Parse arguments for monitor instance.
-  """
-  parser = argparse.ArgumentParser()
-  parser.add_argument('--config_folder',
-                      help='Path where json configuration/document will be read and write')
-  parser.add_argument('--htpasswd_bin',
-                      help='Path apache htpasswd binary. Needed to write htpasswd file.')
-  parser.add_argument('--output_cfg_file',
-                      help='Ouput parameters in cfg file.')
-
-  return parser.parse_args()
-
-def fileWrite(file_path, content):
-  if os.path.exists(file_path):
-    try:
-      with open(file_path, 'w') as wf:
-        wf.write(content)
-        return True
-    except OSError, e:
-      print "ERROR while writing changes to %s.\n %s" % (file_path, str(e))
-  return False
-
-def htpasswdWrite(htpasswd_bin,  parameter_dict, value):
-  if not os.path.exists(parameter_dict['file']):
-    return False
-  command = [htpasswd_bin, '-cb', parameter_dict['htpasswd'], parameter_dict['user'], value]
-  process = subprocess.Popen(
-    command,
-    stdin=None,
-    stdout=subprocess.PIPE,
-    stderr=subprocess.PIPE
-  )
-  result = process.communicate()[0]
-  if process.returncode != 0:
-    print result
-    return False
-  with open(parameter_dict['file'], 'w') as pfile:
-    pfile.write(value)
-  return True
-
-def httpdCorsDomainWrite(httpd_cors_file, httpd_gracefull_bin, cors_domain):
-  cors_string = ""
-  cors_domain_list = cors_domain.split()
-  old_httpd_cors_file = os.path.join(
-    os.path.dirname(httpd_cors_file),
-    'prev_%s' % os.path.basename(httpd_cors_file)
-  )
-  if os.path.exists(old_httpd_cors_file) and os.path.isfile(old_httpd_cors_file):
-    try:
-      with open(old_httpd_cors_file, 'r') as cors_file:
-        if cors_file.read() == cors_domain:
-          if os.path.exists(httpd_cors_file) and (os.stat(httpd_cors_file).st_size > 0
-            or (cors_domain == "" and os.stat(httpd_cors_file).st_size == 0)):
-            # Skip if cors file is not empty
-            return True
-    except OSError, e:
-      print "Failed to open file at %s. \n%s" % (old_httpd_cors_file, str(e))
-  for domain in cors_domain_list:
-    if cors_string:
-      cors_string += '|'
-    cors_string += re.escape(domain)
-  try:
-    with open(httpd_cors_file, 'w') as file:
-      file.write('SetEnvIf Origin "^http(s)?://(.+\.)?(%s)$" origin_is=$0\n' % cors_string)
-      file.write('Header always set Access-Control-Allow-Origin %{origin_is}e env=origin_is')
-  except OSError, e:
-    print "ERROR while writing CORS changes to %s.\n %s" % (httpd_cors_file, str(e))
-    return False
-
-  # Save current cors domain list
-  try:
-    with open(old_httpd_cors_file, 'w') as cors_file:
-      cors_file.write(cors_domain)
-  except OSError, e:
-    print "Failed to open file at %s. \n%s" % (old_httpd_cors_file, str(e))
-    return False
-
-  # Restart httpd process
-  try:
-    subprocess.call(httpd_gracefull_bin)
-  except OSError, e:
-    print "Failed to execute command %s.\n %s" % (httpd_gracefull_bin, str(e))
-    return False
-
-def applyEditChage(parser):
-  parameter_tmp_file = os.path.join(parser.config_folder, 'config.tmp.json')
-  config_file = os.path.join(parser.config_folder, 'config.json')
-  parameter_config_file = os.path.join(parser.config_folder, 'config.parameters.json')
-  if not os.path.exists(parameter_tmp_file) or not os.path.isfile(parameter_tmp_file):
-    return {}
-  if not os.path.exists(config_file):
-    print "ERROR: Config file doesn't exist... Exiting"
-    return {}
-
-  new_parameter_list = []
-  parameter_list = []
-  description_dict = {}
-  result_dict = {}
-
-  try:
-    with open(parameter_tmp_file) as tmpfile:
-      new_parameter_list = json.loads(tmpfile.read())
-  except ValueError:
-    print "Error: Couldn't parse json file %s" % parameter_tmp_file
-
-  with open(parameter_config_file) as tmpfile:
-    description_dict = json.loads(tmpfile.read())
-
-  for i in range(0, len(new_parameter_list)):
-    key = new_parameter_list[i]['key']
-    if key != '':
-      description_entry = description_dict[key]
-      if description_entry['type'] == 'file':
-        result_dict[key] = fileWrite(description_entry['file'], new_parameter_list[i]['value'])
-      elif description_entry['type'] == 'htpasswd':
-        result_dict[key] = htpasswdWrite(parser.htpasswd_bin, description_entry, new_parameter_list[i]['value'])
-      elif description_entry['type'] == 'httpdcors':
-        result_dict[key] = httpdCorsDomainWrite(description_entry['cors_file'], description_entry['gracefull_bin'], new_parameter_list[i]['value'])
-
-  if (parser.output_cfg_file):
-    try:
-      with open(parser.output_cfg_file, 'w') as pfile:
-        pfile.write('[public]\n')
-        for parameter in new_parameter_list:
-          if parameter['key']:
-            pfile.write('%s = %s\n' % (parameter['key'], parameter['value']))
-    except OSError, e:
-      print "Error failed to create file %s" % parser.output_cfg_file
-      pass
-
-  return result_dict
-
-if __name__ == "__main__":
-  parser = parseArguments()
-  parameter_tmp_file = os.path.join(parser.config_folder, 'config.tmp.json')
-  config_file = os.path.join(parser.config_folder, 'config.json')
-
-  # Run 4 times with sleep
-  run_counter = 1
-  max_runn = 4
-  sleep_time = 15
-
-  while True:
-    result_dict = applyEditChage(parser)
-    if result_dict != {}:
-      status = True
-      for key in result_dict:
-        if not result_dict[key]:
-          status = False
-    
-      if status and os.path.exists(parameter_tmp_file):
-        try:
-          os.unlink(config_file)
-        except OSError, e:
-          print "ERROR cannot remove file: %s" % parameter_tmp_file
-        else:
-          os.rename(parameter_tmp_file, config_file)
-    if run_counter == max_runn:
-      break
-    else:
-      run_counter += 1
-      time.sleep(sleep_time)
-
-
-
-
-
-
-
-

stack/monitor/scripts/run-promise.py

-#!{{ python }}
-# -*- coding: utf-8 -*-
-
-import sys
-import os
-import subprocess
-import json
-import psutil
-import time
-from shutil import copyfile
-import glob
-import argparse
-import traceback
-
-def parseArguments():
-  """
-  Parse arguments for monitor collector instance.
-  """
-  parser = argparse.ArgumentParser()
-  parser.add_argument('--pid_path',
-                      help='Path where the pid of this process will be writen.')
-  parser.add_argument('--output',
-                      help='The Path of file where Json result of this promise will be saved.')
-  parser.add_argument('--promise_script',
-                      help='Promise script to execute.')
-  parser.add_argument('--promise_name',
-                      help='Title to give to this promise.')
-  parser.add_argument('--promise_type',
-                      default='status',
-                      help='Type of promise to execute. [status, report].')
-  parser.add_argument('--monitor_url',
-                      help='Monitor Instance website URL.')
-  parser.add_argument('--history_folder',
-                      help='Path where old result file will be placed before generate a new json result file.')
-  parser.add_argument('--instance_name',
-                      default='UNKNOWN Software Instance',
-                      help='Software Instance name.')
-  parser.add_argument('--hosting_name',
-                      default='UNKNOWN Hosting Subscription',
-                      help='Hosting Subscription name.')
-
-  return parser.parse_args()
-
-def main():
-  parser = parseArguments()
-
-  if os.path.exists(parser.pid_path):
-    with open(parser.pid_path, "r") as pidfile:
-      try:
-        pid = int(pidfile.read(6))
-      except ValueError:
-        pid = None
-      if pid and os.path.exists("/proc/" + str(pid)):
-        print("A process is already running with pid " + str(pid))
-        return 1
-  start_date = ""
-  with open(parser.pid_path, "w") as pidfile:
-    process = executeCommand(parser.promise_script)
-    ps_process = psutil.Process(process.pid)
-    start_date = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ps_process.create_time()))
-    pidfile.write(str(process.pid))
-
-  status_json = generateStatusJsonFromProcess(process, start_date=start_date)
-
-  status_json['_links'] = {"monitor": {"href": parser.monitor_url}}
-  status_json['title'] = parser.promise_name
-  status_json['instance'] = parser.instance_name
-  status_json['hosting_subscription'] = parser.hosting_name
-  status_json['type'] = parser.promise_type
-
-  # Save the lastest status change date (needed for rss)
-  status_json['change-time'] = ps_process.create_time()
-  if os.path.exists(parser.output):
-    with open(parser.output) as f:
-      try:
-        last_result = json.loads(f.read())
-        if status_json['status'] == last_result['status'] and last_result.has_key('change-time'):
-          status_json['change-time'] = last_result['change-time']
-      except ValueError:
-        pass
-
-  updateStatusHistoryFolder(
-    parser.promise_name,
-    parser.output,
-    parser.history_folder,
-    parser.promise_type
-  )
-  with open(parser.output, "w") as outputfile:
-    json.dump(status_json, outputfile)
-  os.remove(parser.pid_path)
-
-def updateStatusHistoryFolder(name, status_file, history_folder, promise_type):
-  history_path = os.path.join(history_folder)
-  if not os.path.exists(status_file):
-    return
-  if not os.path.exists(history_folder):
-    return
-  if not os.path.exists(history_path):
-    try:
-      os.makedirs(history_path)
-    except OSError, e:
-      if e.errno == os.errno.EEXIST and os.path.isdir(history_path):
-        pass
-      else: raise
-  with open(status_file, 'r') as sf:
-    try:
-      status_dict = json.loads(sf.read())
-    except ValueError:
-      traceback.print_exc()
-      return
-
-  if promise_type == 'status':
-    filename = '%s.history.json' % name
-    history_file = os.path.join(history_path, filename)
-    # Remove links from history (not needed)
-    status_dict.pop('_links', None)
-    if not os.path.exists(history_file):
-      with open(history_file, 'w') as f_history:
-        data_dict = {
-          "date": time.time(),
-          "data": [status_dict]
-        }
-        f_history.write(json.dumps(data_dict))
-    else:
-      # Remove useless informations
-      status_dict.pop('hosting_subscription', '')
-      status_dict.pop('title', '')
-      status_dict.pop('instance', '')
-      status_dict.pop('type', '')
-
-      with open (history_file, mode="r+") as f_history:
-        f_history.seek(0,2)
-        position = f_history.tell() -2
-        f_history.seek(position)
-        #f_history.write(',%s]}' % str(status_dict))
-        f_history.write('%s}' % ',{}]'.format(json.dumps(status_dict)))
-  elif promise_type == 'report':
-    # keep_item_amount = 3
-    filename = '%s.history.json' % (
-      name)
-
-    copyfile(status_file, os.path.join(history_path, filename))
-    """# Don't let history foler grow too much, keep xx files
-    file_list = filter(os.path.isfile,
-        glob.glob("%s/*.%s.history.json" % (history_path, promise_type))
-      )
-    file_count = len(file_list)
-    if file_count > keep_item_amount:
-      file_list.sort(key=lambda x: os.path.getmtime(x))
-      while file_count > keep_item_amount:
-        to_delete = file_list.pop(0)
-        try:
-          os.unlink(to_delete)
-          file_count -= 1
-        except OSError:
-          raise"""
-
-def generateStatusJsonFromProcess(process, start_date=None, title=None):
-  stdout, stderr = process.communicate()
-  try:
-    status_json = json.loads(stdout)
-  except ValueError:
-    status_json = {}
-  if process.returncode != 0:
-    status_json["status"] = "ERROR"
-  elif not status_json.get("status"):
-    status_json["status"] = "OK"
-  if stderr:
-    status_json["message"] = stderr
-  if start_date:
-    status_json["start-date"] = start_date
-  if title:
-    status_json["title"] = title
-  return status_json
-
-
-def executeCommand(args):
-  return subprocess.Popen(
-    args,
-    #cwd=instance_path,
-    #env=None if sys.platform == 'cygwin' else {},
-    stdin=None,
-    stdout=subprocess.PIPE,
-    stderr=subprocess.PIPE
-  )
-
-if __name__ == "__main__":
-  sys.exit(main())

stack/monitor/scripts/status2rss.py

-import sys
-import os
-import json
-from datetime import datetime
-import base64
-import hashlib
-import PyRSS2Gen
-import argparse
-
-def parseArguments():
-  """
-  Parse arguments for monitor Rss Generator.
-  """
-  parser = argparse.ArgumentParser()
-  parser.add_argument('--items_folder',
-                      help='Path where to get *.status.json files which contain result of promises.')
-  parser.add_argument('--output',
-                      help='The Path of file where feed file will be saved.')
-  parser.add_argument('--feed_url',
-                      help='Url of this feed file.')
-  parser.add_argument('--public_url',
-                      help='Monitor Instance public URL.')
-  parser.add_argument('--private_url',
-                      help='Monitor Instance private URL.')
-  parser.add_argument('--instance_name',
-                      default='UNKNOW Software Instance',
-                      help='Software Instance name.')
-  parser.add_argument('--hosting_name',
-                      default='',
-                      help='Hosting Subscription name.')
-
-  return parser.parse_args()
-
-def getKey(item):
-  return item.pubDate
-
-def main():
-  parser = parseArguments()
-
-  rss_item_list = []
-  report_date = datetime.utcnow()
-  for filename in os.listdir(parser.items_folder):
-    if filename.endswith(".status.json"):
-      filepath = os.path.join(parser.items_folder, filename)
-      result_dict = None
-      try:
-        result_dict = json.load(open(filepath, "r"))
-      except ValueError:
-        print "Failed to load json file: %s" % filepath
-        continue
-      description = result_dict.get('message', '')
-      event_time = datetime.fromtimestamp(result_dict['change-time'])
-      rss_item = PyRSS2Gen.RSSItem(
-        categories = [result_dict['status']],
-        source = PyRSS2Gen.Source(result_dict['title'], parser.public_url),
-        title = '[%s] %s' % (result_dict['status'], result_dict['title']),
-        comments = description,
-        description = "%s: %s\n%s" % (event_time, result_dict['status'], description),
-        link = parser.private_url,
-        pubDate = event_time,
-        guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (parser.hosting_name, result_dict['title'])))
-      )
-      rss_item_list.append(rss_item)
-
-
-  ### Build the rss feed
-  sorted(rss_item_list, key=getKey)
-  rss_feed = PyRSS2Gen.RSS2 (
-    title = parser.instance_name,
-    link = parser.feed_url,
-    description = parser.hosting_name,
-    lastBuildDate = report_date,
-    items = rss_item_list
-    )
-
-  with open(parser.output, 'w') as frss:
-    frss.write(rss_feed.to_xml())
-
-if __name__ == "__main__":
-  exit(main())

stack/slapos.cfg

@@ -108,33 +108,32 @@ zc.recipe.egg = 1.3.2.post5
 hexagonit.recipe.download = 1.7.post4
 
 Jinja2 = 2.8
-PyYAML = 3.11
+PyYAML = 3.12
 Werkzeug = 0.11.10
 buildout-versions = 1.7
 cffi = 1.7.0
 click = 6.6
-cliff = 2.1.0
+cliff = 2.2.0
 cmd2 = 0.6.8
 collective.recipe.template = 1.13
-cryptography = 1.4
+cryptography = 1.5
 decorator = 4.0.10
 idna = 2.1
 inotifyx = 0.2.2
 itsdangerous = 0.24
-lxml = 3.6.0
+lxml = 3.6.4
 meld3 = 1.0.2
 netaddr = 0.7.18
 pbr = 1.10.0
 plone.recipe.command = 1.1
 prettytable = 0.7.2
 psutil = 4.3.0
-pyOpenSSL = 16.0.0
+pyOpenSSL = 16.1.0
 pyasn1 = 0.1.9
-pyparsing = 2.1.5
-pytz = 2016.4
-requests = 2.10.0
+pyparsing = 2.1.8
+pytz = 2016.6.1
+requests = 2.11.1
 setuptools = 19.6.2
-simplejson = 3.8.2
 six = 1.10.0
 slapos.cookbook = 1.0.31
 slapos.core = 1.3.15
@@ -142,7 +141,7 @@ slapos.extension.strip = 0.1
 slapos.libnetworkcache = 0.14.5
 slapos.recipe.build = 0.23
 slapos.recipe.cmmi = 0.2
-stevedore = 1.15.0
+stevedore = 1.17.1
 unicodecsv = 0.14.1
 xml-marshaller = 0.9.7
 
@@ -155,7 +154,7 @@ Flask = 0.11.1
 MarkupSafe = 0.23
 
 # Required by:
-# cryptography==1.4
+# cryptography==1.5
 enum34 = 1.1.6
 
 # Required by:
@@ -163,7 +162,7 @@ enum34 = 1.1.6
 functools32 = 3.2.3.post2
 
 # Required by:
-# cryptography==1.4
+# cryptography==1.5
 ipaddress = 1.0.16
 
 # Required by:
@@ -176,6 +175,7 @@ lock-file = 2.0
 
 # Required by:
 # slapos.core==1.3.15
+# XXX 'slapos node format' raises an exception with netifaces 0.10.5.
 netifaces = 0.10.4
 
 # Required by:
@@ -184,11 +184,11 @@ pycparser = 2.14
 
 # Required by:
 # slapos.core==1.3.15
-supervisor = 3.3.0
+supervisor = 3.3.1
 
 # Required by:
 # slapos.core==1.3.15
-uritemplate = 0.6
+uritemplate = 3.0.0
 
 # Required by:
 # slapos.core==1.3.15