Update Release Candidate

Conflicts:
	software/slapos-master/software.cfg

component/apache/apache-backend.conf.in

@@ -78,7 +78,7 @@ LoadModule headers_module modules/mod_headers.so
 LoadModule deflate_module modules/mod_deflate.so
 LoadModule filter_module modules/mod_filter.so
 
-AddOutputFilterByType DEFLATE text/cache-manifest text/html text/plain text/css application/hal+json application/json application/x-javascript text/xml application/xml application/rss+xml text/javascript image/svg+xml
+AddOutputFilterByType DEFLATE text/cache-manifest text/html text/plain text/css application/hal+json application/json application/x-javascript text/xml application/xml application/rss+xml text/javascript image/svg+xml application/x-font-ttf application/font-woff application/font-woff2 application/x-font-opentype
 
 PidFile "{{ parameter_dict['pid-file'] }}"
 ServerAdmin admin@

component/apache/buildout.cfg

@@ -187,5 +187,5 @@ make-targets =
 [template-apache-backend-conf]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/apache-backend.conf.in
-md5sum = e376fdbe8b19551ec08604e64e9a53b9
+md5sum = feef079241bda3407b7ceed5876cb61f
 mode = 640

component/curl/buildout.cfg

@@ -12,8 +12,8 @@ parts =
 
 [curl]
 recipe = slapos.recipe.cmmi
-url = http://curl.haxx.se/download/curl-7.49.0.tar.bz2
-md5sum = 7416aaff4a9210b43edda7615ffa4169
+url = http://curl.haxx.se/download/curl-7.50.1.tar.bz2
+md5sum = 015f6a0217ca6f2c5442ca406476920b
 configure-options =
   --disable-static
   --disable-ldap

component/dropbear/buildout.cfg

@@ -20,11 +20,9 @@ md5sum = 0284ea239083f04c8b874e08e1aca243
 #      in order have all patches working.
 url = http://matt.ucc.asn.au/dropbear/releases/dropbear-0.53.1.tar.bz2
 
-# NOTE DEFAULT_RECV_WINDOW and RECV_MAX_PAYLOAD_LEN are tweaked to support
-#      faster network throughput compared to dropbear defaults.
 configure-options =
   --with-zlib=${zlib:location}
-  CFLAGS="-DENABLE_SINGLEUSER -D__DIRTY_NO_SHELL_CHECKING -DDEFAULT_RECV_WINDOW=1048576 -DRECV_MAX_PAYLOAD_LEN=524288"
+  CFLAGS="-DENABLE_SINGLEUSER -D__DIRTY_NO_SHELL_CHECKING"
 
 environment =
   CPPFLAGS =-I${zlib:location}/include

component/egg-patch/ZODB3-persistent-ghostify-slots.patch

+From d387a425941b37b99355077657edf7a2f117cf47 Mon Sep 17 00:00:00 2001
+From: Kirill Smelkov <kirr@nexedi.com>
+Date: Thu, 21 Jul 2016 22:34:55 +0300
+Subject: [PATCH] persistent: On deactivate release in-slots objects too
+
+( This is backport of https://github.com/zopefoundation/persistent/pull/44
+  to ZODB-3.10 )
+
+On ._p_deactivate() and ._p_invalidate(), when an object goes to ghost
+state, objects referenced by all its attributes, except related to
+persistence machinery, are released, this way freeing memory (if they
+were referenced only from going-to-ghost object).
+
+That's the idea - an object in ghost state is simply a stub, which loads
+its content on first access (via hooking into get/set attr) while
+occupying minimal memory in not-yet-loaded state.
+
+However the above is not completely true right now, as currently on
+ghostification only object's .__dict__ is released, while in-slots objects
+are retained attached to ghost object staying in RAM:
+
+    ---- 8< ----
+    from ZODB import DB
+    from persistent import Persistent
+    import gc
+
+    db = DB(None)
+    jar = db.open()
+
+    class C:
+        def __init__(self, v):
+            self.v = v
+        def __del__(self):
+            print 'released (%s)' % self.v
+
+    class P1(Persistent):
+        pass
+
+    class P2(Persistent):
+        __slots__ = ('aaa')
+
+    p1 = P1()
+    jar.add(p1)
+    p1.aaa = C(1)
+
+    p2 = P2()
+    jar.add(p2)
+    p2.aaa = C(2)
+
+    p1._p_invalidate()
+    # "released (1)" is printed
+
+    p2._p_invalidate()
+    gc.collect()
+    # "released (2)" is NOT printed     <--
+    ---- 8< ----
+
+So teach ghostify() & friends to release objects in slots to free-up
+memory when an object goes to ghost state.
+
+NOTE PyErr_Occurred() added after ghostify() calls because
+pickle_slotnames() can raise an error, but we do not want to change
+ghostify() prototype for backward compatibility reason - as it is used
+in cPersistenceCAPIstruct.
+
+( I hit this bug with wendelin.core which uses proxies to load
+  data from DB to virtual memory manager and then deactivate proxy right
+  after load has been completed:
+
+  https://lab.nexedi.com/nexedi/wendelin.core/blob/f7803634/bigfile/file_zodb.py#L239
+  https://lab.nexedi.com/nexedi/wendelin.core/blob/f7803634/bigfile/file_zodb.py#L295 )
+---
+ src/persistent/cPersistence.c          | 41 +++++++++++++++++++++++++++++++++-
+ src/persistent/tests/testPersistent.py | 24 ++++++++++++++++++++
+ 2 files changed, 64 insertions(+), 1 deletion(-)
+
+diff --git a/src/persistent/cPersistence.c b/src/persistent/cPersistence.c
+index b4a185c..28d1f9a 100644
+--- a/src/persistent/cPersistence.c
++++ b/src/persistent/cPersistence.c
+@@ -75,6 +75,7 @@ fatal_1350(cPersistentObject *self, const char *caller, const char *detail)
+ #endif
+ 
+ static void ghostify(cPersistentObject*);
++static PyObject * pickle_slotnames(PyTypeObject *cls);
+ 
+ /* Load the state of the object, unghostifying it.  Upon success, return 1.
+  * If an error occurred, re-ghostify the object and return -1.
+@@ -141,7 +142,7 @@ accessed(cPersistentObject *self)
+ static void
+ ghostify(cPersistentObject *self)
+ {
+-  PyObject **dictptr;
++  PyObject **dictptr, *slotnames;
+ 
+   /* are we already a ghost? */
+   if (self->state == cPersistent_GHOST_STATE)
+@@ -171,6 +172,8 @@ ghostify(cPersistentObject *self)
+     _estimated_size_in_bytes(self->estimated_size);
+   ring_del(&self->ring);
+   self->state = cPersistent_GHOST_STATE;
++
++  /* clear __dict__ */
+   dictptr = _PyObject_GetDictPtr((PyObject *)self);
+   if (dictptr && *dictptr)
+     {
+@@ -178,6 +181,38 @@ ghostify(cPersistentObject *self)
+       *dictptr = NULL;
+     }
+ 
++  /* clear all slots besides _p_* */
++  slotnames = pickle_slotnames(Py_TYPE(self));
++  if (slotnames && slotnames != Py_None)
++  {
++    int i;
++
++    for (i = 0; i < PyList_GET_SIZE(slotnames); i++)
++    {
++        PyObject *name;
++        char *cname;
++        int is_special;
++
++        name = PyList_GET_ITEM(slotnames, i);
++        if (PyBytes_Check(name))
++        {
++            cname = PyBytes_AS_STRING(name);
++            is_special = !strncmp(cname, "_p_", 3);
++            if (is_special) /* skip persistent */
++            {
++                continue;
++            }
++        }
++
++        /* NOTE: this skips our delattr hook */
++        if (PyObject_GenericSetAttr((PyObject *)self, name, NULL) < 0)
++            /* delattr of non-set slot will raise AttributeError - we
++             * simply ignore. */
++            PyErr_Clear();
++    }
++  }
++  Py_XDECREF(slotnames);
++
+   /* We remove the reference to the just ghosted object that the ring
+    * holds.  Note that the dictionary of oids->objects has an uncounted
+    * reference, so if the ring's reference was the only one, this frees
+@@ -261,6 +296,8 @@ Per__p_deactivate(cPersistentObject *self)
+          called directly. Methods that override this need to
+          do the same! */
+       ghostify(self);
++      if (PyErr_Occurred())
++        return NULL;
+     }
+ 
+   Py_INCREF(Py_None);
+@@ -289,6 +326,8 @@ Per__p_invalidate(cPersistentObject *self)
+       if (Per_set_changed(self, NULL) < 0)
+         return NULL;
+       ghostify(self);
++      if (PyErr_Occurred())
++        return NULL;
+     }
+   Py_INCREF(Py_None);
+   return Py_None;
+diff --git a/src/persistent/tests/testPersistent.py b/src/persistent/tests/testPersistent.py
+index 51e0382..fdb8b67 100644
+--- a/src/persistent/tests/testPersistent.py
++++ b/src/persistent/tests/testPersistent.py
+@@ -180,6 +180,30 @@ class PersistenceTest(unittest.TestCase):
+         self.assertEqual(obj._p_changed, None)
+         self.assertEqual(obj._p_state, GHOST)
+ 
++    def test__p_invalidate_from_changed_w_slots(self):
++        from persistent import Persistent
++        class Derived(Persistent):
++            __slots__ = ('myattr1', 'myattr2')
++            def __init__(self):
++                self.myattr1 = 'value1'
++                self.myattr2 = 'value2'
++        obj = Derived()
++        jar = self._makeJar()
++        jar.add(obj)
++        obj._p_activate()
++        obj._p_changed = True
++        jar._loaded = []
++        jar._registered = []
++        self.assertEqual(Derived.myattr1.__get__(obj), 'value1')
++        self.assertEqual(Derived.myattr2.__get__(obj), 'value2')
++        obj._p_invalidate()
++        self.assertIs(obj._p_changed, None)
++        self.assertEqual(list(jar._loaded), [])
++        self.assertRaises(AttributeError, lambda: Derived.myattr1.__get__(obj))
++        self.assertRaises(AttributeError, lambda: Derived.myattr2.__get__(obj))
++        self.assertEqual(list(jar._loaded), [])
++        self.assertEqual(list(jar._registered), [])
++
+     def test_initial_serial(self):
+         NOSERIAL = "\000" * 8
+         obj = self._makeOne()
+-- 
+2.9.2.701.gf965a18.dirty
+

component/flex/buildout.cfg

 [buildout]
 extends =
   ../m4/buildout.cfg
+  ../xz-utils/buildout.cfg
 parts =
   flex
 
 [flex]
 recipe = slapos.recipe.cmmi
-url = http://downloads.sourceforge.net/project/flex/flex/flex-2.5.35/flex-2.5.35.tar.gz
-md5sum = 201d3f38758d95436cbc64903386de0b
+url = http://downloads.sourceforge.net/project/flex/flex-2.6.0.tar.xz
+md5sum = 3cbbfa1554d0b75fad9f8100732454de
 environment =
   M4=${m4:location}/bin/m4
+  PATH=${xz-utils:location}/bin:%(PATH)s

component/fontconfig/buildout.cfg

@@ -13,8 +13,8 @@ parts =
 
 [fontconfig]
 recipe = slapos.recipe.cmmi
-url = http://fontconfig.org/release/fontconfig-2.11.1.tar.bz2
-md5sum = 824d000eb737af6e16c826dd3b2d6c90
+url = http://fontconfig.org/release/fontconfig-2.12.1.tar.bz2
+md5sum = b5af5a423ee3b5cfc34846838963c058
 pkg_config_depends = ${freetype:pkg_config_depends}:${freetype:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig
 # XXX-Cedric : should we use --with-add-fonts={somefont:location}/share,{someotherfont:location}/share?
 configure-options =

component/gcc/buildout.cfg

@@ -44,8 +44,8 @@ environment =
 
 [gcc-common]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/gcc/gcc-5.3.0/gcc-5.3.0.tar.bz2
-md5sum = c9616fd448f980259c31de613e575719
+url = http://ftp.gnu.org/gnu/gcc/gcc-5.4.0/gcc-5.4.0.tar.bz2
+md5sum = 4c626ac2a83ef30dfb9260e6f59c2b30
 # make install does not work when several core are used
 make-targets = install -j1
 

component/gettext/buildout.cfg

@@ -10,8 +10,8 @@ extends =
 
 [gettext]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/pub/gnu/gettext/gettext-0.19.6.tar.lz
-md5sum = 45b2a123cdc7cef54df98152a0da3fcc
+url = http://ftp.gnu.org/pub/gnu/gettext/gettext-0.19.8.1.tar.lz
+md5sum = d838d2c4144261d0c5fbab4a0aceb5c1
 
 configure-options =
   --disable-static

component/gmp/buildout.cfg

@@ -8,9 +8,9 @@ parts =
 
 [gmp]
 recipe = slapos.recipe.cmmi
-version = 6.1.0
+version = 6.1.1
 url = https://gmplib.org/download/gmp/gmp-${:version}.tar.xz
-md5sum = a9868ef2556ad6a2909babcd1428f3c7
+md5sum = e70e183609244a332d80529e7e155a35
 configure-options =
   --enable-cxx
   --disable-static

component/grep/buildout.cfg

@@ -8,8 +8,8 @@ parts =
 
 [grep]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/grep/grep-2.21.tar.xz
-md5sum = 43c48064d6409862b8a850db83c8038a
+url = http://ftp.gnu.org/gnu/grep/grep-2.25.tar.xz
+md5sum = 04e96b0e6f0fe6a180ae62c88fcd0af6
 environment =
   PATH=${xz-utils:location}/bin:%(PATH)s
   CPPFLAGS=-I${pcre:location}/include

component/groonga/buildout.cfg

@@ -14,11 +14,12 @@ extends =
 
 [groonga]
 recipe = slapos.recipe.cmmi
-url = http://packages.groonga.org/source/groonga/groonga-6.0.2.tar.gz
-md5sum = 10fde26350296a479953fd28679f2566
+url = http://packages.groonga.org/source/groonga/groonga-6.0.8.tar.gz
+md5sum = 42c78baddfca42d3a19fe8b06fac821d
 # temporary patch to respect more tokens in natural language mode.
 patches =
   ${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b
+  ${:_profile_base_location_}/groonga-6.0.8-bugfix04107.patch#7618361b006abdbcceb1fa1df0bd2136
 patch-options = -p1
 configure-options =
   --disable-static

component/groonga/groonga-6.0.8-bugfix04107.patch

+diff --git a/lib/ii.c b/lib/ii.c
+index 97b34ec..9523cf7 100644
+--- a/lib/ii.c
++++ b/lib/ii.c
+@@ -5110,14 +5110,18 @@ grn_ii_cursor_next_internal(grn_ctx *ctx, grn_ii_cursor *c,
+               c->pb.rid = 0;
+               if (br->jump > 0 && !BUFFER_REC_DELETED(br)) {
+                 buffer_rec *jump_br = BUFFER_REC_AT(c->buf, br->jump);
+-                uint8_t *jump_bp;
+-                uint32_t jump_rid;
+-                jump_bp = GRN_NEXT_ADDR(jump_br);
+-                GRN_B_DEC(jump_rid, jump_bp);
+-                if (jump_rid < c->min) {
+-                  c->nextb = br->jump;
+-                } else {
++                if (BUFFER_REC_DELETED(jump_br)) {
+                   c->nextb = br->step;
++                } else {
++                  uint8_t *jump_bp;
++                  uint32_t jump_rid;
++                  jump_bp = GRN_NEXT_ADDR(jump_br);
++                  GRN_B_DEC(jump_rid, jump_bp);
++                  if (jump_rid < c->min) {
++                    c->nextb = br->jump;
++                  } else {
++                    c->nextb = br->step;
++                  }
+                 }
+               } else {
+                 c->nextb = br->step;

component/gzip/buildout.cfg

 [buildout]
 extends =
-  ../patch/buildout.cfg
   ../xz-utils/buildout.cfg
 parts =
   gzip
 
 [gzip]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.gnu.org/pub/gnu/gzip/gzip-1.6.tar.xz
-md5sum = da981f86677d58a106496e68de6f8995
+url = ftp://ftp.gnu.org/pub/gnu/gzip/gzip-1.8.tar.xz
+md5sum = f7caabb65cddc1a4165b398009bd05b9
 environment =
-  PATH=${patch:location}/bin:${xz-utils:location}/bin:%(PATH)s
-patch-options = -p1
-# The --rsyncable patch is from debian/ubuntu,
-# specifically https://launchpad.net/ubuntu/+source/gzip/1.6-3ubuntu1
-# It is required to minimize the bandwidth used by rsync.
-# For an explanation, see http://beeznest.wordpress.com/2005/02/03/rsyncable-gzip/
-# Hunks for .texi files have been removed to avoid a dependency on makeinfo.
-patches =
-  ${:_profile_base_location_}/rsyncable.diff#0587af03a5580e2b7b4007469ee2b601
-
+  PATH=${xz-utils:location}/bin:%(PATH)s

component/gzip/rsyncable.diff

---- a/deflate.c
-+++ b/deflate.c
-@@ -131,6 +131,14 @@
- #endif
- /* Matches of length 3 are discarded if their distance exceeds TOO_FAR */
- 
-+#ifndef RSYNC_WIN
-+#  define RSYNC_WIN 8192
-+#endif
-+/* Size of rsync window, must be < MAX_DIST */
-+
-+#define RSYNC_SUM_MATCH(sum) (((sum) & (RSYNC_WIN - 1)) == 0)
-+/* Whether window sum matches magic value */
-+
- /* ===========================================================================
-  * Local data used by the "longest match" routines.
-  */
-@@ -212,6 +220,8 @@
- unsigned good_match;
- /* Use a faster search when the previous match is longer than this */
- 
-+local ulg rsync_sum;  /* rolling sum of rsync window */
-+local ulg rsync_chunk_end; /* next rsync sequence point */
- 
- /* Values for max_lazy_match, good_match and max_chain_length, depending on
-  * the desired pack level (0..9). The values given below have been tuned to
-@@ -314,6 +324,10 @@
- #endif
-     /* prev will be initialized on the fly */
- 
-+    /* rsync params */
-+    rsync_chunk_end = 0xFFFFFFFFUL;
-+    rsync_sum = 0;
-+
-     /* Set the default configuration parameters:
-      */
-     max_lazy_match   = configuration_table[pack_level].max_lazy;
-@@ -550,6 +564,8 @@
-         memcpy((char*)window, (char*)window+WSIZE, (unsigned)WSIZE);
-         match_start -= WSIZE;
-         strstart    -= WSIZE; /* we now have strstart >= MAX_DIST: */
-+	if (rsync_chunk_end != 0xFFFFFFFFUL)
-+	    rsync_chunk_end -= WSIZE;
- 
-         block_start -= (long) WSIZE;
- 
-@@ -579,13 +595,44 @@
-     }
- }
- 
-+local void rsync_roll(unsigned start, unsigned num)
-+{
-+    unsigned i;
-+
-+    if (start < RSYNC_WIN) {
-+	/* before window fills. */
-+	for (i = start; i < RSYNC_WIN; i++) {
-+	    if (i == start + num) return;
-+	    rsync_sum += (ulg)window[i];
-+	}
-+	num -= (RSYNC_WIN - start);
-+	start = RSYNC_WIN;
-+    }
-+
-+    /* buffer after window full */
-+    for (i = start; i < start+num; i++) {
-+	/* New character in */
-+	rsync_sum += (ulg)window[i];
-+	/* Old character out */
-+	rsync_sum -= (ulg)window[i - RSYNC_WIN];
-+	if (rsync_chunk_end == 0xFFFFFFFFUL && RSYNC_SUM_MATCH(rsync_sum))
-+	    rsync_chunk_end = i;
-+    }
-+}
-+
-+/* ===========================================================================
-+ * Set rsync_chunk_end if window sum matches magic value.
-+ */
-+#define RSYNC_ROLL(s, n) \
-+   do { if (rsync) rsync_roll((s), (n)); } while(0)
-+
- /* ===========================================================================
-  * Flush the current block, with given end-of-file flag.
-  * IN assertion: strstart is set to the end of the current match.
-  */
- #define FLUSH_BLOCK(eof) \
-    flush_block(block_start >= 0L ? (char*)&window[(unsigned)block_start] : \
--                (char*)NULL, (long)strstart - block_start, (eof))
-+                (char*)NULL, (long)strstart - block_start, flush-1, (eof))
- 
- /* ===========================================================================
-  * Processes a new input file and return its compressed length. This
-@@ -596,7 +643,7 @@
- local off_t deflate_fast()
- {
-     IPos hash_head; /* head of the hash chain */
--    int flush;      /* set if current block must be flushed */
-+    int flush = 0;      /* set if current block must be flushed, 2=>and padded  */
-     unsigned match_length = 0;  /* length of best match */
- 
-     prev_length = MIN_MATCH-1;
-@@ -626,7 +673,8 @@
- 
-             lookahead -= match_length;
- 
--            /* Insert new strings in the hash table only if the match length
-+	    RSYNC_ROLL(strstart, match_length);
-+	    /* Insert new strings in the hash table only if the match length
-              * is not too large. This saves time but degrades compression.
-              */
-             if (match_length <= max_insert_length) {
-@@ -654,9 +702,14 @@
-             /* No match, output a literal byte */
-             Tracevv((stderr,"%c",window[strstart]));
-             flush = ct_tally (0, window[strstart]);
-+	    RSYNC_ROLL(strstart, 1);
-             lookahead--;
-             strstart++;
-         }
-+        if (rsync && strstart > rsync_chunk_end) {
-+            flush = 2;
-+            rsync_chunk_end = 0xFFFFFFFFUL;
-+        }
-         if (flush) FLUSH_BLOCK(0), block_start = strstart;
- 
-         /* Make sure that we always have enough lookahead, except
-@@ -679,7 +732,7 @@
- {
-     IPos hash_head;          /* head of hash chain */
-     IPos prev_match;         /* previous match */
--    int flush;               /* set if current block must be flushed */
-+    int flush = 0;           /* set if current block must be flushed */
-     int match_available = 0; /* set if previous match exists */
-     register unsigned match_length = MIN_MATCH-1; /* length of best match */
- 
-@@ -730,6 +783,7 @@
-              */
-             lookahead -= prev_length-1;
-             prev_length -= 2;
-+	    RSYNC_ROLL(strstart, prev_length+1);
-             do {
-                 strstart++;
-                 INSERT_STRING(strstart, hash_head);
-@@ -742,24 +796,38 @@
-             match_available = 0;
-             match_length = MIN_MATCH-1;
-             strstart++;
--            if (flush) FLUSH_BLOCK(0), block_start = strstart;
- 
-+	    if (rsync && strstart > rsync_chunk_end) {
-+		rsync_chunk_end = 0xFFFFFFFFUL;
-+		flush = 2;
-+	    }
-+            if (flush) FLUSH_BLOCK(0), block_start = strstart;
-         } else if (match_available) {
-             /* If there was no match at the previous position, output a
-              * single literal. If there was a match but the current match
-              * is longer, truncate the previous match to a single literal.
-              */
-             Tracevv((stderr,"%c",window[strstart-1]));
--            if (ct_tally (0, window[strstart-1])) {
--                FLUSH_BLOCK(0), block_start = strstart;
--            }
-+	    flush = ct_tally (0, window[strstart-1]);
-+	    if (rsync && strstart > rsync_chunk_end) {
-+		rsync_chunk_end = 0xFFFFFFFFUL;
-+		flush = 2;
-+	    }
-+            if (flush) FLUSH_BLOCK(0), block_start = strstart;
-+	    RSYNC_ROLL(strstart, 1);
-             strstart++;
-             lookahead--;
-         } else {
-             /* There is no previous match to compare with, wait for
-              * the next step to decide.
-              */
-+	    if (rsync && strstart > rsync_chunk_end) {
-+		rsync_chunk_end = 0xFFFFFFFFUL;
-+		flush = 2;
-+		FLUSH_BLOCK(0), block_start = strstart;
-+	    }
-             match_available = 1;
-+	    RSYNC_ROLL(strstart, 1);
-             strstart++;
-             lookahead--;
-         }
---- a/gzip.c
-+++ b/gzip.c
-@@ -215,6 +215,7 @@
- unsigned insize;           /* valid bytes in inbuf */
- unsigned inptr;            /* index of next byte to be processed in inbuf */
- unsigned outcnt;           /* bytes in output buffer */
-+int rsync = 0;             /* make ryncable chunks */
- 
- static int handled_sig[] =
-   {
-@@ -275,7 +276,7 @@
-     {"best",       0, 0, '9'}, /* compress better */
-     {"lzw",        0, 0, 'Z'}, /* make output compatible with old compress */
-     {"bits",       1, 0, 'b'}, /* max number of bits per code (implies -Z) */
--
-+    {"rsyncable",  0, 0, 'R'}, /* make rsync-friendly archive */
-     { 0, 0, 0, 0 }
- };
- 
-@@ -359,6 +360,7 @@
-  "  -Z, --lzw         produce output compatible with old compress",
-  "  -b, --bits=BITS   max number of bits per code (implies -Z)",
- #endif
-+ "  --rsyncable       Make rsync-friendly archive",
-  "",
-  "With no FILE, or when FILE is -, read standard input.",
-  "",
-@@ -489,8 +491,11 @@
- #else
-             recursive = 1;
- #endif
--            break;
--        case 'S':
-+	    break;
-+        case 'R':
-+	    rsync = 1; break;
-+
-+	case 'S':
- #ifdef NO_MULTIPLE_DOTS
-             if (*optarg == '.') optarg++;
- #endif
---- a/gzip.h
-+++ b/gzip.h
-@@ -140,6 +140,7 @@
- extern unsigned insize; /* valid bytes in inbuf */
- extern unsigned inptr;  /* index of next byte to be processed in inbuf */
- extern unsigned outcnt; /* bytes in output buffer */
-+extern int rsync;  /* deflate into rsyncable chunks */
- 
- extern off_t bytes_in;   /* number of input bytes */
- extern off_t bytes_out;  /* number of output bytes */
-@@ -287,7 +288,7 @@
-         /* in trees.c */
- extern void ct_init     (ush *attr, int *method);
- extern int  ct_tally    (int dist, int lc);
--extern off_t flush_block (char *buf, ulg stored_len, int eof);
-+extern off_t flush_block (char *buf, ulg stored_len, int pad, int eof);
- 
-         /* in bits.c */
- extern void     bi_init    (file_t zipfile);
---- a/gzip.1
-+++ b/gzip.1
-@@ -5,6 +5,7 @@
- .ll +8
- .B gzip
- .RB [ " \-acdfhklLnNrtvV19 " ]
-+.RB [ --rsyncable ]
- .RB [ \-S\ suffix ]
- [
- .I "name \&..."
-@@ -287,6 +288,16 @@
- .I gunzip
- ).
- .TP
-+.B --rsyncable
-+While compressing, synchronize the output occasionally based on the input.
-+This increases size by less than 1 percent most cases, but means that the
-+.BR rsync (1)
-+program can take advantage of similarities in the uncompressed input
-+when syncronizing two files compressed with this flag.
-+.I gunzip
-+cannot tell the difference between a compressed file created with this option,
-+and one created without it.
-+.TP
- .B \-S .suf   --suffix .suf
- When compressing, use suffix .suf instead of .gz.
- Any non-empty suffix can be given, but suffixes
---- a/trees.c
-+++ b/trees.c
-@@ -856,9 +856,10 @@
-  * trees or store, and output the encoded block to the zip file. This function
-  * returns the total compressed length for the file so far.
-  */
--off_t flush_block(buf, stored_len, eof)
-+off_t flush_block(buf, stored_len, pad, eof)
-     char *buf;        /* input block, or NULL if too old */
-     ulg stored_len;   /* length of input block */
-+    int pad;          /* pad output to byte boundary */
-     int eof;          /* true if this is the last block for a file */
- {
-     ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */
-@@ -951,6 +952,10 @@
-         Assert (input_len == bytes_in, "bad input size");
-         bi_windup();
-         compressed_len += 7;  /* align on byte boundary */
-+    } else if (pad && (compressed_len % 8) != 0) {
-+        send_bits((STORED_BLOCK<<1)+eof, 3);  /* send block type */
-+        compressed_len = (compressed_len + 3 + 7) & ~7L;
-+        copy_block(buf, 0, 1); /* with header */
-     }
- 
-     return compressed_len >> 3;

component/imagemagick/buildout.cfg

@@ -25,9 +25,9 @@ extends =
 
 [imagemagick]
 recipe = slapos.recipe.cmmi
-version = 7.0.1-1
+version = 7.0.2-10
 url = https://www.imagemagick.org/download/releases/ImageMagick-${:version}.tar.xz
-md5sum = 24673d00fcc8aa00313eeca6aa20fd3c
+md5sum = e1cb23d9c10a8eff228ef30ee281711a
 pkg_config_depends = ${fontconfig:location}/lib/pkgconfig:${fontconfig:pkg_config_depends}:${lcms2:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig
 configure-options =
   --disable-static
@@ -58,8 +58,8 @@ configure-options =
   --with-frozenpaths
 patch-options = -p1
 patches =
-  ${:_profile_base_location_}/imagemagick-6.6.6-1-no-gsx-gsc-probe.patch#3f28ecd9f6722cf2c3238ce6ec3d7a68
-  ${:_profile_base_location_}/safe_policy.patch#07889fefbd9b55f19b9136ed6c17aa8c
+  ${:_profile_base_location_}/imagemagick-7.0.2-10-no-gsx-gsc-probe.patch#64898455d5175efedd1a7bef9f1f18b5
+  ${:_profile_base_location_}/safe_policy.patch#383c0392de7257c9dff7270973342914
 environment =
   PATH=${freetype:location}/bin:${ghostscript:location}/bin:${inkscape:location}/bin:${libxml2:location}/bin:${patch:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${:pkg_config_depends}

component/imagemagick/imagemagick-6.6.6-1-no-gsx-gsc-probe.patch → component/imagemagick/imagemagick-7.0.2-10-no-gsx-gsc-probe.patch

---- ImageMagick-6.6.6-1/configure.ac~	2010-11-28 21:51:05.000000000 +0100
-+++ ImageMagick-6.6.6-1/configure.ac	2010-12-03 14:13:14.000000000 +0100
-@@ -2791,7 +2791,7 @@
+--- ImageMagick-7.0.2-10/configure.ac.orig      2016-08-30 11:33:39.160279386 +0200
++++ ImageMagick-7.0.2-10/configure.ac   2016-08-30 11:35:34.753290590 +0200
+@@ -3110,7 +3110,7 @@
+ AC_PATH_PROG(MrSIDDecodeDelegate, "$MrSIDDecodeDelegateDefault", "$MrSIDDecodeDelegateDefault")
+ AC_PATH_PROG(MVDelegate, "$MVDelegateDefault", "$MVDelegateDefault")
  AC_PATH_PROG(PCLDelegate, "$PCLDelegateDefault", "$PCLDelegateDefault")
- AC_PATH_PROG(PGPDecodeDelegate, "$PGPDecodeDelegateDefault", "$PGPDecodeDelegateDefault")
- AC_PATH_PROG(POVDelegate, "$POVDelegateDefault", "$POVDelegateDefault")
 -AC_PATH_PROGS(PSDelegate, gsx gsc "$PSDelegateDefault", "$PSDelegateDefault")
 +AC_PATH_PROGS(PSDelegate, "$PSDelegateDefault", "$PSDelegateDefault")
- AC_PATH_PROG(RLEEncodeDelegate, "$RLEEncodeDelegateDefault", "$RLEEncodeDelegateDefault")
  AC_PATH_PROG(RMDelegate, "$RMDelegateDefault", "$RMDelegateDefault")
  AC_PATH_PROG(RSVGDecodeDelegate, "$RSVGDecodeDelegateDefault", "$RSVGDecodeDelegateDefault")
---- ImageMagick-6.6.6-1/configure~	2010-11-28 23:27:16.000000000 +0100
-+++ ImageMagick-6.6.6-1/configure	2010-12-03 14:13:57.000000000 +0100
-@@ -30931,7 +30931,7 @@
- fi
- 
- 
--for ac_prog in gsx gsc "$PSDelegateDefault"
-+for ac_prog in "$PSDelegateDefault"
- do
-   # Extract the first word of "$ac_prog", so it can be a program name with args.
- set dummy $ac_prog; ac_word=$2
+ AC_PATH_PROG(SVGDecodeDelegate, "$SVGDecodeDelegateDefault", "$SVGDecodeDelegateDefault")

component/imagemagick/safe_policy.patch

---- ImageMagick-7.0.1-1/config/policy.xml.orig	2016-05-04 00:18:54.000000000 +0200
-+++ ImageMagick-7.0.1-1/config/policy.xml	2016-05-06 08:22:30.002045742 +0200
-@@ -50,16 +50,25 @@
+--- ImageMagick-7.0.2-10/config/policy.xml.orig	2016-08-30 11:37:29.110253211 +0200
++++ ImageMagick-7.0.2-10/config/policy.xml	2016-08-30 11:40:09.719555899 +0200
+@@ -50,19 +50,28 @@
  -->
  <policymap>
    <!-- <policy domain="resource" name="temporary-path" value="/tmp"/> -->
@@ -15,6 +15,9 @@
 -  <!-- <policy domain="resource" name="throttle" value="0"/> -->
 -  <!-- <policy domain="resource" name="time" value="3600"/> -->
 -  <!-- <policy domain="system" name="precision" value="6"/> -->
+-  <!-- <policy domain="coder" rights="none" pattern="MVG" /> -->
+-  <!-- <policy domain="delegate" rights="none" pattern="HTTPS" /> -->
+-  <!-- <policy domain="path" rights="none" pattern="@*" /> -->
 +  <policy domain="resource" name="memory" value="2GiB"/>
 +  <policy domain="resource" name="map" value="4GiB"/>
 +  <policy domain="resource" name="width" value="10MP"/>
@@ -26,7 +29,10 @@
 +  <policy domain="resource" name="throttle" value="0"/>
 +  <policy domain="resource" name="time" value="3600"/>
 +  <policy domain="system" name="precision" value="6"/>
-   <policy domain="cache" name="shared-secret" value="passphrase"/>
++  <policy domain="coder" rights="none" pattern="MVG" />
++  <policy domain="delegate" rights="none" pattern="HTTPS" />
++  <policy domain="path" rights="none" pattern="@*" />
+   <policy domain="cache" name="shared-secret" value="passphrase" stealth="true"/>
 +  <policy domain="coder" rights="none" pattern="EPHEMERAL" />
 +  <policy domain="coder" rights="none" pattern="HTTPS" />
 +  <policy domain="coder" rights="none" pattern="MSL" />

component/inkscape/buildout.cfg

@@ -14,6 +14,7 @@ extends =
   ../libsigc/buildout.cfg
   ../libxml2/buildout.cfg
   ../libxslt/buildout.cfg
+  ../patch/buildout.cfg
   ../perl/buildout.cfg
   ../pkgconfig/buildout.cfg
   ../popt/buildout.cfg
@@ -35,6 +36,11 @@ environment =
 recipe = slapos.recipe.cmmi
 url = https://inkscape.org/en/gallery/item/3860/inkscape-0.91.tar.bz2
 md5sum = 278dfa4514adcde23546370ec2c84581
+patch-options = -p0
+# based on
+# http://bazaar.launchpad.net/~inkscape.dev/inkscape/trunk/revision/15017
+patches =
+  ${:_profile_base_location_}/inkscape-0.91_gcc6.patch#ce3ddbb90f7e5e81fd322469194b6c3c
 pkg_config_depends = ${gtkmm:location}/lib/pkgconfig:${gtkmm:pkg_config_depends}:${gsl:location}/lib/pkgconfig:${popt:location}/lib/pkgconfig:${garbage-collector:location}/lib/pkgconfig:${libxslt:location}/lib/pkgconfig
 configure-options =
   --disable-static
@@ -45,7 +51,7 @@ configure-options =
   --disable-cdr
   --without-gnome-vfs
 environment =
-  PATH=${freetype:location}/bin:${gdk-pixbuf:location}/bin:${gettext:location}/bin:${glib:location}/bin:${intltool:location}/bin:${libxml2:location}/bin:${pkgconfig:location}/bin:${pango:location}/bin:${perl:location}/bin:%(PATH)s
+  PATH=${freetype:location}/bin:${gdk-pixbuf:location}/bin:${gettext:location}/bin:${glib:location}/bin:${intltool:location}/bin:${libxml2:location}/bin:${pango:location}/bin:${patch:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${:pkg_config_depends}
   CPPFLAGS=-I${boost-lib:location}/include -I${cairo:location}/include -I${garbage-collector:location}/include -I${libpng:location}/include -I${popt:location}/include  -I${zlib:location}/include
 # rpath seems not taken from pkgconfig...

component/inkscape/inkscape-0.91_gcc6.patch

+--- src/ui/dialog/layer-properties.cpp	2014-12-21 21:58:32 +0000
++++ src/ui/dialog/layer-properties.cpp	2016-07-18 17:19:25 +0000
+@@ -146,7 +146,7 @@
+     destroy_();
+     Glib::signal_idle().connect(
+         sigc::bind_return(
+-            sigc::bind(sigc::ptr_fun(&::operator delete), this),
++            sigc::bind(sigc::ptr_fun<void*, void>(&::operator delete), this),
+             false 
+         )
+     );

component/kumo/buildout.cfg

 [buildout]
 extends =
+  ../gcc/buildout.cfg
   ../tokyocabinet/buildout.cfg
   ../messagepack/buildout.cfg
   ../openssl/buildout.cfg
@@ -24,5 +25,6 @@ configure-options =
 
 environment =
   CPPFLAGS=-I${zlib:location}/include -I${openssl:location}/include
-  LDFLAGS=-L${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${tokyocabinet:location}/lib -Wl,-rpath=${messagepack:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib
-  PATH=${patch:location}/bin:%(PATH)s
+  LD_LIBRARY_PATH=${gcc-fortran:location}/lib:${gcc-fortran:location}/lib64
+  LDFLAGS=-Wl,-rpath=${gcc-fortran:location}/lib -Wl,-rpath=${gcc-fortran:location}/lib64 -L${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${tokyocabinet:location}/lib -Wl,-rpath=${messagepack:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib
+  PATH=${gcc-fortran:location}/bin:${patch:location}/bin:%(PATH)s

component/mariadb/buildout.cfg

@@ -22,9 +22,9 @@ parts =
 
 [mariadb]
 recipe = slapos.recipe.cmmi
-version = 10.1.14
+version = 10.1.17
 url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http:/ftp.osuosl.org/pub/mariadb/?serve
-md5sum = 294925531e0fd2f0461e3894496a5adc
+md5sum = 036aca95257cb2948dd100605ec6d5a1
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 patch-options = -p0
 patches =
@@ -70,8 +70,8 @@ post-install =
 # mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
 # http://mroonga.github.com/
 recipe = slapos.recipe.cmmi
-url = http://packages.groonga.org/source/mroonga/mroonga-6.02.tar.gz
-md5sum = a7c6320e273eaa407860cac11970dae8
+url = http://packages.groonga.org/source/mroonga/mroonga-6.08.tar.gz
+md5sum = 10c13aa9f7e520d93799be893aa44d3c
 pre-configure =
   mkdir fake_mariadb_source &&
   ln -s ${mariadb:location}/include/mysql/private fake_mariadb_source/sql

component/postgresql/buildout.cfg

@@ -35,5 +35,5 @@ environment =
 
 [postgresql92]
 <= postgresql-common
-url = http://ftp.postgresql.org/pub/source/v9.2.17/postgresql-9.2.17.tar.bz2
-md5sum = a75d4a82eae1edda04eda2e60656e74c
+url = http://ftp.postgresql.org/pub/source/v9.2.18/postgresql-9.2.18.tar.bz2
+md5sum = fd175eb5f29557c6ef2eeaf340330f9a

component/tar/buildout.cfg

 [buildout]
 extends =
-  ../patch/buildout.cfg
+  ../xz-utils/buildout.cfg
 
 parts = tar
 
 [tar]
-patch-options = -p1
-patches =
-  ${:_profile_base_location_}/tar-drop.gets.patch#9352820566aa3534a04bd269c9f89f48
-
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/tar/tar-1.26.tar.gz
-md5sum = 00d1e769c6af702c542cca54b728920d
+url = http://ftp.gnu.org/gnu/tar/tar-1.29.tar.xz
+md5sum = a1802fec550baaeecff6c381629653ef
 environment =
   FORCE_UNSAFE_CONFIGURE=1
-  PATH=${patch:location}/bin:%(PATH)s
+  PATH=${xz-utils:location}/bin:%(PATH)s

component/tar/tar-drop.gets.patch

-diff -ur tar-1.26.orig/gnu/stdio.in.h tar-1.26/gnu/stdio.in.h
---- tar-1.26.orig/gnu/stdio.in.h	2011-03-12 10:14:33.000000000 +0100
-+++ tar-1.26/gnu/stdio.in.h	2012-08-24 15:35:22.299190847 +0200
-@@ -164,7 +164,10 @@
-    so any use of gets warrants an unconditional warning.  Assume it is
-    always declared, since it is required by C89.  */
- #undef gets
-+#if defined(__GLIBC__) && !defined(__UCLIBC__) && !__GLIBC_PREREQ(2, 16)
- _GL_WARN_ON_USE (gets, "gets is a security hole - use fgets instead");
-+#endif
-+
- 
- #if @GNULIB_FOPEN@
- # if @REPLACE_FOPEN@

component/wget/buildout.cfg

@@ -11,8 +11,8 @@ parts =
 
 [wget]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/wget/wget-1.16.3.tar.xz
-md5sum = d2e4455781a70140ae83b54ca594ce21
+url = http://ftp.gnu.org/gnu/wget/wget-1.18.tar.xz
+md5sum = af9ca95a4bb8ac4a9bf10aeae66fa5ec
 configure-options =
   --enable-ipv6
   --enable-opie

software/agent/software.cfg

@@ -43,34 +43,34 @@ eggs =
 [versions]
 apache-libcloud = 0.18.0
 ecdsa = 0.13
-erp5.util = 0.4.44
+erp5.util = 0.4.45
 gitdb = 0.6.4
 pycrypto = 2.6.1
 slapos.recipe.download = 1.0
 slapos.recipe.template = 2.8
-slapos.toolbox = 0.56
+slapos.toolbox = 0.58
 smmap = 0.9.0
 
 # Required by:
-# slapos.toolbox==0.56
-GitPython = 2.0.6
+# slapos.toolbox = 0.58
+GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 feedparser = 5.2.1
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 lockfile = 0.12.2
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 paramiko = 2.0.1
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 rpdb = 0.1.5

software/apache-frontend/common.cfg

@@ -167,7 +167,7 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
 recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/templates/trafficserver/${:filename}
-md5sum = 65afeef0229430ad8a6fbc57298b787b
+md5sum = 59287d2de3d948b135619edd211a5e84
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 filename = records.config.jinja2
 download-only = true

software/apache-frontend/software.cfg

@@ -11,29 +11,29 @@ plone.recipe.command = 1.1
 pycrypto = 2.6.1
 rdiff-backup = 1.0.5
 slapos.recipe.template = 2.8
-slapos.toolbox = 0.56
+slapos.toolbox = 0.58
 smmap = 0.9.0
 
 # Required by:
-# slapos.toolbox==0.56
-GitPython = 2.0.6
+# slapos.toolbox = 0.58
+GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 feedparser = 5.2.1
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 lockfile = 0.12.2
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 paramiko = 2.0.1
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 rpdb = 0.1.5

software/apache-frontend/templates/trafficserver/records.config.jinja2

@@ -184,10 +184,10 @@ CONFIG proxy.config.http.background_fill_completed_threshold FLOAT 0.5
    ##################################
    # origin server connect attempts #
    ##################################
-CONFIG proxy.config.http.connect_attempts_max_retries INT 6
+CONFIG proxy.config.http.connect_attempts_max_retries INT 1
 CONFIG proxy.config.http.connect_attempts_max_retries_dead_server INT 3
 CONFIG proxy.config.http.connect_attempts_rr_retries INT 3
-CONFIG proxy.config.http.connect_attempts_timeout INT 30
+CONFIG proxy.config.http.connect_attempts_timeout INT 160
 CONFIG proxy.config.http.post_connect_attempts_timeout INT 1800
 CONFIG proxy.config.http.down_server.cache_time INT 300
 CONFIG proxy.config.http.down_server.abort_threshold INT 10

software/cdn-me/software.cfg

@@ -38,6 +38,6 @@ eggs =
 [versions]
 cns.recipe.symlink = 0.2.3
 collective.recipe.environment = 0.2.0
-erp5.util = 0.4.44
+erp5.util = 0.4.45
 plone.recipe.command = 1.1
 slapos.recipe.template = 2.8

software/erp5/instance-erp5-input-schema.json

@@ -38,7 +38,7 @@
     "developer-list": {
       "description": "List of logins which should get the Developper role (required to modify portal_components' content), defaulting to inituser-login's value",
       "items": {
-        "pattern": "/^\\S+$/",
+        "pattern": "^\\S+$",
         "type": "string"
       },
       "uniqueItems": true,
@@ -143,28 +143,28 @@
     "kumofs": {
       "description": "Persistent memcached service",
       "additionalProperties": {
-        "$ref": "./instance-kumofs-schema.json#properties"
+        "$ref": "./instance-kumofs-schema.json#/properties"
       },
       "type": "object"
     },
     "memcached": {
       "description": "Volatile memcached service",
       "additionalProperties": {
-        "$ref": "./instance-kumofs-schema.json#properties"
+        "$ref": "./instance-kumofs-schema.json#/properties"
       },
       "type": "object"
     },
     "cloudooo": {
       "description": "Format conversion service",
       "additionalProperties": {
-        "$ref": "./instance-cloudooo-schema.json#properties"
+        "$ref": "./instance-cloudooo-schema.json#/properties"
       },
       "type": "object"
     },
     "mariadb": {
       "description": "Relational database service",
       "additionalProperties": {
-        "$ref": "./instance-mariadb-schema.json#properties"
+        "$ref": "./instance-mariadb-schema.json#/properties"
       },
       "type": "object"
     },

software/erp5/instance-erp5-output-schema.json

 {
   "$schema": "http://json-schema.org/draft-04/schema#",
   "description": "Values returned by ERP5 instantiation",
+  "additionalProperties": false,
   "properties": {
     "hosts-dict": {
       "description": "Hosts mapping, including auto-generated entries",
       "patternProperties": {
         ".*": {
-          "description": "IP or domain names current entry resolves to",
+          "description": "IP current entry resolves to",
           "type": "string"
         }
       },
@@ -24,31 +25,59 @@
       "description": "Initial user password",
       "type": "string"
     },
-    "kumofs-url": {
+    "deadlock-debugger-password": {
+      "description": "Deadlock debugger password",
+      "type": "string"
+    },
+    "memcached-persistent-url": {
       "description": "Persistent memcached access information",
+      "pattern": "^memcached://",
       "type": "string"
     },
-    "memcached-url": {
+    "memcached-volatile-url": {
       "description": "Volatile memcached access information",
+      "pattern": "^memcached://",
       "type": "string"
     },
     "cloudooo-url": {
-      "description": "Conversion service access information",
+      "description": "Conversion service access information - DEPRECATED",
+      "pattern": "^cloudooo://",
+      "type": "string"
+    },
+    "mariadb-database-list": {
+      "description": "Relational database access information",
+      "items": {
+        "pattern": "^mysql://",
         "type": "string"
       },
-    "mariadb-url": {
+      "uniqueItems": true,
+      "type": "array"
+    },
+    "mariadb-test-database-list": {
       "description": "Relational database access information",
+      "items": {
+	"pattern": "^mysql://",
         "type": "string"
       },
+      "uniqueItems": true,
+      "type": "array"
+    },
+    "neo-masters": {
+      "$ref": "../neoppod/instance-neo-output-schema.json#/properties/masters"
+    },
+    "neo-admins": {
+      "$ref": "../neoppod/instance-neo-output-schema.json#/properties/admins"
+    },
     "jupyter-url": {
       "description": "Jupyter notebook web UI access information",
-      "type": "string",
-      "optional": true
+      "pattern": "^https://",
+      "type": "string"
     }
   },
   "patternProperties": {
     "family-.*": {
       "description": "Zope family access information",
+      "pattern": "^https://",
       "type": "string"
     }
   },

software/erp5/software.cfg.json

@@ -5,14 +5,14 @@
   "software-type": {
     "default": {
       "title": "Default",
-      "description": "Default deployment of ERP5 with auto-create instance.",
+      "description": "No automated database modification (ERP5Site is not automatically created).",
       "request": "instance-erp5-input-schema.json",
       "response": "instance-erp5-output-schema.json",
       "index": 0
     },
     "create-erp5-site": {
       "title": "Create ERP5 Site", 
-      "description": "Default deployment of ERP5 with automatic creation of the instance.",
+      "description": "Automated ERP5Site creation on instanciation when ZODB is found empty.",
       "request": "instance-erp5-input-schema.json",
       "response": "instance-erp5-output-schema.json",
       "index": 1

software/html5as/software.cfg

@@ -79,7 +79,7 @@ plone.recipe.command = 1.1
 
 # Required by:
 # slapos.toolbox==0.40.2
-GitPython = 2.0.6
+GitPython = 2.0.8
 
 # Required by:
 # slapos.toolbox==0.40.2

software/jstestnode/software.cfg

@@ -111,6 +111,6 @@ output = ${buildout:directory}/runTestSuite.in
 mode = 0644
 
 [versions]
-erp5.util = 0.4.44
+erp5.util = 0.4.45
 slapos.recipe.template = 2.9
 selenium = 2.53.1

software/kvm/software.cfg

@@ -15,12 +15,12 @@ plone.recipe.command = 1.1
 pycrypto = 2.6.1
 slapos.recipe.template = 2.7
 smmap = 0.9.0
-erp5.util = 0.4.44
+erp5.util = 0.4.45
 pycurl = 7.19.5.1
 
 # Required by:
 # slapos.toolbox==0.48
-GitPython = 2.0.6
+GitPython = 2.0.8
 
 # Required by:
 # slapos.toolbox==0.48

software/neoppod/instance-neo-storage-mysql.cfg.in

@@ -14,6 +14,7 @@ mysql-base-directory = {{ mariadb_location }}
 [my-cnf-parameters]
 socket = ${directory:var_run}/mariadb.sock
 data-directory = ${mysqld:data-directory}
+tmp-directory = ${directory:tmp}
 pid-file = ${directory:var_run}/mariadb.pid
 error-log = ${directory:log}/mariadb_error.log
 slow-query-log = ${directory:log}/mariadb_slowquery.log

software/neoppod/my.cnf.in

@@ -5,6 +5,7 @@
 skip_networking
 socket = {{ socket }}
 datadir = {{ parameter_dict['data-directory'] }}
+tmpdir = {{ parameter_dict['tmp-directory'] }}
 pid_file = {{ parameter_dict['pid-file'] }}
 log_error = {{ parameter_dict['error-log'] }}
 slow_query_log

software/neoppod/software-common.cfg

@@ -45,6 +45,7 @@ eggs = neoppod[admin, ctl, master, storage-importer, storage-mysqldb, tests]
 patch-binary = ${patch:location}/bin/patch
 ZODB3-patches =
   ${:_profile_base_location_}/../../component/egg-patch/ZODB3-3.10.5.patch#c5fe331b1e3a930446f93ab4f6e97c6e
+  ${:_profile_base_location_}/../../component/egg-patch/ZODB3-persistent-ghostify-slots.patch#3a66e9c018d7269bd522d5b0a746f510
 ZODB3-patch-options = -p1
 
 [slapos-deps-eggs]
@@ -95,27 +96,28 @@ md5sum = 82f3f76f54ee9db355966a7ada61f56e
 
 [instance-neo-storage-mysql]
 <= download-base-neo
-md5sum = 4572f8d3f92f1b1639600d0eb7119ab5
+md5sum = cd2a978a09c5686205592923866f6584
 
 [template-neo-my-cnf]
 <= download-base-neo
 url = ${:_profile_base_location_}/my.cnf.in
-md5sum = febd3ed58043ce1367b86cf6e4e69700
+md5sum = 81ab5e842ecf8385b12d735585497cc8
 
 [versions]
 slapos.recipe.template = 2.9
 # patched egg
-ZODB3 = 3.10.5+SlapOSPatched001
-# Required by slapos.toolbox==0.56
-slapos.toolbox = 0.56
+ZODB3 = 3.10.5+SlapOSPatched002
+# Required by slapos.toolbox = 0.58
+slapos.toolbox = 0.58
 apache-libcloud = 0.20.1
 atomize = 0.2.0
 ecdsa = 0.13
 feedparser = 5.2.1
-GitPython = 2.0.6
+GitPython = 2.0.8
 gitdb = 0.6.4
 lockfile = 0.12.2
 mysqlclient = 1.3.7
-paramiko = 2.0.1
+paramiko = 2.0.2
+passlib = 1.6.5
 pycrypto = 2.6.1
 smmap = 0.9.0

software/neoppod/software.cfg

@@ -38,8 +38,8 @@ ZODB3-patches +=
   ${neoppod-repository:location}/ZODB3.patch
 
 [versions]
-ZODB3 = 3.10.5+SlapOSPatched002
-erp5.util = 0.4.44
+ZODB3 = 3.10.5+SlapOSPatched003
+erp5.util = 0.4.45
 # To match ERP5
 transaction = 1.1.1
 ZConfig = 2.9.3

software/re6stnet/software.cfg

@@ -112,15 +112,15 @@ gitdb = 0.6.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
 slapos.recipe.template = 2.7
-slapos.toolbox = 0.56
+slapos.toolbox = 0.58
 smmap = 0.9.0
 
 # Required by:
-# slapos.toolbox==0.56
-GitPython = 2.0.6
+# slapos.toolbox = 0.58
+GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 atomize = 0.2.0
 
 # Required by:
@@ -128,11 +128,11 @@ atomize = 0.2.0
 backports.ssl-match-hostname = 3.4.0.2
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 feedparser = 5.1.3
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 lockfile = 0.12.2
 
 # Required by:
@@ -140,10 +140,10 @@ lockfile = 0.12.2
 miniupnpc = 1.9
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 paramiko = 2.0.1
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 rpdb = 0.1.5
 

software/slapos-master/software.cfg

 [buildout]
 extends =
-  ../../stack/erp5/buildout.cfg
+  ../../software/erp5/sofware.cfg
 
 parts +=
   vifib-fix-products-paths
@@ -10,29 +10,17 @@ parts +=
 # Used to generate bt5lists.
 list = ${erp5:location}/bt5 ${erp5:location}/product/ERP5/bootstrap ${vifib:location}/master/bt5
 
-[genbt5list]
-recipe = plone.recipe.command
-stop-on-error = true
-genbt5list = ${erp5:location}/product/ERP5/bin/genbt5list
-command =
-  ${buildout:executable} ${:genbt5list} ${local-bt5-repository:list}
-update-command = ${:command}
-
 [erp5_repository_list]
 repository_id_list = erp5 vifib/master
 
 [erp5]
-recipe = slapos.recipe.build:gitclone
-repository = https://lab.nexedi.com/nexedi/erp5.git
 branch = erp5-vifib
-git-executable = ${git:location}/bin/git
 revision = b22077622b356aade6981957ddc6ff271c228bbb 
 
 [vifib]
-recipe = slapos.recipe.build:gitclone
-branch = master
+<= erp5
 repository = https://lab.nexedi.com/nexedi/slapos.core.git
-git-executable = ${git:location}/bin/git
+branch = master
 revision = b3b9da6158f0dd3956e72a804c57f18b6f966fc1 
 
 [vifib-fix-products-paths]

software/slaprunner/common.cfg

@@ -101,7 +101,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/instance-resilient-test.cfg.jinja2
 download-only = true
-md5sum = fb8c45e5c35548331fb06c4633ec592a
+md5sum = c31de887459dfb45fe1213f2837cadcb
 filename = instance-resilient-test.cfg.jinja2
 mode = 0644
 
@@ -110,7 +110,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/nginx_conf.in
 download-only = true
-md5sum = 94d83ef3eb89c2d75c8c079ab12b4518
+md5sum = 2b06f7eb9a1d45d250d4b92a944db925
 filename = nginx_conf.in
 mode = 0644
 

software/slaprunner/instance-resilient-test.cfg.jinja2

@@ -61,6 +61,6 @@ config-ignore-known-hosts-file = true
 #config-frontend-domain = google.com
 # XXX Hack to deploy Root Instance on the same computer as the type-test Instance
 sla-computer_guid = ${slap-connection:computer-id}
-return = backend_url
+return = backend-url
 
 [slap-parameter]

software/slaprunner/nginx_conf.in

@@ -35,21 +35,6 @@ http {
         scgi_temp_path {{ param_tempdir['scgi_temp_path'] }};
 
         location / {
-            # When no .htpasswd exist, redirect the user to account creation page
-            if ( !-f {{ param_nginx_frontend['etc_dir'] }}/.htpasswd ) {
-                # redirect URL is different wether nginx is accessed directly or behind apache.
-                # nginx does not support nested if or multiple conditions, so we use this well known hack.
-                set $test no_htpasswd;
-            }
-            if ( $host = [{{ param_nginx_frontend['global-ip'] }}] ) {
-                set $test "${test}_backend_access";
-            }
-            if ( $test = no_htpasswd) {
-                return 301 $scheme://$host/setAccount ;
-            }
-            if ( $test = no_htpasswd_backend_access) {
-                return 301 /setAccount ;
-            }
             auth_basic "Restricted";
             auth_basic_user_file {{ param_nginx_frontend['etc_dir'] }}/.htpasswd;
             proxy_redirect off;

software/slaprunner/software.cfg

@@ -13,36 +13,36 @@ apache-libcloud = 0.20.1
 cns.recipe.symlink = 0.2.3
 collective.recipe.environment = 0.2.0
 ecdsa = 0.13
-erp5.util = 0.4.44
+erp5.util = 0.4.45
 futures = 3.0.5
 gitdb = 0.6.4
 gunicorn = 19.5.0
 prettytable = 0.7.2
 pycrypto = 2.6.1
 slapos.recipe.template = 2.9
-slapos.toolbox = 0.56
+slapos.toolbox = 0.58
 smmap = 0.9.0
 
 # Required by:
-# slapos.toolbox==0.56
-GitPython = 2.0.6
+# slapos.toolbox = 0.58
+GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 feedparser = 5.2.1
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 lockfile = 0.12.2
 
 # Required by:
-# slapos.toolbox==0.56
+# slapos.toolbox = 0.58
 paramiko = 2.0.1
 
 # Required by:
-# slapos.toolbox==0.55
+# slapos.toolbox = 0.58
 passlib = 1.6.5
\ No newline at end of file

software/wendelin/software.cfg

@@ -52,6 +52,5 @@ revision = 9c1384dc94905f05b8bac807c59f38990668b648
 revision = 4dfdf05a89445a9286dbdf364d495bedf10f9394 
 
 [versions]
-scipy = 0.15.1
-msgpack-python = 0.4.6
+msgpack-python = 0.4.8
 wendelin.core = 0.7

stack/boinc/buildout.cfg

@@ -75,7 +75,7 @@ smmap = 0.8.2
 
 # Required by:
 # slapos.toolbox==0.40.2
-GitPython = 2.0.6
+GitPython = 2.0.8
 
 # Required by:
 # slapos.toolbox==0.40.2

stack/cloudooo.cfg

@@ -102,4 +102,4 @@ PasteDeploy = 1.5.2
 
 # Required by:
 # cloudooo==1.2.5.dev0
-erp5.util = 0.4.44
+erp5.util = 0.4.45

stack/erp5/buildout.cfg

@@ -156,7 +156,7 @@ mode = 755
 [template-mariadb]
 <= download-base
 filename = instance-mariadb.cfg.in
-md5sum = ede2481d6ce60a335bde920d8cec5318
+md5sum = 9312af2f9d9faf06d2f26f073ad60180
 link-binary =
   ${coreutils:location}/bin/basename
   ${coreutils:location}/bin/cat
@@ -192,7 +192,7 @@ md5sum = bc6048b85b410693e60e5a77399dd1b7
 [template-my-cnf]
 <= download-base
 filename = my.cnf.in
-md5sum = ac5c87991d95907f62bf2129ecfd42c4
+md5sum = d50920c942b8ee98402f8551fef38383
 
 [template-mariadb-initial-setup]
 <= download-base
@@ -622,8 +622,8 @@ scripts +=
 Acquisition = 2.13.9+SlapOSPatched001
 Products.DCWorkflow = 2.2.4+SlapOSPatched001
 pysvn = 1.7.10+SlapOSPatched002
-python-ldap = 2.4.25+SlapOSPatched001
-python-magic = 0.4.11+SlapOSPatched001
+python-ldap = 2.4.27+SlapOSPatched001
+python-magic = 0.4.12+SlapOSPatched001
 
 # specify dev version to be sure that an old released version is not used
 cloudooo = 1.2.5-dev
@@ -632,7 +632,7 @@ cloudooo = 1.2.5-dev
 PasteDeploy = 1.5.2
 Pygments = 2.1.3
 argparse = 1.4.0
-coverage = 4.0.3
+coverage = 4.2
 zope.dottedname = 4.1.0
 
 # test_UserManagerInterfaces in testERP5Security fails with 1.10.0.
@@ -664,14 +664,14 @@ zope.app.publication = 3.14.0
 zope.app.testing = 3.8.1
 
 # Pinned versions
-Pillow = 3.2.0
+Pillow = 3.3.1
 Products.CMFActionIcons = 2.1.3
 Products.DCWorkflowGraph = 0.4.1
 # Products.ExternalEditor 2.0.0's dtml is not based on Zope2 OFS's one.
 Products.ExternalEditor = 1.1.1
 Products.GenericSetup = 1.8.3
 Products.LongRequestLogger = 2.0.0
-Products.MimetypesRegistry = 2.0.9
+Products.MimetypesRegistry = 2.0.10
 Products.PluginRegistry = 1.4
 Products.TIDStorage = 5.4.9
 PyPDF2 = 1.26.0
@@ -696,19 +696,20 @@ httplib2 = 0.9.2
 huBarcode = 1.0.0
 interval = 1.0.0
 ipdb = 0.10.1
-ipykernel = 4.3.1
-ipython = 4.2.1
+ipykernel = 4.4.1
+ipython = 5.1.0
 ipywidgets = 5.1.5
 logilab-common = 1.2.2
-matplotlib = 1.5.1
+matplotlib = 1.5.2
 mistune = 0.7.3
-notebook = 4.2.1
+notebook = 4.2.2
 numpy = 1.11.1
 objgraph = 3.0.0
 pandas = 0.18.1
 ply = 3.8
 polib = 1.0.7
 pprofile = 1.9.1
+prompt-toolkit = 1.0.3
 ptyprocess = 0.5.1
 pycountry = 1.20
 pyflakes = 1.2.3
@@ -716,12 +717,12 @@ pyflakes = 1.2.3
 pylint = 1.4.4
 python-memcached = 1.58
 pytracemalloc = 1.2
-pyzmq = 15.2.0
+pyzmq = 15.4.0
 qrcode = 5.3
 restkit = 4.2.2
 rtjp-eventlet = 0.3.2
 scikit-learn = 0.17.1
-scipy = 0.17.1
+scipy = 0.18.0
 simplegeneric = 0.8.1
 socketpool = 0.5.3
 spyne = 2.12.11
@@ -729,8 +730,9 @@ suds = 0.4
 terminado = 0.6
 threadframe = 0.2
 timerserver = 2.0.2
-tornado = 4.3
-urlnorm = 1.1.2
+tornado = 4.4.1
+traitlets = 4.2.2
+urlnorm = 1.1.4
 uuid = 1.30
 validictory = 1.0.2
 widgetsnbextension = 1.2.3
@@ -742,23 +744,23 @@ xupdate-processor = 0.4
 Products.ZSQLMethods = 2.13.4
 
 # Required by:
-# ipython==4.2.1
+# ipython==5.1.0
 backports.shutil-get-terminal-size = 1.0.0
 
 # Required by:
-# tornado==4.3
+# tornado==4.4.1
 backports.ssl-match-hostname = 3.5.0.1
 
 # Required by:
-# tornado==4.3
-certifi = 2016.2.28
+# tornado==4.4.1
+certifi = 2016.8.8
 
 # Required by:
 # matplotlib==1.5.1
 cycler = 0.10.0
 
 # Required by:
-# ipython==4.2.1
+# ipython==5.1.0
 # traitlets==4.2.2
 decorator = 4.0.10
 
@@ -767,32 +769,32 @@ decorator = 4.0.10
 fpconst = 0.7.2
 
 # Required by:
-# nbformat==4.0.1
-# notebook==4.2.1
+# nbformat==4.1.0
+# notebook==4.2.2
 # traitlets==4.2.2
 ipython-genutils = 0.1.0
 
 # Required by:
-# notebook==4.2.1
+# notebook==4.2.2
 # nbconvert 4.2.0 depends on entrypoints egg that is not available as tar/zip source.
 nbconvert = 4.1.0
 
 # Required by:
 # nbconvert==4.1.0
-# notebook==4.2.1
-nbformat = 4.0.1
+# notebook==4.2.2
+nbformat = 4.1.0
 
 # Required by:
-# pickleshare==0.7.2
+# ipython==5.1.0
 pathlib2 = 2.1.0
 
 # Required by:
-# ipython==4.2.1
-pexpect = 4.1.0
+# ipython==5.1.0
+pexpect = 4.2.1
 
 # Required by:
-# ipython==4.2.1
-pickleshare = 0.7.2
+# ipython==5.1.0
+pickleshare = 0.7.4
 
 # Required by:
 # matplotlib==1.5.1
@@ -800,13 +802,12 @@ pickleshare = 0.7.2
 python-dateutil = 2.5.3
 
 # Required by:
-# tornado==4.3
+# tornado==4.4.1
 singledispatch = 3.4.0.3
 
 # Required by:
-# ipython==4.2.1
-# notebook==4.2.1
-traitlets = 4.2.2
+# prompt-toolkit==1.0.3
+wcwidth = 0.1.7
 
 # Required by:
 # zope.app.testing==3.8.1

stack/erp5/instance-mariadb.cfg.in

@@ -102,6 +102,7 @@ ip = {{ ip }}
 port = {{ port }}
 socket = ${directory:run}/mariadb.sock
 data-directory = ${directory:mariadb-data}
+tmp-directory = ${directory:tmp}
 pid-file = ${directory:run}/mariadb.pid
 error-log = ${directory:log}/mariadb_error.log
 slow-query-log = ${directory:log}/mariadb_slowquery.log
@@ -204,6 +205,7 @@ etc = ${buildout:directory}/etc
 services = ${:etc}/run
 promise = ${:etc}/promise
 srv = ${buildout:directory}/srv
+tmp = ${buildout:directory}/tmp
 backup = ${:srv}/backup
 mariadb-backup-full = ${:backup}/mariadb-full
 mariadb-backup-incremental = ${:backup}/mariadb-incremental

stack/erp5/my.cnf.in

@@ -19,6 +19,7 @@ skip_networking
 {% endif -%}
 socket = {{ socket }}
 datadir = {{ parameter_dict['data-directory'] }}
+tmpdir = {{ parameter_dict['tmp-directory'] }}
 pid_file = {{ parameter_dict['pid-file'] }}
 log_error = {{ parameter_dict['error-log'] }}
 slow_query_log

stack/lamp.cfg

@@ -50,7 +50,7 @@ smmap = 0.8.2
 
 # Required by:
 # slapos.toolbox==0.40.2
-GitPython = 2.0.6
+GitPython = 2.0.8
 
 # Required by:
 # slapos.toolbox==0.40.2

stack/lamp/buildout.cfg

@@ -194,7 +194,7 @@ smmap = 0.8.2
 
 # Required by:
 # slapos.toolbox==0.40.2
-GitPython = 2.0.6
+GitPython = 2.0.8
 
 # Required by:
 # slapos.toolbox==0.40.2

stack/lapp/buildout.cfg

@@ -191,7 +191,7 @@ smmap = 0.8.2
 
 # Required by:
 # slapos.toolbox==0.40.2
-GitPython = 2.0.6
+GitPython = 2.0.8
 
 # Required by:
 # slapos.toolbox==0.40.2

stack/monitor/buildout.cfg

 [buildout]
-# XXX THIS STACK IS A KIND OF FORK OF `stack/monitor`. THIS ONE WAS
-#     CREATED AS A REDESIGNED ONE TO REMOVE UNWANTED FEATURES AND
-#     TO GO FURTHER TO THE GOOD DESIGN DIRECTION. SEE THE README FOR
-#     MORE INFORMATION.
 
 extends =
   ../../component/apache/buildout.cfg
@@ -43,6 +39,7 @@ eggs =
   plone.recipe.command
   collective.recipe.template
   cns.recipe.symlink
+  slapos.toolbox
 
 [extra-eggs]
 <= monitor-eggs
@@ -90,17 +87,19 @@ recipe = slapos.recipe.template:jinja2
 filename = template-monitor.cfg
 template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in
 rendered = ${buildout:directory}/template-monitor.cfg
-md5sum = 7ac78495de73cadafea87f97428e487f
+md5sum = 84998b1ca3c29445dca70b495515c35b
 context =
     key apache_location apache:location
     key gzip_location gzip:location
-    raw monitor_bin ${monitor2-bin:location}/${monitor2-bin:filename}
-    raw monitor_collect ${monitor-collect:location}/${monitor-collect:filename}
+    raw monitor_bin ${buildout:directory}/bin/monitor.bootstrap
+    raw monitor_collect ${buildout:directory}/bin/monitor.collect
+    raw monitor_runpromise ${buildout:directory}/bin/monitor.runpromise
+    raw monitor_genstatus ${buildout:directory}/bin/monitor.genstatus
+    raw monitor_genrss ${buildout:directory}/bin/monitor.genrss
+    raw monitor_configwrite ${buildout:directory}/bin/monitor.configwrite
     raw monitor_conf_template ${monitor-conf:location}/${monitor-conf:filename}
-    raw monitor_document_edit ${monitor-document-edit:location}/${monitor-document-edit:filename}
     raw monitor_https_cors ${monitor-httpd-cors:location}/${monitor-httpd-cors:filename}
     raw monitor_instance_info ${monitor-instance-info:location}/${monitor-instance-info:filename}
-    raw monitor_globalstate ${monitor-globalstate:location}/${monitor-globalstate:filename}
     raw curl_executable_location ${curl:location}/bin/curl
     raw dash_executable_location ${dash:location}/bin/dash
     raw dcron_executable_location ${dcron:location}/sbin/crond
@@ -109,48 +108,12 @@ context =
     raw openssl_executable_location ${openssl:location}/bin/openssl
     raw python_executable ${buildout:executable}
     raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter}
-    raw promise_executor_py ${run-promise-py:rendered}
     raw template_wrapper ${monitor-template-wrapper:location}/${monitor-template-wrapper:filename}
-    raw status2rss_executable_path ${status2rss-executable:location}/${status2rss-executable:filename}
 depends = 
   ${monitor-eggs:eggs}
 
-
-[monitor2-bin]
-<= monitor-template-script
-filename = monitor.py
-md5sum = 5525e7445dab16fd03f4eeccf069b74b
-
-[run-promise-py]
-recipe = slapos.recipe.template:jinja2
-template = ${:_profile_base_location_}/scripts/run-promise.py
-rendered = ${buildout:parts-directory}/monitor-scripts/run-promise.py
-md5sum = 97148dfbb730cc4f55ed54513ce823e0
-mode = 0755
-context =
-  raw python ${buildout:directory}/bin/${extra-eggs:interpreter}
-
-[status2rss-executable]
-<= monitor-template-script
-filename = status2rss.py
-md5sum = 88e3bf955e1e4eac76a444d50fa4f020
-
-[monitor-globalstate]
-<= monitor-template-script
-filename = globalstate.py
-md5sum = 3377e325baa4ecfcd6eee06945fb69fc
-
-[monitor-collect]
-<= monitor-template-script
-filename = collect.py
-md5sum = 78fbcb56761315bde354fe7914d3c54f
-
-[monitor-document-edit]
-<= monitor-template-script
-filename = monitor-document.py
-md5sum = 399ff4939b55ff74e6d48bec5a495981
-
 [versions]
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
+slapos.toolbox = 0.58
 

stack/monitor/instance-monitor.cfg.jinja2.in

@@ -135,7 +135,7 @@ monitor-hal-json = ${monitor-directory:public}/monitor.hal.json
 service-pid-folder = ${monitor-directory:pids}
 crond-folder = ${logrotate-directory:cron-entries}
 logrotate-folder = ${logrotate:logrotate-entries}
-promise-runner = {{ promise_executor_py }}
+promise-runner = {{ monitor_runpromise }}
 promise-folder-list = 
   ${directory:promises}
   ${directory:monitor-promise}
@@ -159,6 +159,8 @@ collector-db = ${monitor-instance-parameter:collector-db}
 collect-script = {{ monitor_collect }}
 python = {{ python_with_eggs }}
 
+promise-output-file = ${directory:monitor}/monitor-bootstrap-status
+
 [monitor-conf]
 recipe = slapos.recipe.template:jinja2
 template = {{ monitor_conf_template }}
@@ -272,7 +274,7 @@ command = kill -USR1 $(cat ${monitor-httpd-conf-parameter:pid-file})
 [monitor-status2rss-wrapper]
 recipe = slapos.cookbook:wrapper
 # XXX - hard-coded Urls
-command-line = {{ python_with_eggs }} {{ status2rss_executable_path }} --output '${monitor-directory:public}/feed' --items_folder '${monitor-directory:public}'  --feed_url '${monitor-conf-parameters:base-url}/public/feed' --public_url '${monitor-conf-parameters:base-url}/share/jio_public/' --private_url '${monitor-conf-parameters:base-url}/share/jio_private/' --instance_name '${monitor-conf-parameters:title}' --hosting_name '${monitor-conf-parameters:root-title}'
+command-line = {{ monitor_genrss }} --output '${monitor-directory:public}/feed' --items_folder '${monitor-directory:public}'  --feed_url '${monitor-conf-parameters:base-url}/public/feed' --public_url '${monitor-conf-parameters:base-url}/share/jio_public/' --private_url '${monitor-conf-parameters:base-url}/share/jio_private/' --instance_name '${monitor-conf-parameters:title}' --hosting_name '${monitor-conf-parameters:root-title}'
 
 wrapper-path = ${directory:bin}/monitor-status2rss.py
 
@@ -285,13 +287,13 @@ command = ${monitor-status2rss-wrapper:wrapper-path}
 
 [monitor-globalstate-wrapper]
 recipe = slapos.cookbook:wrapper
-command-line = {{ python_with_eggs }} {{ monitor_globalstate }} '${monitor-conf:rendered}' '${monitor-instance-info:rendered}'
+command-line = {{ monitor_genstatus }} '${monitor-conf:rendered}' '${monitor-instance-info:rendered}'
 wrapper-path = ${directory:bin}/monitor-globalstate
 
 [monitor-configurator-wrapper]
 recipe = slapos.cookbook:wrapper
 # XXX - hard coded path
-command-line = {{ python_with_eggs }} {{ monitor_document_edit }} --config_folder '${monitor-conf-parameters:private-folder}/config/.jio_documents' --output_cfg_file '${monitor-instance-parameter:configuration-file-path}' --htpasswd_bin '{{ apache_location }}/bin/htpasswd'
+command-line = {{ monitor_configwrite }} --config_folder '${monitor-conf-parameters:private-folder}/config/.jio_documents' --output_cfg_file '${monitor-instance-parameter:configuration-file-path}' --htpasswd_bin '{{ apache_location }}/bin/htpasswd'
 wrapper-path = ${directory:bin}/monitor-configurator
 
 [monitor-globalstate-cron-entry]
@@ -330,7 +332,7 @@ monitor-httpd-ipv6 = ${slap-configuration:ipv6-random}
 monitor-httpd-port = 8196
 # XXX - Set monitor-base-url = ${monitor-httpd-conf-parameter:url} => https://[ipv6]:port
 monitor-base-url = ${monitor-frontend-promise:url}
-# monitor-base-url = ${monitor-httpd-conf-parameter:url}
+#monitor-base-url = ${monitor-httpd-conf-parameter:url}
 root-instance-title = ${slap-configuration:root-instance-title}
 monitor-url-list =
 cors-domains = monitor.app.officejs.com 
@@ -367,6 +369,17 @@ dash_path = {{ dash_executable_location }}
 curl_path = {{ curl_executable_location }}
 check-secure = 1
 
+[monitor-bootstrap-promise]
+recipe = slapos.recipe.template:jinja2
+template = {{ template_wrapper }}
+rendered = ${directory:promises}/monitor-bootstrap-status
+file = ${monitor-conf-parameters:promise-output-file}
+command = if [ ! -f "${:file}" ]; then echo "Monitor bootstrap exited with error." && exit 2; else echo "Bootstrap OK"; fi
+mode = 0700
+context =
+    key content :command
+    raw dash_binary {{ dash_executable_location }}
+
 [monitor-base]
 # create dependencies between required monitor parts
 recipe = plone.recipe.command
@@ -383,6 +396,7 @@ depends =
   ${ca-httpd:wrapper}
   ${monitor-httpd-promise:filename}
   ${monitor-status2rss-cron-entry:name}
+  ${monitor-bootstrap-promise:file}
 
 [monitor-publish]
 monitor-base-url = ${publish:monitor-base-url}

stack/monitor/scripts/collect.py

-# -*- coding: utf-8 -*-
-##############################################################################
-#
-# Copyright (c) 2010-2014 Vifib SARL and Contributors.
-# All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public License
-# as published by the Free Software Foundation; either version 2.1
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-
-import sqlite3
-import os
-import pwd
-import time
-import json
-import argparse
-import psutil
-from time import strftime
-from datetime import datetime, timedelta
-
-
-def parseArguments():
-  """
-  Parse arguments for monitor collector instance.
-  """
-  parser = argparse.ArgumentParser()
-  parser.add_argument('--output_folder',
-                      help='Path of the folder where output files should be written.')
-  parser.add_argument('--partition_id',
-                      help='ID of the computer partition to collect data from.')
-  parser.add_argument('--collector_db',
-                      help='The path of slapos collect database.')
-
-  return parser.parse_args()
-
-
-class RessourceCollect:
-
-  def __init__(self, db_path = None):
-    assert os.path.exists(db_path) and os.path.isfile(db_path)
-    self.uri = db_path
-    self.connection = None
-    self.cursor = None
-
-  def connect(self):
-    self.connection = sqlite3.connect(self.uri)
-    self.cursor = self.connection.cursor()
-
-  def close(self):
-    assert self.connection is not None
-    self.cursor.close()
-    self.connection.close()
-
-  def _execute(self, sql):
-    assert self.connection is not None
-    return self.cursor.execute(sql)
-
-  def select(self, table, date=None, columns="*", where=None):
-    """ Query database for a full table information """
-    if date is not None:
-      where_clause = " WHERE date = '%s' " % date
-    else:
-      where_clause = ""
-    
-    if where is not None:
-      if where_clause == "":
-        where_clause += " WHERE 1 = 1 "
-      where_clause += " AND %s " % where 
-    select_sql = "SELECT %s FROM %s %s " % (columns, table, where_clause)
-    return self._execute(select_sql)
-
-  def has_table(self, name):
-    self.connect()
-    check_result_cursor = self.select(
-      table="sqlite_master",
-      columns='name',
-      where="type='table' AND name='%s'" % name)
-    table_exists_result = zip(*check_result_cursor)
-    if not len(table_exists_result) or table_exists_result[0][0] is None:
-      return False
-    return True
-
-  def getPartitionCPULoadAverage(self, partition_id, date_scope):
-    self.connect()
-    query_result_cursor = self.select("user", date_scope,
-                       columns="SUM(cpu_percent)", 
-                       where="partition = '%s'" % partition_id)
-
-    cpu_percent_sum = zip(*query_result_cursor)
-    if len(cpu_percent_sum) and cpu_percent_sum[0][0] is None:
-      return
-
-    query_result_cursor = self.select("user", date_scope,
-                       columns="COUNT(DISTINCT time)", 
-                       where="partition = '%s'" % partition_id)
-
-    sample_amount = zip(*query_result_cursor)
-    self.close()
-
-    if len(sample_amount) and len(cpu_percent_sum):
-      return round(cpu_percent_sum[0][0]/sample_amount[0][0], 2)
-
-  def getPartitionUsedMemoryAverage(self, partition_id, date_scope):
-    self.connect()
-    query_result_cursor = self.select("user", date_scope,
-                       columns="SUM(memory_rss)", 
-                       where="partition = '%s'" % partition_id)
-
-    memory_sum = zip(*query_result_cursor)
-    if len(memory_sum) and memory_sum[0][0] is None:
-      return
-
-    query_result_cursor = self.select("user", date_scope,
-                       columns="COUNT(DISTINCT time)", 
-                       where="partition = '%s'" % partition_id)
-
-    sample_amount = zip(*query_result_cursor)
-    self.close()
-
-    if len(sample_amount) and len(memory_sum):
-      return round(memory_sum[0][0]/(sample_amount[0][0]*1024*1024.0), 2)
-
-  def getPartitionDiskUsedAverage(self, partition_id, date_scope):
-    if not self.has_table('folder'):
-      return
-    self.db.connect()
-    query_result_cursor = self.select("folder", date_scope,
-                       columns="SUM(disk_used)", 
-                       where="partition = '%s'" % partition_id)
-
-    disk_used_sum = zip(*query_result_cursor)
-    if len(disk_used_sum) and disk_used_sum[0][0] is None:
-      return
-    query_result_cursor = self.select("folder", date_scope,
-                       columns="COUNT(DISTINCT time)", 
-                       where="partition = '%s'" % partition_id)
-  
-    collect_amount = zip(*query_result_cursor)
-    self.db.close()
-  
-    if len(collect_amount) and len(disk_used_sum):
-      return round(disk_used_sum[0][0]/(collect_amount[0][0]*1024.0), 2)
-
-
-  def getPartitionConsumption(self, partition_id, where=""):
-    """
-      Query collector db to get consumed ressource for last minute
-    """
-    self.connect()
-    comsumption_list = []
-    if where != "":
-      where = "and %s" % where
-    date_scope = datetime.now().strftime('%Y-%m-%d')
-    min_time = (datetime.now() - timedelta(minutes=1)).strftime('%H:%M:00')
-    max_time = (datetime.now() - timedelta(minutes=1)).strftime('%H:%M:59')
-
-    sql_query = """select count(pid), SUM(cpu_percent) as cpu_result, SUM(cpu_time), 
-MAX(cpu_num_threads), SUM(memory_percent), SUM(memory_rss), pid, SUM(io_rw_counter), 
-SUM(io_cycles_counter) from user 
-where date='%s' and partition='%s' and (time between '%s' and '%s') %s 
-group by pid order by cpu_result desc""" % (
-        date_scope, partition_id, min_time, max_time, where)
-    query_result = self._execute(sql_query)
-    for result in query_result:
-      count = int(result[0])
-      if not count > 0:
-        continue
-      resource_dict = {
-        'pid': result[6],
-        'cpu_percent': round(result[1]/count, 2),
-        'cpu_time': round((result[2] or 0)/(60.0), 2),
-        'cpu_num_threads': round(result[3]/count, 2),
-        'memory_percent': round(result[4]/count, 2),
-        'memory_rss': round((result[5] or 0)/(1024*1024.0), 2),
-        'io_rw_counter': round(result[7]/count, 2),
-        'io_cycles_counter': round(result[8]/count, 2)
-      }
-      try:
-        pprocess = psutil.Process(int(result[6]))
-      except psutil.NoSuchProcess:
-        pass
-      else:
-        resource_dict['name'] = pprocess.name()
-        resource_dict['command'] = pprocess.cmdline()
-        resource_dict['user'] = pprocess.username()
-        resource_dict['date'] = datetime.fromtimestamp(pprocess.create_time()).strftime("%Y-%m-%d %H:%M:%S")
-      comsumption_list.append(resource_dict)
-    self.close()
-    return comsumption_list
-  
-  def getPartitionComsumptionStatus(self, partition_id, where=""):
-    self.connect()
-    if where != "":
-      where = " and %s" % where
-    date_scope = datetime.now().strftime('%Y-%m-%d')
-    min_time = (datetime.now() - timedelta(minutes=1)).strftime('%H:%M:00')
-    max_time = (datetime.now() - timedelta(minutes=1)).strftime('%H:%M:59')
-    sql_query = """select count(pid), SUM(cpu_percent), SUM(cpu_time), 
-SUM(cpu_num_threads), SUM(memory_percent), SUM(memory_rss), SUM(io_rw_counter), 
-SUM(io_cycles_counter) from user where 
-date='%s' and partition='%s' and (time between '%s' and '%s') %s""" % (
-        date_scope, partition_id, min_time, max_time, where)
-    query_result = self._execute(sql_query)
-    result_list = zip(*query_result)
-
-    process_dict = memory_dict = io_dict = {}
-    if len(result_list):
-      result = result_list
-      process_dict = {'total_process': result[0][0],
-        'cpu_percent': round((result[1][0] or 0), 2),
-        'cpu_time': round((result[2][0] or 0)/(60.0), 2),
-        'cpu_num_threads': round((result[3][0] or 0), 2),
-        'date': '%s %s' % (date_scope, min_time)
-      }
-      memory_dict = {'memory_percent': round((result[4][0] or 0), 2),
-        'memory_rss': round((result[5][0] or 0)/(1024*1024.0), 2),
-        'date': '%s %s' % (date_scope, min_time)
-      }
-      io_dict = {'io_rw_counter': round((result[6][0] or 0), 2),
-        'io_cycles_counter': round((result[7][0] or 0), 2),
-        'disk_used': 0,
-        'date': '%s %s' % (date_scope, min_time)
-      }
-      if self.has_table('folder'):
-        disk_result_cursor = self.select(
-          "folder", date_scope,
-          columns="SUM(disk_used)", 
-          where="partition='%s' and (time between '%s' and '%s') %s" % (
-            partition_id, min_time, max_time, where
-          )
-        )
-    
-        disk_used_sum = zip(*disk_result_cursor)
-        if len(disk_used_sum) and disk_used_sum[0][0] is not None:
-          io_dict['disk_used'] = round(disk_used_sum[0][0]/1024.0, 2)
-    self.close()
-    return (process_dict, memory_dict, io_dict)
-
-def appendToJsonFile(file_path, content, stepback=2):
-  with open (file_path, mode="r+") as jfile:
-    jfile.seek(0, 2)
-    position = jfile.tell() - stepback
-    jfile.seek(position)
-    jfile.write('%s}' % ',"{}"]'.format(content))
-
-def initProcessDataFile(file_path):
-  with open(process_file, 'w') as fprocess:
-    data_dict = {
-      "date": time.time(),
-      "data": ["date, total process, CPU percent, CPU time, CPU threads"]
-    }
-    fprocess.write(json.dumps(data_dict))
-
-def initMemoryDataFile(file_path):
-  with open(mem_file, 'w') as fmem:
-    data_dict = {
-      "date": time.time(),
-      "data": ["date, memory used percent, memory used"]
-    }
-    fmem.write(json.dumps(data_dict))
-
-def initIODataFile(file_path):
-  with open(io_file, 'w') as fio:
-    data_dict = {
-      "date": time.time(),
-      "data": ["date, io rw counter, io cycles counter, disk used"]
-    }
-    fio.write(json.dumps(data_dict))
-
-if __name__ == "__main__":
-  parser = parseArguments()
-  if not os.path.exists(parser.output_folder) and os.path.isdir(parser.output_folder):
-    raise Exception("Invalid ouput folder: %s" % parser.output_folder)
-
-  # Consumption global status
-  process_file = os.path.join(parser.output_folder, 'monitor_resource_process.data.json')
-  mem_file = os.path.join(parser.output_folder, 'monitor_resource_memory.data.json')
-  io_file = os.path.join(parser.output_folder, 'monitor_resource_io.data.json')
-  resource_file = os.path.join(parser.output_folder, 'monitor_process_resource.status.json')
-  status_file = os.path.join(parser.output_folder, 'monitor_resource.status.json')
-
-  if not os.path.exists(parser.collector_db):
-    print "Collector database not found..."
-    initProcessDataFile(process_file)
-    initMemoryDataFile(mem_file)
-    initIODataFile(io_file)
-    with open(status_file, "w") as status_file:
-      status_file.write('{"cpu_time": 0, "cpu_percent": 0, "memory_rss": 0, "memory_percent": 0, "io_rw_counter": 0, "date": "", "total_process": 0, "disk_used": 0, "io_cycles_counter": 0, "cpu_num_threads": 0}')
-    with open(resource_file, "w") as resource_file:
-      resource_file.write('[]')
-    exit(1)
-
-  collector = RessourceCollect(parser.collector_db)
-
-  date_scope = datetime.now().strftime('%Y-%m-%d')
-  stat_info = os.stat(parser.output_folder)
-  partition_user = pwd.getpwuid(stat_info.st_uid)[0]
-
-  process_result, memory_result, io_result = collector.getPartitionComsumptionStatus(partition_user)
-
-  label_list = ['date', 'total_process', 'cpu_percent', 'cpu_time', 'cpu_num_threads',
-                  'memory_percent', 'memory_rss', 'io_rw_counter', 'io_cycles_counter',
-                  'disk_used']
-  resource_status_dict = {}
-  if not os.path.exists(process_file):
-    initProcessDataFile(process_file)
-
-  if not os.path.exists(mem_file):
-    initMemoryDataFile(mem_file)
-
-  if not os.path.exists(io_file):
-    initIODataFile(io_file)
-
-  if process_result and process_result['total_process'] != 0.0:
-    appendToJsonFile(process_file, ", ".join(
-      [str(process_result[key]) for key in label_list if process_result.has_key(key)])
-    )
-    resource_status_dict.update(process_result)
-  if memory_result and memory_result['memory_rss'] != 0.0:
-    appendToJsonFile(mem_file, ", ".join(
-      [str(memory_result[key]) for key in label_list if memory_result.has_key(key)])
-    )
-    resource_status_dict.update(memory_result)
-  if io_result and io_result['io_rw_counter'] != 0.0:
-    appendToJsonFile(io_file, ", ".join(
-      [str(io_result[key]) for key in label_list if io_result.has_key(key)])
-    )
-    resource_status_dict.update(io_result)
-
-  with open(status_file, 'w') as fp:
-    fp.write(json.dumps(resource_status_dict))
-
-  # Consumption Ressource
-  resource_process_status_list = collector.getPartitionConsumption(partition_user)
-  if resource_process_status_list:
-    with open(resource_file, 'w') as rf:
-      rf.write(json.dumps(resource_process_status_list))

stack/monitor/scripts/globalstate.py

-#!/usr/bin/env python
-
-import sys
-import os
-import glob
-import json
-import ConfigParser
-import time
-from datetime import datetime
-
-def softConfigGet(config, *args, **kwargs):
-  try:
-    return config.get(*args, **kwargs)
-  except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
-    return ""
-
-def generateStatisticsData(stat_file_path, content):
-  # csv document for statictics
-  if not os.path.exists(stat_file_path):
-    with open(stat_file_path, 'w') as fstat:
-      data_dict = {
-        "date": time.time(),
-        "data": ["Date, Success, Error, Warning"]
-      }
-      fstat.write(json.dumps(data_dict))
-
-  current_state = ''
-  if content.has_key('state'):
-    current_state = '%s, %s, %s, %s' % (
-      content['date'],
-      content['state']['success'],
-      content['state']['error'],
-      content['state']['warning'])
-
-  # append to file
-  if current_state:
-    with open (stat_file_path, mode="r+") as fstat:
-      fstat.seek(0,2)
-      position = fstat.tell() -2
-      fstat.seek(position)
-      fstat.write('%s}' % ',"{}"]'.format(current_state))
-
-def main(args_list):
-  monitor_file, instance_file = args_list
-
-  monitor_config = ConfigParser.ConfigParser()
-  monitor_config.read(monitor_file)
-
-  base_folder = monitor_config.get('monitor', 'private-folder')
-  status_folder = monitor_config.get('monitor', 'public-folder')
-  base_url = monitor_config.get('monitor', 'base-url')
-  related_monitor_list = monitor_config.get("monitor", "monitor-url-list").split()
-  statistic_folder = os.path.join(base_folder, 'data', '.jio_documents')
-  parameter_file = os.path.join(base_folder, 'config', '.jio_documents', 'config.json')
-
-  report_date = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
-
-  if not os.path.exists(statistic_folder):
-    try:
-      os.makedirs(statistic_folder)
-    except OSError, e:
-      if e.errno == os.errno.EEXIST and os.path.isdir(statistic_folder):
-        pass
-      else: raise
-
-  # search for all status files
-  file_list = filter(os.path.isfile,
-      glob.glob("%s/*.status.json" % status_folder)
-    )
-  error = warning = success = 0
-  status = 'OK'
-  promise_list = []
-  global_state_file = os.path.join(base_folder, 'monitor.global.json')
-  public_state_file = os.path.join(status_folder, 'monitor.global.json')
-  for file in file_list:
-    try:
-      with open(file, 'r') as temp_file:
-        tmp_json = json.loads(temp_file.read())
-    except ValueError:
-      # bad json file ?
-      continue
-    if tmp_json['status'] == 'ERROR':
-      error  += 1
-    elif tmp_json['status'] == 'OK':
-      success += 1
-    elif tmp_json['status'] == 'WARNING':
-      warning += 1
-    tmp_json['time'] = tmp_json['start-date'].split(' ')[1]
-    promise_list.append(tmp_json)
-
-  if error:
-    status = 'ERROR'
-  elif warning:
-    status = 'WARNING'
-
-  global_state_dict = dict(
-      status=status,
-      state={
-        'error': error,
-        'success': success,
-        'warning': warning,
-      },
-      type='global',
-      date=report_date,
-      _links={"rss_url": {"href": "%s/public/feed" % base_url},
-              "public_url": {"href": "%s/share/jio_public/" % base_url},
-              "private_url": {"href": "%s/share/jio_private/" % base_url}
-            },
-      data={'state': 'monitor_state.data',
-            'process_state': 'monitor_process_resource.status',
-            'process_resource': 'monitor_resource_process.data',
-            'memory_resource': 'monitor_resource_memory.data',
-            'io_resource': 'monitor_resource_io.data',
-            'monitor_process_state': 'monitor_resource.status'}
-    )
-
-  global_state_dict['_embedded'] = {'promises': promise_list}
-
-  if os.path.exists(instance_file):
-    config = ConfigParser.ConfigParser()
-    config.read(instance_file)
-    if 'instance' in config.sections():
-      instance_dict = {}
-      global_state_dict['title'] = config.get('instance', 'name')
-      global_state_dict['hosting-title'] = config.get('instance', 'root-name')
-      if not global_state_dict['title']:
-        global_state_dict['title'] = 'Instance Monitoring'
-      
-      instance_dict['computer'] = config.get('instance', 'computer')
-      instance_dict['ipv4'] = config.get('instance', 'ipv4')
-      instance_dict['ipv6'] = config.get('instance', 'ipv6')
-      instance_dict['software-release'] = config.get('instance', 'software-release')
-      instance_dict['software-type'] = config.get('instance', 'software-type')
-      instance_dict['partition'] = config.get('instance', 'partition')
-
-      global_state_dict['_embedded'].update({'instance' : instance_dict})
-
-  if related_monitor_list:
-    global_state_dict['_links']['related_monitor'] = [{'href': "%s/share/jio_public" % url}
-                          for url in related_monitor_list]
-
-  if os.path.exists(parameter_file):
-    with open(parameter_file) as cfile:
-      global_state_dict['parameters'] = json.loads(cfile.read())
-
-  # Public information with the link to private folder
-  public_state_dict = dict(
-    status=status,
-    date=report_date,
-    _links={'monitor': {'href': '%s/share/jio_private/' % base_url}},
-    title=global_state_dict.get('title', '')
-  )
-  public_state_dict['hosting-title'] = global_state_dict.get('hosting-title', '')
-  public_state_dict['_links']['related_monitor'] = global_state_dict['_links'].get('related_monitor', [])
-
-  with open(global_state_file, 'w') as fglobal:
-    fglobal.write(json.dumps(global_state_dict))
-
-  with open(public_state_file, 'w') as fpglobal:
-    fpglobal.write(json.dumps(public_state_dict))
-
-  generateStatisticsData(
-    os.path.join(statistic_folder, 'monitor_state.data.json'),
-    global_state_dict)
-
-  return 0
-
-if __name__ == "__main__":
-  if len(sys.argv) < 3:
-    print("Usage: %s <monitor_conf_path> <instance_conf_path>" % sys.argv[0])
-    sys.exit(2)
-  sys.exit(main(sys.argv[1:]))

stack/monitor/scripts/monitor-document.py

-#!/usr/bin/env python
-
-import sys
-import os
-import re
-import json
-import argparse
-import subprocess
-from datetime import datetime
-import time
-
-
-def parseArguments():
-  """
-  Parse arguments for monitor instance.
-  """
-  parser = argparse.ArgumentParser()
-  parser.add_argument('--config_folder',
-                      help='Path where json configuration/document will be read and write')
-  parser.add_argument('--htpasswd_bin',
-                      help='Path apache htpasswd binary. Needed to write htpasswd file.')
-  parser.add_argument('--output_cfg_file',
-                      help='Ouput parameters in cfg file.')
-
-  return parser.parse_args()
-
-def fileWrite(file_path, content):
-  if os.path.exists(file_path):
-    try:
-      with open(file_path, 'w') as wf:
-        wf.write(content)
-        return True
-    except OSError, e:
-      print "ERROR while writing changes to %s.\n %s" % (file_path, str(e))
-  return False
-
-def htpasswdWrite(htpasswd_bin,  parameter_dict, value):
-  if not os.path.exists(parameter_dict['file']):
-    return False
-  command = [htpasswd_bin, '-cb', parameter_dict['htpasswd'], parameter_dict['user'], value]
-  process = subprocess.Popen(
-    command,
-    stdin=None,
-    stdout=subprocess.PIPE,
-    stderr=subprocess.PIPE
-  )
-  result = process.communicate()[0]
-  if process.returncode != 0:
-    print result
-    return False
-  with open(parameter_dict['file'], 'w') as pfile:
-    pfile.write(value)
-  return True
-
-def httpdCorsDomainWrite(httpd_cors_file, httpd_gracefull_bin, cors_domain):
-  cors_string = ""
-  cors_domain_list = cors_domain.split()
-  old_httpd_cors_file = os.path.join(
-    os.path.dirname(httpd_cors_file),
-    'prev_%s' % os.path.basename(httpd_cors_file)
-  )
-  if os.path.exists(old_httpd_cors_file) and os.path.isfile(old_httpd_cors_file):
-    try:
-      with open(old_httpd_cors_file, 'r') as cors_file:
-        if cors_file.read() == cors_domain:
-          if os.path.exists(httpd_cors_file) and (os.stat(httpd_cors_file).st_size > 0
-            or (cors_domain == "" and os.stat(httpd_cors_file).st_size == 0)):
-            # Skip if cors file is not empty
-            return True
-    except OSError, e:
-      print "Failed to open file at %s. \n%s" % (old_httpd_cors_file, str(e))
-  for domain in cors_domain_list:
-    if cors_string:
-      cors_string += '|'
-    cors_string += re.escape(domain)
-  try:
-    with open(httpd_cors_file, 'w') as file:
-      file.write('SetEnvIf Origin "^http(s)?://(.+\.)?(%s)$" origin_is=$0\n' % cors_string)
-      file.write('Header always set Access-Control-Allow-Origin %{origin_is}e env=origin_is')
-  except OSError, e:
-    print "ERROR while writing CORS changes to %s.\n %s" % (httpd_cors_file, str(e))
-    return False
-
-  # Save current cors domain list
-  try:
-    with open(old_httpd_cors_file, 'w') as cors_file:
-      cors_file.write(cors_domain)
-  except OSError, e:
-    print "Failed to open file at %s. \n%s" % (old_httpd_cors_file, str(e))
-    return False
-
-  # Restart httpd process
-  try:
-    subprocess.call(httpd_gracefull_bin)
-  except OSError, e:
-    print "Failed to execute command %s.\n %s" % (httpd_gracefull_bin, str(e))
-    return False
-
-def applyEditChage(parser):
-  parameter_tmp_file = os.path.join(parser.config_folder, 'config.tmp.json')
-  config_file = os.path.join(parser.config_folder, 'config.json')
-  parameter_config_file = os.path.join(parser.config_folder, 'config.parameters.json')
-  if not os.path.exists(parameter_tmp_file) or not os.path.isfile(parameter_tmp_file):
-    return {}
-  if not os.path.exists(config_file):
-    print "ERROR: Config file doesn't exist... Exiting"
-    return {}
-
-  new_parameter_list = []
-  parameter_list = []
-  description_dict = {}
-  result_dict = {}
-
-  try:
-    with open(parameter_tmp_file) as tmpfile:
-      new_parameter_list = json.loads(tmpfile.read())
-  except ValueError:
-    print "Error: Couldn't parse json file %s" % parameter_tmp_file
-
-  with open(parameter_config_file) as tmpfile:
-    description_dict = json.loads(tmpfile.read())
-
-  for i in range(0, len(new_parameter_list)):
-    key = new_parameter_list[i]['key']
-    if key != '':
-      description_entry = description_dict[key]
-      if description_entry['type'] == 'file':
-        result_dict[key] = fileWrite(description_entry['file'], new_parameter_list[i]['value'])
-      elif description_entry['type'] == 'htpasswd':
-        result_dict[key] = htpasswdWrite(parser.htpasswd_bin, description_entry, new_parameter_list[i]['value'])
-      elif description_entry['type'] == 'httpdcors':
-        result_dict[key] = httpdCorsDomainWrite(description_entry['cors_file'], description_entry['gracefull_bin'], new_parameter_list[i]['value'])
-
-  if (parser.output_cfg_file):
-    try:
-      with open(parser.output_cfg_file, 'w') as pfile:
-        pfile.write('[public]\n')
-        for parameter in new_parameter_list:
-          if parameter['key']:
-            pfile.write('%s = %s\n' % (parameter['key'], parameter['value']))
-    except OSError, e:
-      print "Error failed to create file %s" % parser.output_cfg_file
-      pass
-
-  return result_dict
-
-if __name__ == "__main__":
-  parser = parseArguments()
-  parameter_tmp_file = os.path.join(parser.config_folder, 'config.tmp.json')
-  config_file = os.path.join(parser.config_folder, 'config.json')
-
-  # Run 4 times with sleep
-  run_counter = 1
-  max_runn = 4
-  sleep_time = 15
-
-  while True:
-    result_dict = applyEditChage(parser)
-    if result_dict != {}:
-      status = True
-      for key in result_dict:
-        if not result_dict[key]:
-          status = False
-    
-      if status and os.path.exists(parameter_tmp_file):
-        try:
-          os.unlink(config_file)
-        except OSError, e:
-          print "ERROR cannot remove file: %s" % parameter_tmp_file
-        else:
-          os.rename(parameter_tmp_file, config_file)
-    if run_counter == max_runn:
-      break
-    else:
-      run_counter += 1
-      time.sleep(sleep_time)
-
-
-
-
-
-
-
-

stack/monitor/scripts/monitor.py

-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import sys
-import os
-import stat
-import json
-import ConfigParser
-import traceback
-import argparse
-import urllib2
-import ssl
-import glob
-from datetime import datetime
-
-OPML_START = """<?xml version="1.0" encoding="UTF-8"?>
-<!-- OPML generated by SlapOS -->
-<opml version="1.1">
-	<head>
-		<title>%(root_title)s</title>
-		<dateCreated>%(creation_date)s</dateCreated>
-		<dateModified>%(modification_date)s</dateModified>
-	</head>
-	<body>
-	  <outline text="%(outline_title)s">"""
-OPML_END = """	  </outline>
-  </body>
-</opml>"""
-
-OPML_OUTLINE_FEED = '<outline text="%(title)s" title="%(title)s" type="rss" version="RSS" htmlUrl="%(html_url)s" xmlUrl="%(xml_url)s" url="%(global_url)s" />'
-
-
-def parseArguments():
-  """
-  Parse arguments for monitor instance.
-  """
-  parser = argparse.ArgumentParser()
-  parser.add_argument('--config_file',
-                      default='monitor.cfg',
-                      help='Monitor Configuration file')
-
-  return parser.parse_args()
-
-
-def mkdirAll(path):
-  try:
-    os.makedirs(path)
-  except OSError, e:
-    if e.errno == os.errno.EEXIST and os.path.isdir(path):
-      pass
-    else: raise
-
-def softConfigGet(config, *args, **kwargs):
-  try:
-    return config.get(*args, **kwargs)
-  except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
-    return None
-
-def createSymlink(source, destination):
-  try:
-    os.symlink(source, destination)
-  except OSError, e:
-    if e.errno != os.errno.EEXIST:
-      raise
-
-class Monitoring(object):
-
-  def __init__(self, configuration_file):
-    config = self.loadConfig([configuration_file])
-
-    # Set Monitor variables
-    self.monitor_hal_json = config.get("monitor", "monitor-hal-json")
-    self.title = config.get("monitor", "title")
-    self.root_title = config.get("monitor", "root-title")
-    self.service_pid_folder = config.get("monitor", "service-pid-folder")
-    self.crond_folder = config.get("monitor", "crond-folder")
-    self.logrotate_d = config.get("monitor", "logrotate-folder")
-    self.promise_runner = config.get("monitor", "promise-runner")
-    self.promise_folder_list = config.get("monitor", "promise-folder-list").split()
-    self.public_folder = config.get("monitor", "public-folder")
-    self.private_folder = config.get("monitor", "private-folder")
-    self.collector_db  = config.get("monitor", "collector-db")
-    self.collect_script = config.get("monitor", "collect-script")
-    self.webdav_folder = config.get("monitor", "webdav-folder")
-    self.report_script_folder = config.get("monitor", "report-folder")
-    self.webdav_url = '%s/share' % config.get("monitor", "base-url")
-    self.public_url = '%s/public' % config.get("monitor", "base-url")
-    self.python = config.get("monitor", "python") or "python"
-    self.public_path_list = config.get("monitor", "public-path-list").split()
-    self.private_path_list = config.get("monitor", "private-path-list").split()
-    self.monitor_url_list = config.get("monitor", "monitor-url-list").split()
-    self.parameter_list = [param.strip() for param in config.get("monitor", "parameter-list").split('\n') if param]
-    # Use this file to write knowledge0_cfg required by webrunner
-    self.parameter_cfg_file = config.get("monitor", "parameter-file-path").strip()
-
-    self.config_folder = os.path.join(self.private_folder, 'config')
-    self.report_folder = self.private_folder
-
-    self.promise_dict = {}
-    for promise_folder in self.promise_folder_list:
-      self.setupPromiseDictFromFolder(promise_folder)
-
-  def loadConfig(self, pathes, config=None):
-    if config is None:
-      config = ConfigParser.ConfigParser()
-    try:
-      config.read(pathes)
-    except ConfigParser.MissingSectionHeaderError:
-      traceback.print_exc()
-    return config
-
-  def readInstanceConfiguration(self):
-    type_list = ['raw', 'file', 'htpasswd', 'httpdcors']
-    configuration_list = []
-
-    if not self.parameter_list:
-      return []
-  
-    for config in self.parameter_list:
-      config_list = config.strip().split(' ')
-      # type: config_list[0]
-      if len(config_list) >= 3 and config_list[0] in type_list:
-        if config_list[0] == 'raw':
-          configuration_list.append(dict(
-            key='',
-            title=config_list[1],
-            value=' '.join(config_list[2:])
-          ))
-        elif (config_list[0] == 'file' or config_list[0] == 'htpasswd')  and \
-            os.path.exists(config_list[2]) and os.path.isfile(config_list[2]):
-          try:
-            with open(config_list[2]) as cfile:
-              parameter = dict(
-                key=config_list[1],
-                title=config_list[1],
-                value=cfile.read(),
-                description={
-                  "type": config_list[0],
-                  "file": config_list[2]
-                }
-              )
-              if config_list[0] == 'htpasswd':
-                if len(config_list) != 5 or not os.path.exists(config_list[4]):
-                  print 'htpasswd file is not specified: %s' % str(config_list)
-                  continue
-                parameter['description']['user'] = config_list[3]
-                parameter['description']['htpasswd'] = config_list[4]
-              configuration_list.append(parameter)
-          except OSError, e:
-            print 'Cannot read file %s, Error is: %s' % (config_list[2], str(e))
-            pass
-        elif config_list[0] == 'httpdcors' and os.path.exists(config_list[2]) and \
-            os.path.exists(config_list[3]):
-          old_cors_file = os.path.join(
-            os.path.dirname(config_list[2]),
-            'prev_%s' % os.path.basename(config_list[2])
-          )
-          try:
-            cors_content = ""
-            if os.path.exists(old_cors_file):
-              with open(old_cors_file) as cfile:
-                cors_content = cfile.read()
-            else:
-              # Create empty file
-              with open(old_cors_file, 'w') as cfile:
-                cfile.write("")
-            parameter = dict(
-              key=config_list[1],
-              title=config_list[1],
-              value=cors_content,
-              description={
-                "type": config_list[0],
-                "cors_file": config_list[2],
-                "gracefull_bin": config_list[3]
-              }
-            )
-            configuration_list.append(parameter)
-          except OSError, e:
-            print 'Cannot read file at %s, Error is: %s' % (old_cors_file, str(e))
-            pass
-    return configuration_list
-
-  def setupPromiseDictFromFolder(self, folder):
-    for filename in os.listdir(folder):
-      path = os.path.join(folder, filename)
-      if os.path.isfile(path) and os.access(path, os.X_OK):
-        self.promise_dict[filename] = {"path": path,
-                                  "configuration": ConfigParser.ConfigParser()}
-
-  def createSymlinksFromConfig(self, destination_folder, source_path_list, name=""):
-    if destination_folder:
-      if source_path_list:
-        for path in source_path_list:
-          path = path.rstrip('/')
-          dirname = os.path.join(destination_folder, name)
-          try:
-            mkdirAll(dirname)  # could also raise OSError
-            os.symlink(path, os.path.join(dirname, os.path.basename(path)))
-          except OSError, e:
-            if e.errno != os.errno.EEXIST:
-              raise
-
-  def getMonitorTitleFromUrl(self, monitor_url):
-    # This file should be generated
-    if not monitor_url.startswith('https://') and not monitor_url.startswith('http://'):
-      return 'Unknown Instance'
-    if not monitor_url.endswith('/'):
-      monitor_url = monitor_url + '/'
-
-    url  = monitor_url + '/.jio_documents/monitor.global.json' # XXX Hard Coded path
-    try:
-      # XXX - working here with public url
-      if hasattr(ssl, '_create_unverified_context'):
-        context = ssl._create_unverified_context()
-        response = urllib2.urlopen(url, context=context)
-      else:
-        response = urllib2.urlopen(url)
-    except urllib2.HTTPError:
-      return 'Unknown Instance'
-    else:
-      try:
-        monitor_dict = json.loads(response.read())
-        return monitor_dict.get('title', 'Unknown Instance')
-      except ValueError, e:
-        print "Bad Json file at %s" % url
-    return 'Unknown Instance'
-
-  def getReportInfoFromFilename(self, filename):
-    splited_filename = filename.split('_every_')
-    possible_time_list = ['hour', 'minute']
-    if len(splited_filename) == 1:
-      return (filename, "* * * * *")
-
-    run_time = splited_filename[1].split('_')
-    report_name = splited_filename[0]
-    if len(run_time) != 2 or not run_time[1] in possible_time_list:
-      return (report_name, "* * * * *")
-
-    try:
-      value = int(run_time[0])
-    except ValueError:
-      print "Warning: Bad report filename: %s" % filename
-      return (report_name, "* * * * *")
-
-    if run_time[1] == 'hour':
-      return (report_name, "* */%s * * *" % value)
-    if run_time[1] == 'minute':
-      return (report_name, "*/%s * * * *" % value)
-
-  def configureFolders(self):
-    # configure public and private folder
-    self.createSymlinksFromConfig(self.webdav_folder, [self.public_folder])
-    self.createSymlinksFromConfig(self.webdav_folder, [self.private_folder])
-
-    #configure jio_documents folder
-    jio_public = os.path.join(self.webdav_folder, 'jio_public')
-    jio_private = os.path.join(self.webdav_folder, 'jio_private')
-    mkdirAll(jio_public)
-    mkdirAll(jio_private)
-
-    createSymlink(self.public_folder,
-                  os.path.join(jio_public, '.jio_documents'))
-    createSymlink(self.private_folder,
-                  os.path.join(jio_private, '.jio_documents'))
-
-    self.data_folder = os.path.join(self.private_folder, 'data', '.jio_documents')
-    self.document_folder = os.path.join(self.private_folder, 'documents')
-    config_folder = os.path.join(self.config_folder, '.jio_documents')
-
-    mkdirAll(self.data_folder)
-    mkdirAll(config_folder)
-
-    createSymlink(os.path.join(self.private_folder, 'data'),
-                  os.path.join(jio_private, 'data'))
-    createSymlink(self.config_folder, os.path.join(jio_private, 'config'))
-    createSymlink(self.data_folder, self.document_folder)
-
-    # Cleanup private folder
-    for file in glob.glob("%s/*.history.json" % self.private_folder):
-      try:
-        os.unlink(file)
-      except OSError:
-        print "failed to remove file %s. Ignoring..." % file
-
-  def makeConfigurationFiles(self):
-    config_folder = os.path.join(self.config_folder, '.jio_documents')
-    parameter_config_file = os.path.join(config_folder, 'config.parameters.json')
-    parameter_file = os.path.join(config_folder, 'config.json')
-    #mkdirAll(config_folder)
-
-    parameter_list = self.readInstanceConfiguration()
-    description_dict = {}
-
-    if parameter_list:
-      for i in range(0, len(parameter_list)):
-        key = parameter_list[i]['key']
-        if key:
-          description_dict[key] = parameter_list[i].pop('description')
-
-    with open(parameter_config_file, 'w') as config_file:
-      config_file.write(json.dumps(description_dict))
-
-    with open(parameter_file, 'w') as config_file:
-      config_file.write(json.dumps(parameter_list))
-
-    try:
-      with open(self.parameter_cfg_file, 'w') as pfile:
-        pfile.write('[public]\n')
-        for parameter in parameter_list:
-          if parameter['key']:
-            pfile.write('%s = %s\n' % (parameter['key'], parameter['value']))
-    except OSError, e:
-      print "Error failed to create file %s" % self.parameter_cfg_file
-      pass
-      
-
-  def generateOpmlFile(self, feed_url_list, output_file):
-
-    if os.path.exists(output_file):
-      creation_date = datetime.fromtimestamp(os.path.getctime(output_file)).utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
-      modification_date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
-    else:
-      creation_date = modification_date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
-
-    opml_content = OPML_START % {'creation_date': creation_date,
-                                  'modification_date': modification_date,
-                                  'outline_title': 'Monitoring RSS Feed list',
-                                  'root_title': self.root_title}
-
-    opml_content += OPML_OUTLINE_FEED % {'title': self.title,
-        'html_url': self.public_url + '/feed',
-        'xml_url': self.public_url + '/feed',
-        'global_url': "%s/jio_private/" % self.webdav_url}
-    for feed_url in feed_url_list:
-      opml_content += OPML_OUTLINE_FEED % {'title': self.getMonitorTitleFromUrl(feed_url + "/share/jio_public/"),
-        'html_url': feed_url + '/public/feed',
-        'xml_url': feed_url + '/public/feed',
-        'global_url': "%s/share/jio_private/" % feed_url}
-
-    opml_content += OPML_END
-
-    with open(output_file, 'w') as wfile:
-      wfile.write(opml_content)
-
-  def generateLogrotateEntry(self, name, file_list, option_list):
-    """
-      Will add a new entry in logrotate.d folder. This can help to rotate data file daily
-    """
-    content = "%(logfiles)s {\n%(options)s\n}\n" % {
-                'logfiles': ' '.join(file_list),
-                'options': '\n'.join(option_list)
-              }
-    file_path = os.path.join(self.logrotate_d, name)
-    with open(file_path, 'w') as flog:
-      flog.write(content)
-
-  def generateReportCronEntries(self):
-    cron_line_list = []
-
-    report_name_list = [name.replace('.report.json', '')
-      for name in os.listdir(self.report_folder) if name.endswith('.report.json')]
-
-    for filename in os.listdir(self.report_script_folder):
-      report_script = os.path.join(self.report_script_folder, filename)
-      if os.path.isfile(report_script) and os.access(report_script, os.X_OK):
-        report_name, frequency = self.getReportInfoFromFilename(filename)
-        # report_name = os.path.splitext(filename)[0]
-        report_json_path = "%s.report.json" % report_name
-
-        report_cmd_line = [
-          frequency,
-          self.promise_runner,
-          '--pid_path "%s"' % os.path.join(self.service_pid_folder,
-            "%s.pid" % filename),
-          '--output "%s"' % os.path.join(self.report_folder,report_json_path),
-          '--promise_script "%s"' % report_script,
-          '--promise_name "%s"' % report_name,
-          '--monitor_url "%s/jio_private/"' % self.webdav_url, # XXX hardcoded,
-          '--history_folder "%s"' % self.data_folder,
-          '--instance_name "%s"' % self.title,
-          '--hosting_name "%s"' % self.root_title,
-          '--promise_type "report"']
-
-        cron_line_list.append(' '.join(report_cmd_line))
-
-      if report_name in report_name_list:
-        report_name_list.pop(report_name_list.index(report_name))
-
-    # cleanup removed report json result
-    if report_name_list != []:
-      for report_name in report_name_list:
-        result_path = os.path.join(self.public_folder, '%s.report.json' % report_name)
-        if os.path.exists(result_path):
-          try:
-            os.unlink(result_path)
-          except OSError, e:
-            print "Error: Failed to delete %s" % result_path, str(e)
-            pass
-
-    with open(self.crond_folder + "/monitor-reports", "w") as freport:
-      freport.write("\n".join(cron_line_list))
-
-  def generateServiceCronEntries(self):
-    # XXX only if at least one configuration file is modified, then write in the cron
-    #cron_line_list = ['PATH=%s\n' % os.environ['PATH']]
-    cron_line_list = []
-
-    service_name_list = [name.replace('.status.json', '')
-      for name in os.listdir(self.public_folder) if name.endswith('.status.json')]
-
-    for service_name, promise in self.promise_items:
-      service_config = promise["configuration"]
-      service_status_path = "%s/%s.status.json" % (self.public_folder, service_name)
-      mkdirAll(os.path.dirname(service_status_path))
-
-      promise_cmd_line = [
-        softConfigGet(service_config, "service", "frequency") or "* * * * *",
-        self.promise_runner,
-        '--pid_path "%s"' % os.path.join(self.service_pid_folder,
-          "%s.pid" % service_name),
-        '--output "%s"' % service_status_path,
-        '--promise_script "%s"' % promise["path"],
-        '--promise_name "%s"' % service_name,
-        '--monitor_url "%s/jio_private/"' % self.webdav_url, # XXX hardcoded,
-        '--history_folder "%s"' % self.public_folder,
-        '--instance_name "%s"' % self.title,
-        '--hosting_name "%s"' % self.root_title]
-
-      cron_line_list.append(' '.join(promise_cmd_line))
-
-      if service_name in service_name_list:
-        service_name_list.pop(service_name_list.index(service_name))
-
-    if service_name_list != []:
-      # XXX Some service was removed, delete his status file so monitor will not consider his status anymore
-      for service_name in service_name_list:
-        status_path = os.path.join(self.public_folder, '%s.status.json' % service_name)
-        if os.path.exists(status_path):
-          try:
-            os.unlink(status_path)
-          except OSError, e:
-            print "Error: Failed to delete %s" % status_path, str(e)
-            pass
-
-    with open(self.crond_folder + "/monitor-promises", "w") as fp:
-      fp.write("\n".join(cron_line_list))
-
-  def addCronEntry(self, name, frequency, command):
-    entry_line = '%s %s' % (frequency, command)
-    cron_entry_file = os.path.join(self.crond_folder, name)
-    with open(cron_entry_file, "w") as cronf:
-      cronf.write(entry_line)
-
-  def bootstrapMonitor(self):
-
-    # create symlinks from monitor.conf
-    self.createSymlinksFromConfig(self.public_folder, self.public_path_list)
-    self.createSymlinksFromConfig(self.private_folder, self.private_path_list)
-
-    self.configureFolders()
-
-    # create symlinks from service configurations
-    self.promise_items = self.promise_dict.items()
-    for service_name, promise in self.promise_items:
-      service_config = promise["configuration"]
-      public_path_list = softConfigGet(service_config, "service", "public-path-list")
-      private_path_list = softConfigGet(service_config, "service", "private-path-list")
-      if public_path_list:
-        self.createSymlinksFromConfig(self.public_folder,
-                                      public_path_list.split(),
-                                      service_name)
-      if private_path_list:
-        self.createSymlinksFromConfig(self.private_folder,
-                                      private_path_list.split(),
-                                      service_name)
-
-    # Generate OPML file
-    self.generateOpmlFile(self.monitor_url_list,
-      os.path.join(self.public_folder, 'feeds'))
-
-    # put promises to a cron file
-    self.generateServiceCronEntries()
-
-    # put report script to cron
-    self.generateReportCronEntries()
-
-    # Generate parameters files and scripts
-    self.makeConfigurationFiles()
-
-    # Rotate monitor data files
-    option_list = [
-      'daily', 'nocreate', 'olddir %s' % self.data_folder, 'rotate 5',
-      'nocompress', 'extension .json', 'dateext',
-      'dateformat -%Y-%m-%d', 'notifempty'
-    ]
-    file_list = [
-      "%s/*.data.json" % self.private_folder,
-      "%s/*.data.json" % self.data_folder]
-    self.generateLogrotateEntry('monitor.data', file_list, option_list)
-
-    # Rotate public history status file, delete data of previous days
-    option_list = [
-      'daily', 'nocreate', 'rotate 0',
-      'nocompress', 'notifempty'
-    ]
-    file_list = ["%s/*.history.json" % self.public_folder]
-    self.generateLogrotateEntry('monitor.service.status', file_list, option_list)
-
-    # Add cron entry for SlapOS Collect
-    command = "%s %s --output_folder %s --collector_db %s" % (self.python,
-      self.collect_script, self.data_folder, self.collector_db)
-    self.addCronEntry('monitor_collect', '* * * * *', command)
-
-    return 0
-
-
-
-if __name__ == "__main__":
-  parser = parseArguments()
-
-  monitor = Monitoring(parser.config_file)
-  
-  sys.exit(monitor.bootstrapMonitor())

stack/monitor/scripts/run-promise.py

-#!{{ python }}
-# -*- coding: utf-8 -*-
-
-import sys
-import os
-import subprocess
-import json
-import psutil
-import time
-from shutil import copyfile
-import glob
-import argparse
-import traceback
-
-def parseArguments():
-  """
-  Parse arguments for monitor collector instance.
-  """
-  parser = argparse.ArgumentParser()
-  parser.add_argument('--pid_path',
-                      help='Path where the pid of this process will be writen.')
-  parser.add_argument('--output',
-                      help='The Path of file where Json result of this promise will be saved.')
-  parser.add_argument('--promise_script',
-                      help='Promise script to execute.')
-  parser.add_argument('--promise_name',
-                      help='Title to give to this promise.')
-  parser.add_argument('--promise_type',
-                      default='status',
-                      help='Type of promise to execute. [status, report].')
-  parser.add_argument('--monitor_url',
-                      help='Monitor Instance website URL.')
-  parser.add_argument('--history_folder',
-                      help='Path where old result file will be placed before generate a new json result file.')
-  parser.add_argument('--instance_name',
-                      default='UNKNOWN Software Instance',
-                      help='Software Instance name.')
-  parser.add_argument('--hosting_name',
-                      default='UNKNOWN Hosting Subscription',
-                      help='Hosting Subscription name.')
-
-  return parser.parse_args()
-
-def main():
-  parser = parseArguments()
-
-  if os.path.exists(parser.pid_path):
-    with open(parser.pid_path, "r") as pidfile:
-      try:
-        pid = int(pidfile.read(6))
-      except ValueError:
-        pid = None
-      if pid and os.path.exists("/proc/" + str(pid)):
-        print("A process is already running with pid " + str(pid))
-        return 1
-  start_date = ""
-  with open(parser.pid_path, "w") as pidfile:
-    process = executeCommand(parser.promise_script)
-    ps_process = psutil.Process(process.pid)
-    start_date = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ps_process.create_time()))
-    pidfile.write(str(process.pid))
-
-  status_json = generateStatusJsonFromProcess(process, start_date=start_date)
-
-  status_json['_links'] = {"monitor": {"href": parser.monitor_url}}
-  status_json['title'] = parser.promise_name
-  status_json['instance'] = parser.instance_name
-  status_json['hosting_subscription'] = parser.hosting_name
-  status_json['type'] = parser.promise_type
-
-  # Save the lastest status change date (needed for rss)
-  status_json['change-time'] = ps_process.create_time()
-  if os.path.exists(parser.output):
-    with open(parser.output) as f:
-      try:
-        last_result = json.loads(f.read())
-        if status_json['status'] == last_result['status'] and last_result.has_key('change-time'):
-          status_json['change-time'] = last_result['change-time']
-      except ValueError:
-        pass
-
-  updateStatusHistoryFolder(
-    parser.promise_name,
-    parser.output,
-    parser.history_folder,
-    parser.promise_type
-  )
-  with open(parser.output, "w") as outputfile:
-    json.dump(status_json, outputfile)
-  os.remove(parser.pid_path)
-
-def updateStatusHistoryFolder(name, status_file, history_folder, promise_type):
-  history_path = os.path.join(history_folder)
-  if not os.path.exists(status_file):
-    return
-  if not os.path.exists(history_folder):
-    return
-  if not os.path.exists(history_path):
-    try:
-      os.makedirs(history_path)
-    except OSError, e:
-      if e.errno == os.errno.EEXIST and os.path.isdir(history_path):
-        pass
-      else: raise
-  with open(status_file, 'r') as sf:
-    try:
-      status_dict = json.loads(sf.read())
-    except ValueError:
-      traceback.print_exc()
-      return
-
-  if promise_type == 'status':
-    filename = '%s.history.json' % name
-    history_file = os.path.join(history_path, filename)
-    # Remove links from history (not needed)
-    status_dict.pop('_links', None)
-    if not os.path.exists(history_file):
-      with open(history_file, 'w') as f_history:
-        data_dict = {
-          "date": time.time(),
-          "data": [status_dict]
-        }
-        f_history.write(json.dumps(data_dict))
-    else:
-      # Remove useless informations
-      status_dict.pop('hosting_subscription', '')
-      status_dict.pop('title', '')
-      status_dict.pop('instance', '')
-      status_dict.pop('type', '')
-
-      with open (history_file, mode="r+") as f_history:
-        f_history.seek(0,2)
-        position = f_history.tell() -2
-        f_history.seek(position)
-        #f_history.write(',%s]}' % str(status_dict))
-        f_history.write('%s}' % ',{}]'.format(json.dumps(status_dict)))
-  elif promise_type == 'report':
-    # keep_item_amount = 3
-    filename = '%s.history.json' % (
-      name)
-
-    copyfile(status_file, os.path.join(history_path, filename))
-    """# Don't let history foler grow too much, keep xx files
-    file_list = filter(os.path.isfile,
-        glob.glob("%s/*.%s.history.json" % (history_path, promise_type))
-      )
-    file_count = len(file_list)
-    if file_count > keep_item_amount:
-      file_list.sort(key=lambda x: os.path.getmtime(x))
-      while file_count > keep_item_amount:
-        to_delete = file_list.pop(0)
-        try:
-          os.unlink(to_delete)
-          file_count -= 1
-        except OSError:
-          raise"""
-
-def generateStatusJsonFromProcess(process, start_date=None, title=None):
-  stdout, stderr = process.communicate()
-  try:
-    status_json = json.loads(stdout)
-  except ValueError:
-    status_json = {}
-  if process.returncode != 0:
-    status_json["status"] = "ERROR"
-  elif not status_json.get("status"):
-    status_json["status"] = "OK"
-  if stderr:
-    status_json["message"] = stderr
-  if start_date:
-    status_json["start-date"] = start_date
-  if title:
-    status_json["title"] = title
-  return status_json
-
-
-def executeCommand(args):
-  return subprocess.Popen(
-    args,
-    #cwd=instance_path,
-    #env=None if sys.platform == 'cygwin' else {},
-    stdin=None,
-    stdout=subprocess.PIPE,
-    stderr=subprocess.PIPE
-  )
-
-if __name__ == "__main__":
-  sys.exit(main())

stack/monitor/scripts/status2rss.py

-import sys
-import os
-import json
-from datetime import datetime
-import base64
-import hashlib
-import PyRSS2Gen
-import argparse
-
-def parseArguments():
-  """
-  Parse arguments for monitor Rss Generator.
-  """
-  parser = argparse.ArgumentParser()
-  parser.add_argument('--items_folder',
-                      help='Path where to get *.status.json files which contain result of promises.')
-  parser.add_argument('--output',
-                      help='The Path of file where feed file will be saved.')
-  parser.add_argument('--feed_url',
-                      help='Url of this feed file.')
-  parser.add_argument('--public_url',
-                      help='Monitor Instance public URL.')
-  parser.add_argument('--private_url',
-                      help='Monitor Instance private URL.')
-  parser.add_argument('--instance_name',
-                      default='UNKNOW Software Instance',
-                      help='Software Instance name.')
-  parser.add_argument('--hosting_name',
-                      default='',
-                      help='Hosting Subscription name.')
-
-  return parser.parse_args()
-
-def getKey(item):
-  return item.pubDate
-
-def main():
-  parser = parseArguments()
-
-  rss_item_list = []
-  report_date = datetime.utcnow()
-  for filename in os.listdir(parser.items_folder):
-    if filename.endswith(".status.json"):
-      filepath = os.path.join(parser.items_folder, filename)
-      result_dict = None
-      try:
-        result_dict = json.load(open(filepath, "r"))
-      except ValueError:
-        print "Failed to load json file: %s" % filepath
-        continue
-      description = result_dict.get('message', '')
-      event_time = datetime.fromtimestamp(result_dict['change-time'])
-      rss_item = PyRSS2Gen.RSSItem(
-        categories = [result_dict['status']],
-        source = PyRSS2Gen.Source(result_dict['title'], parser.public_url),
-        title = '[%s] %s' % (result_dict['status'], result_dict['title']),
-        comments = description,
-        description = "%s: %s\n%s" % (event_time, result_dict['status'], description),
-        link = parser.private_url,
-        pubDate = event_time,
-        guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (parser.hosting_name, result_dict['title'])))
-      )
-      rss_item_list.append(rss_item)
-
-
-  ### Build the rss feed
-  sorted(rss_item_list, key=getKey)
-  rss_feed = PyRSS2Gen.RSS2 (
-    title = parser.instance_name,
-    link = parser.feed_url,
-    description = parser.hosting_name,
-    lastBuildDate = report_date,
-    items = rss_item_list
-    )
-
-  with open(parser.output, 'w') as frss:
-    frss.write(rss_feed.to_xml())
-
-if __name__ == "__main__":
-  exit(main())

stack/slapos.cfg

@@ -108,33 +108,32 @@ zc.recipe.egg = 1.3.2.post5
 hexagonit.recipe.download = 1.7.post4
 
 Jinja2 = 2.8
-PyYAML = 3.11
+PyYAML = 3.12
 Werkzeug = 0.11.10
 buildout-versions = 1.7
 cffi = 1.7.0
 click = 6.6
-cliff = 2.1.0
+cliff = 2.2.0
 cmd2 = 0.6.8
 collective.recipe.template = 1.13
-cryptography = 1.4
+cryptography = 1.5
 decorator = 4.0.10
 idna = 2.1
 inotifyx = 0.2.2
 itsdangerous = 0.24
-lxml = 3.6.0
+lxml = 3.6.4
 meld3 = 1.0.2
 netaddr = 0.7.18
 pbr = 1.10.0
 plone.recipe.command = 1.1
 prettytable = 0.7.2
 psutil = 4.3.0
-pyOpenSSL = 16.0.0
+pyOpenSSL = 16.1.0
 pyasn1 = 0.1.9
-pyparsing = 2.1.5
-pytz = 2016.4
-requests = 2.10.0
+pyparsing = 2.1.8
+pytz = 2016.6.1
+requests = 2.11.1
 setuptools = 19.6.2
-simplejson = 3.8.2
 six = 1.10.0
 slapos.cookbook = 1.0.31
 slapos.core = 1.3.15
@@ -142,7 +141,7 @@ slapos.extension.strip = 0.1
 slapos.libnetworkcache = 0.14.5
 slapos.recipe.build = 0.23
 slapos.recipe.cmmi = 0.2
-stevedore = 1.15.0
+stevedore = 1.17.1
 unicodecsv = 0.14.1
 xml-marshaller = 0.9.7
 
@@ -155,7 +154,7 @@ Flask = 0.11.1
 MarkupSafe = 0.23
 
 # Required by:
-# cryptography==1.4
+# cryptography==1.5
 enum34 = 1.1.6
 
 # Required by:
@@ -163,7 +162,7 @@ enum34 = 1.1.6
 functools32 = 3.2.3.post2
 
 # Required by:
-# cryptography==1.4
+# cryptography==1.5
 ipaddress = 1.0.16
 
 # Required by:
@@ -176,6 +175,7 @@ lock-file = 2.0
 
 # Required by:
 # slapos.core==1.3.15
+# XXX 'slapos node format' raises an exception with netifaces 0.10.5.
 netifaces = 0.10.4
 
 # Required by:
@@ -184,11 +184,11 @@ pycparser = 2.14
 
 # Required by:
 # slapos.core==1.3.15
-supervisor = 3.3.0
+supervisor = 3.3.1
 
 # Required by:
 # slapos.core==1.3.15
-uritemplate = 0.6
+uritemplate = 3.0.0
 
 # Required by:
 # slapos.core==1.3.15