Skip to content

slapos
slapos
Commit e190f434 authored by Rafael Monnerat's avatar Rafael Monnerat
Browse files
Options

Update Release Candidate 

Conflicts:
	software/slapos-master/software.cfg
parents 6e7b8e62 9c3e1921
Hide whitespace changes
Inline Side-by-side
Showing with 506 additions and 2081 deletions
component/apache/apache-backend.conf.in
View file @ e190f434
......@@ -78,7 +78,7 @@ LoadModule headers_module modules/mod_headers.so
LoadModule deflate_module modules/mod_deflate.so
LoadModule filter_module modules/mod_filter.so
AddOutputFilterByType DEFLATE text/cache-manifest text/html text/plain text/css application/hal+json application/json application/x-javascript text/xml application/xml application/rss+xml text/javascript image/svg+xml
AddOutputFilterByType DEFLATE text/cache-manifest text/html text/plain text/css application/hal+json application/json application/x-javascript text/xml application/xml application/rss+xml text/javascript image/svg+xml application/x-font-ttf application/font-woff application/font-woff2 application/x-font-opentype
PidFile "{{ parameter_dict['pid-file'] }}"
ServerAdmin admin@
......
component/apache/buildout.cfg
View file @ e190f434
......@@ -187,5 +187,5 @@ make-targets =
[template-apache-backend-conf]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/apache-backend.conf.in
md5sum = e376fdbe8b19551ec08604e64e9a53b9
md5sum = feef079241bda3407b7ceed5876cb61f
mode = 640
component/curl/buildout.cfg
View file @ e190f434
......@@ -12,8 +12,8 @@ parts =
[curl]
recipe = slapos.recipe.cmmi
url = http://curl.haxx.se/download/curl-7.49.0.tar.bz2
md5sum = 7416aaff4a9210b43edda7615ffa4169
url = http://curl.haxx.se/download/curl-7.50.1.tar.bz2
md5sum = 015f6a0217ca6f2c5442ca406476920b
configure-options =
--disable-static
--disable-ldap
......
component/dropbear/buildout.cfg
View file @ e190f434
......@@ -20,11 +20,9 @@ md5sum = 0284ea239083f04c8b874e08e1aca243
# in order have all patches working.
url = http://matt.ucc.asn.au/dropbear/releases/dropbear-0.53.1.tar.bz2
# NOTE DEFAULT_RECV_WINDOW and RECV_MAX_PAYLOAD_LEN are tweaked to support
# faster network throughput compared to dropbear defaults.
configure-options =
--with-zlib=${zlib:location}
CFLAGS="-DENABLE_SINGLEUSER -D__DIRTY_NO_SHELL_CHECKING -DDEFAULT_RECV_WINDOW=1048576 -DRECV_MAX_PAYLOAD_LEN=524288"
CFLAGS="-DENABLE_SINGLEUSER -D__DIRTY_NO_SHELL_CHECKING"
environment =
CPPFLAGS =-I${zlib:location}/include
......
component/egg-patch/ZODB3-persistent-ghostify-slots.patch 0 → 100644
View file @ e190f434
From d387a425941b37b99355077657edf7a2f117cf47 Mon Sep 17 00:00:00 2001
From: Kirill Smelkov <kirr@nexedi.com>
Date: Thu, 21 Jul 2016 22:34:55 +0300
Subject: [PATCH] persistent: On deactivate release in-slots objects too
( This is backport of https://github.com/zopefoundation/persistent/pull/44
to ZODB-3.10 )
On ._p_deactivate() and ._p_invalidate(), when an object goes to ghost
state, objects referenced by all its attributes, except related to
persistence machinery, are released, this way freeing memory (if they
were referenced only from going-to-ghost object).
That's the idea - an object in ghost state is simply a stub, which loads
its content on first access (via hooking into get/set attr) while
occupying minimal memory in not-yet-loaded state.
However the above is not completely true right now, as currently on
ghostification only object's .__dict__ is released, while in-slots objects
are retained attached to ghost object staying in RAM:
---- 8< ----
from ZODB import DB
from persistent import Persistent
import gc
db = DB(None)
jar = db.open()
class C:
def __init__(self, v):
self.v = v
def __del__(self):
print 'released (%s)' % self.v
class P1(Persistent):
pass
class P2(Persistent):
__slots__ = ('aaa')
p1 = P1()
jar.add(p1)
p1.aaa = C(1)
p2 = P2()
jar.add(p2)
p2.aaa = C(2)
p1._p_invalidate()
# "released (1)" is printed
p2._p_invalidate()
gc.collect()
# "released (2)" is NOT printed <--
---- 8< ----
So teach ghostify() & friends to release objects in slots to free-up
memory when an object goes to ghost state.
NOTE PyErr_Occurred() added after ghostify() calls because
pickle_slotnames() can raise an error, but we do not want to change
ghostify() prototype for backward compatibility reason - as it is used
in cPersistenceCAPIstruct.
( I hit this bug with wendelin.core which uses proxies to load
data from DB to virtual memory manager and then deactivate proxy right
after load has been completed:
https://lab.nexedi.com/nexedi/wendelin.core/blob/f7803634/bigfile/file_zodb.py#L239
https://lab.nexedi.com/nexedi/wendelin.core/blob/f7803634/bigfile/file_zodb.py#L295 )
---
src/persistent/cPersistence.c | 41 +++++++++++++++++++++++++++++++++-
src/persistent/tests/testPersistent.py | 24 ++++++++++++++++++++
2 files changed, 64 insertions(+), 1 deletion(-)
diff --git a/src/persistent/cPersistence.c b/src/persistent/cPersistence.c
index b4a185c..28d1f9a 100644
--- a/src/persistent/cPersistence.c
+++ b/src/persistent/cPersistence.c
@@ -75,6 +75,7 @@ fatal_1350(cPersistentObject *self, const char *caller, const char *detail)
#endif
static void ghostify(cPersistentObject*);
+static PyObject * pickle_slotnames(PyTypeObject *cls);
/* Load the state of the object, unghostifying it. Upon success, return 1.
* If an error occurred, re-ghostify the object and return -1.
@@ -141,7 +142,7 @@ accessed(cPersistentObject *self)
static void
ghostify(cPersistentObject *self)
{
- PyObject **dictptr;
+ PyObject **dictptr, *slotnames;
/* are we already a ghost? */
if (self->state == cPersistent_GHOST_STATE)
@@ -171,6 +172,8 @@ ghostify(cPersistentObject *self)
_estimated_size_in_bytes(self->estimated_size);
ring_del(&self->ring);
self->state = cPersistent_GHOST_STATE;
+
+ /* clear __dict__ */
dictptr = _PyObject_GetDictPtr((PyObject *)self);
if (dictptr && *dictptr)
{
@@ -178,6 +181,38 @@ ghostify(cPersistentObject *self)
*dictptr = NULL;
}
+ /* clear all slots besides _p_* */
+ slotnames = pickle_slotnames(Py_TYPE(self));
+ if (slotnames && slotnames != Py_None)
+ {
+ int i;
+
+ for (i = 0; i < PyList_GET_SIZE(slotnames); i++)
+ {
+ PyObject *name;
+ char *cname;
+ int is_special;
+
+ name = PyList_GET_ITEM(slotnames, i);
+ if (PyBytes_Check(name))
+ {
+ cname = PyBytes_AS_STRING(name);
+ is_special = !strncmp(cname, "_p_", 3);
+ if (is_special) /* skip persistent */
+ {
+ continue;
+ }
+ }
+
+ /* NOTE: this skips our delattr hook */
+ if (PyObject_GenericSetAttr((PyObject *)self, name, NULL) < 0)
+ /* delattr of non-set slot will raise AttributeError - we
+ * simply ignore. */
+ PyErr_Clear();
+ }
+ }
+ Py_XDECREF(slotnames);
+
/* We remove the reference to the just ghosted object that the ring
* holds. Note that the dictionary of oids->objects has an uncounted
* reference, so if the ring's reference was the only one, this frees
@@ -261,6 +296,8 @@ Per__p_deactivate(cPersistentObject *self)
called directly. Methods that override this need to
do the same! */
ghostify(self);
+ if (PyErr_Occurred())
+ return NULL;
}
Py_INCREF(Py_None);
@@ -289,6 +326,8 @@ Per__p_invalidate(cPersistentObject *self)
if (Per_set_changed(self, NULL) < 0)
return NULL;
ghostify(self);
+ if (PyErr_Occurred())
+ return NULL;
}
Py_INCREF(Py_None);
return Py_None;
diff --git a/src/persistent/tests/testPersistent.py b/src/persistent/tests/testPersistent.py
index 51e0382..fdb8b67 100644
--- a/src/persistent/tests/testPersistent.py
+++ b/src/persistent/tests/testPersistent.py
@@ -180,6 +180,30 @@ class PersistenceTest(unittest.TestCase):
self.assertEqual(obj._p_changed, None)
self.assertEqual(obj._p_state, GHOST)
+ def test__p_invalidate_from_changed_w_slots(self):
+ from persistent import Persistent
+ class Derived(Persistent):
+ __slots__ = ('myattr1', 'myattr2')
+ def __init__(self):
+ self.myattr1 = 'value1'
+ self.myattr2 = 'value2'
+ obj = Derived()
+ jar = self._makeJar()
+ jar.add(obj)
+ obj._p_activate()
+ obj._p_changed = True
+ jar._loaded = []
+ jar._registered = []
+ self.assertEqual(Derived.myattr1.__get__(obj), 'value1')
+ self.assertEqual(Derived.myattr2.__get__(obj), 'value2')
+ obj._p_invalidate()
+ self.assertIs(obj._p_changed, None)
+ self.assertEqual(list(jar._loaded), [])
+ self.assertRaises(AttributeError, lambda: Derived.myattr1.__get__(obj))
+ self.assertRaises(AttributeError, lambda: Derived.myattr2.__get__(obj))
+ self.assertEqual(list(jar._loaded), [])
+ self.assertEqual(list(jar._registered), [])
+
def test_initial_serial(self):
NOSERIAL = "\000" * 8
obj = self._makeOne()
--
2.9.2.701.gf965a18.dirty
component/flex/buildout.cfg
View file @ e190f434
[buildout]
extends =
../m4/buildout.cfg
../xz-utils/buildout.cfg
parts =
flex
[flex]
recipe = slapos.recipe.cmmi
url = http://downloads.sourceforge.net/project/flex/flex/flex-2.5.35/flex-2.5.35.tar.gz
md5sum = 201d3f38758d95436cbc64903386de0b
url = http://downloads.sourceforge.net/project/flex/flex-2.6.0.tar.xz
md5sum = 3cbbfa1554d0b75fad9f8100732454de
environment =
M4=${m4:location}/bin/m4
PATH=${xz-utils:location}/bin:%(PATH)s
component/fontconfig/buildout.cfg
View file @ e190f434
......@@ -13,8 +13,8 @@ parts =
[fontconfig]
recipe = slapos.recipe.cmmi
url = http://fontconfig.org/release/fontconfig-2.11.1.tar.bz2
md5sum = 824d000eb737af6e16c826dd3b2d6c90
url = http://fontconfig.org/release/fontconfig-2.12.1.tar.bz2
md5sum = b5af5a423ee3b5cfc34846838963c058
pkg_config_depends = ${freetype:pkg_config_depends}:${freetype:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig
# XXX-Cedric : should we use --with-add-fonts={somefont:location}/share,{someotherfont:location}/share?
configure-options =
......
component/gcc/buildout.cfg
View file @ e190f434
......@@ -44,8 +44,8 @@ environment =
[gcc-common]
recipe = slapos.recipe.cmmi
url = http://ftp.gnu.org/gnu/gcc/gcc-5.3.0/gcc-5.3.0.tar.bz2
md5sum = c9616fd448f980259c31de613e575719
url = http://ftp.gnu.org/gnu/gcc/gcc-5.4.0/gcc-5.4.0.tar.bz2
md5sum = 4c626ac2a83ef30dfb9260e6f59c2b30
# make install does not work when several core are used
make-targets = install -j1
......
component/gettext/buildout.cfg
View file @ e190f434
......@@ -10,8 +10,8 @@ extends =
[gettext]
recipe = slapos.recipe.cmmi
url = http://ftp.gnu.org/pub/gnu/gettext/gettext-0.19.6.tar.lz
md5sum = 45b2a123cdc7cef54df98152a0da3fcc
url = http://ftp.gnu.org/pub/gnu/gettext/gettext-0.19.8.1.tar.lz
md5sum = d838d2c4144261d0c5fbab4a0aceb5c1
configure-options =
--disable-static
......
component/gmp/buildout.cfg
View file @ e190f434
......@@ -8,9 +8,9 @@ parts =
[gmp]
recipe = slapos.recipe.cmmi
version = 6.1.0
version = 6.1.1
url = https://gmplib.org/download/gmp/gmp-${:version}.tar.xz
md5sum = a9868ef2556ad6a2909babcd1428f3c7
md5sum = e70e183609244a332d80529e7e155a35
configure-options =
--enable-cxx
--disable-static
......
component/grep/buildout.cfg
View file @ e190f434
......@@ -8,8 +8,8 @@ parts =
[grep]
recipe = slapos.recipe.cmmi
url = http://ftp.gnu.org/gnu/grep/grep-2.21.tar.xz
md5sum = 43c48064d6409862b8a850db83c8038a
url = http://ftp.gnu.org/gnu/grep/grep-2.25.tar.xz
md5sum = 04e96b0e6f0fe6a180ae62c88fcd0af6
environment =
PATH=${xz-utils:location}/bin:%(PATH)s
CPPFLAGS=-I${pcre:location}/include
......
component/groonga/buildout.cfg
View file @ e190f434
......@@ -14,11 +14,12 @@ extends =
[groonga]
recipe = slapos.recipe.cmmi
url = http://packages.groonga.org/source/groonga/groonga-6.0.2.tar.gz
md5sum = 10fde26350296a479953fd28679f2566
url = http://packages.groonga.org/source/groonga/groonga-6.0.8.tar.gz
md5sum = 42c78baddfca42d3a19fe8b06fac821d
# temporary patch to respect more tokens in natural language mode.
patches =
${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b
${:_profile_base_location_}/groonga-6.0.8-bugfix04107.patch#7618361b006abdbcceb1fa1df0bd2136
patch-options = -p1
configure-options =
--disable-static
......
component/groonga/groonga-6.0.8-bugfix04107.patch 0 → 100644
View file @ e190f434
diff --git a/lib/ii.c b/lib/ii.c
index 97b34ec..9523cf7 100644
--- a/lib/ii.c
+++ b/lib/ii.c
@@ -5110,14 +5110,18 @@ grn_ii_cursor_next_internal(grn_ctx *ctx, grn_ii_cursor *c,
c->pb.rid = 0;
if (br->jump > 0 && !BUFFER_REC_DELETED(br)) {
buffer_rec *jump_br = BUFFER_REC_AT(c->buf, br->jump);
- uint8_t *jump_bp;
- uint32_t jump_rid;
- jump_bp = GRN_NEXT_ADDR(jump_br);
- GRN_B_DEC(jump_rid, jump_bp);
- if (jump_rid < c->min) {
- c->nextb = br->jump;
- } else {
+ if (BUFFER_REC_DELETED(jump_br)) {
c->nextb = br->step;
+ } else {
+ uint8_t *jump_bp;
+ uint32_t jump_rid;
+ jump_bp = GRN_NEXT_ADDR(jump_br);
+ GRN_B_DEC(jump_rid, jump_bp);
+ if (jump_rid < c->min) {
+ c->nextb = br->jump;
+ } else {
+ c->nextb = br->step;
+ }
}
} else {
c->nextb = br->step;
component/gzip/buildout.cfg
View file @ e190f434
[buildout]
extends =
../patch/buildout.cfg
../xz-utils/buildout.cfg
parts =
gzip
[gzip]
recipe = slapos.recipe.cmmi
url = ftp://ftp.gnu.org/pub/gnu/gzip/gzip-1.6.tar.xz
md5sum = da981f86677d58a106496e68de6f8995
url = ftp://ftp.gnu.org/pub/gnu/gzip/gzip-1.8.tar.xz
md5sum = f7caabb65cddc1a4165b398009bd05b9
environment =
PATH=${patch:location}/bin:${xz-utils:location}/bin:%(PATH)s
patch-options = -p1
# The --rsyncable patch is from debian/ubuntu,
# specifically https://launchpad.net/ubuntu/+source/gzip/1.6-3ubuntu1
# It is required to minimize the bandwidth used by rsync.
# For an explanation, see http://beeznest.wordpress.com/2005/02/03/rsyncable-gzip/
# Hunks for .texi files have been removed to avoid a dependency on makeinfo.
patches =
${:_profile_base_location_}/rsyncable.diff#0587af03a5580e2b7b4007469ee2b601
PATH=${xz-utils:location}/bin:%(PATH)s
component/gzip/rsyncable.diff deleted 100644 → 0
View file @ 6e7b8e62
--- a/deflate.c
+++ b/deflate.c
@@ -131,6 +131,14 @@
#endif
/* Matches of length 3 are discarded if their distance exceeds TOO_FAR */
+#ifndef RSYNC_WIN
+# define RSYNC_WIN 8192
+#endif
+/* Size of rsync window, must be < MAX_DIST */
+
+#define RSYNC_SUM_MATCH(sum) (((sum) & (RSYNC_WIN - 1)) == 0)
+/* Whether window sum matches magic value */
+
/* ===========================================================================
* Local data used by the "longest match" routines.
*/
@@ -212,6 +220,8 @@
unsigned good_match;
/* Use a faster search when the previous match is longer than this */
+local ulg rsync_sum; /* rolling sum of rsync window */
+local ulg rsync_chunk_end; /* next rsync sequence point */
/* Values for max_lazy_match, good_match and max_chain_length, depending on
* the desired pack level (0..9). The values given below have been tuned to
@@ -314,6 +324,10 @@
#endif
/* prev will be initialized on the fly */
+ /* rsync params */
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ rsync_sum = 0;
+
/* Set the default configuration parameters:
*/
max_lazy_match = configuration_table[pack_level].max_lazy;
@@ -550,6 +564,8 @@
memcpy((char*)window, (char*)window+WSIZE, (unsigned)WSIZE);
match_start -= WSIZE;
strstart -= WSIZE; /* we now have strstart >= MAX_DIST: */
+ if (rsync_chunk_end != 0xFFFFFFFFUL)
+ rsync_chunk_end -= WSIZE;
block_start -= (long) WSIZE;
@@ -579,13 +595,44 @@
}
}
+local void rsync_roll(unsigned start, unsigned num)
+{
+ unsigned i;
+
+ if (start < RSYNC_WIN) {
+ /* before window fills. */
+ for (i = start; i < RSYNC_WIN; i++) {
+ if (i == start + num) return;
+ rsync_sum += (ulg)window[i];
+ }
+ num -= (RSYNC_WIN - start);
+ start = RSYNC_WIN;
+ }
+
+ /* buffer after window full */
+ for (i = start; i < start+num; i++) {
+ /* New character in */
+ rsync_sum += (ulg)window[i];
+ /* Old character out */
+ rsync_sum -= (ulg)window[i - RSYNC_WIN];
+ if (rsync_chunk_end == 0xFFFFFFFFUL && RSYNC_SUM_MATCH(rsync_sum))
+ rsync_chunk_end = i;
+ }
+}
+
+/* ===========================================================================
+ * Set rsync_chunk_end if window sum matches magic value.
+ */
+#define RSYNC_ROLL(s, n) \
+ do { if (rsync) rsync_roll((s), (n)); } while(0)
+
/* ===========================================================================
* Flush the current block, with given end-of-file flag.
* IN assertion: strstart is set to the end of the current match.
*/
#define FLUSH_BLOCK(eof) \
flush_block(block_start >= 0L ? (char*)&window[(unsigned)block_start] : \
- (char*)NULL, (long)strstart - block_start, (eof))
+ (char*)NULL, (long)strstart - block_start, flush-1, (eof))
/* ===========================================================================
* Processes a new input file and return its compressed length. This
@@ -596,7 +643,7 @@
local off_t deflate_fast()
{
IPos hash_head; /* head of the hash chain */
- int flush; /* set if current block must be flushed */
+ int flush = 0; /* set if current block must be flushed, 2=>and padded */
unsigned match_length = 0; /* length of best match */
prev_length = MIN_MATCH-1;
@@ -626,7 +673,8 @@
lookahead -= match_length;
- /* Insert new strings in the hash table only if the match length
+ RSYNC_ROLL(strstart, match_length);
+ /* Insert new strings in the hash table only if the match length
* is not too large. This saves time but degrades compression.
*/
if (match_length <= max_insert_length) {
@@ -654,9 +702,14 @@
/* No match, output a literal byte */
Tracevv((stderr,"%c",window[strstart]));
flush = ct_tally (0, window[strstart]);
+ RSYNC_ROLL(strstart, 1);
lookahead--;
strstart++;
}
+ if (rsync && strstart > rsync_chunk_end) {
+ flush = 2;
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ }
if (flush) FLUSH_BLOCK(0), block_start = strstart;
/* Make sure that we always have enough lookahead, except
@@ -679,7 +732,7 @@
{
IPos hash_head; /* head of hash chain */
IPos prev_match; /* previous match */
- int flush; /* set if current block must be flushed */
+ int flush = 0; /* set if current block must be flushed */
int match_available = 0; /* set if previous match exists */
register unsigned match_length = MIN_MATCH-1; /* length of best match */
@@ -730,6 +783,7 @@
*/
lookahead -= prev_length-1;
prev_length -= 2;
+ RSYNC_ROLL(strstart, prev_length+1);
do {
strstart++;
INSERT_STRING(strstart, hash_head);
@@ -742,24 +796,38 @@
match_available = 0;
match_length = MIN_MATCH-1;
strstart++;
- if (flush) FLUSH_BLOCK(0), block_start = strstart;
+ if (rsync && strstart > rsync_chunk_end) {
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ flush = 2;
+ }
+ if (flush) FLUSH_BLOCK(0), block_start = strstart;
} else if (match_available) {
/* If there was no match at the previous position, output a
* single literal. If there was a match but the current match
* is longer, truncate the previous match to a single literal.
*/
Tracevv((stderr,"%c",window[strstart-1]));
- if (ct_tally (0, window[strstart-1])) {
- FLUSH_BLOCK(0), block_start = strstart;
- }
+ flush = ct_tally (0, window[strstart-1]);
+ if (rsync && strstart > rsync_chunk_end) {
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ flush = 2;
+ }
+ if (flush) FLUSH_BLOCK(0), block_start = strstart;
+ RSYNC_ROLL(strstart, 1);
strstart++;
lookahead--;
} else {
/* There is no previous match to compare with, wait for
* the next step to decide.
*/
+ if (rsync && strstart > rsync_chunk_end) {
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ flush = 2;
+ FLUSH_BLOCK(0), block_start = strstart;
+ }
match_available = 1;
+ RSYNC_ROLL(strstart, 1);
strstart++;
lookahead--;
}
--- a/gzip.c
+++ b/gzip.c
@@ -215,6 +215,7 @@
unsigned insize; /* valid bytes in inbuf */
unsigned inptr; /* index of next byte to be processed in inbuf */
unsigned outcnt; /* bytes in output buffer */
+int rsync = 0; /* make ryncable chunks */
static int handled_sig[] =
{
@@ -275,7 +276,7 @@
{"best", 0, 0, '9'}, /* compress better */
{"lzw", 0, 0, 'Z'}, /* make output compatible with old compress */
{"bits", 1, 0, 'b'}, /* max number of bits per code (implies -Z) */
-
+ {"rsyncable", 0, 0, 'R'}, /* make rsync-friendly archive */
{ 0, 0, 0, 0 }
};
@@ -359,6 +360,7 @@
" -Z, --lzw produce output compatible with old compress",
" -b, --bits=BITS max number of bits per code (implies -Z)",
#endif
+ " --rsyncable Make rsync-friendly archive",
"",
"With no FILE, or when FILE is -, read standard input.",
"",
@@ -489,8 +491,11 @@
#else
recursive = 1;
#endif
- break;
- case 'S':
+ break;
+ case 'R':
+ rsync = 1; break;
+
+ case 'S':
#ifdef NO_MULTIPLE_DOTS
if (*optarg == '.') optarg++;
#endif
--- a/gzip.h
+++ b/gzip.h
@@ -140,6 +140,7 @@
extern unsigned insize; /* valid bytes in inbuf */
extern unsigned inptr; /* index of next byte to be processed in inbuf */
extern unsigned outcnt; /* bytes in output buffer */
+extern int rsync; /* deflate into rsyncable chunks */
extern off_t bytes_in; /* number of input bytes */
extern off_t bytes_out; /* number of output bytes */
@@ -287,7 +288,7 @@
/* in trees.c */
extern void ct_init (ush *attr, int *method);
extern int ct_tally (int dist, int lc);
-extern off_t flush_block (char *buf, ulg stored_len, int eof);
+extern off_t flush_block (char *buf, ulg stored_len, int pad, int eof);
/* in bits.c */
extern void bi_init (file_t zipfile);
--- a/gzip.1
+++ b/gzip.1
@@ -5,6 +5,7 @@
.ll +8
.B gzip
.RB [ " \-acdfhklLnNrtvV19 " ]
+.RB [ --rsyncable ]
.RB [ \-S\ suffix ]
[
.I "name \&..."
@@ -287,6 +288,16 @@
.I gunzip
).
.TP
+.B --rsyncable
+While compressing, synchronize the output occasionally based on the input.
+This increases size by less than 1 percent most cases, but means that the
+.BR rsync (1)
+program can take advantage of similarities in the uncompressed input
+when syncronizing two files compressed with this flag.
+.I gunzip
+cannot tell the difference between a compressed file created with this option,
+and one created without it.
+.TP
.B \-S .suf --suffix .suf
When compressing, use suffix .suf instead of .gz.
Any non-empty suffix can be given, but suffixes
--- a/trees.c
+++ b/trees.c
@@ -856,9 +856,10 @@
* trees or store, and output the encoded block to the zip file. This function
* returns the total compressed length for the file so far.
*/
-off_t flush_block(buf, stored_len, eof)
+off_t flush_block(buf, stored_len, pad, eof)
char *buf; /* input block, or NULL if too old */
ulg stored_len; /* length of input block */
+ int pad; /* pad output to byte boundary */
int eof; /* true if this is the last block for a file */
{
ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */
@@ -951,6 +952,10 @@
Assert (input_len == bytes_in, "bad input size");
bi_windup();
compressed_len += 7; /* align on byte boundary */
+ } else if (pad && (compressed_len % 8) != 0) {
+ send_bits((STORED_BLOCK<<1)+eof, 3); /* send block type */
+ compressed_len = (compressed_len + 3 + 7) & ~7L;
+ copy_block(buf, 0, 1); /* with header */
}
return compressed_len >> 3;
component/imagemagick/buildout.cfg
View file @ e190f434
......@@ -25,9 +25,9 @@ extends =
[imagemagick]
recipe = slapos.recipe.cmmi
version = 7.0.1-1
version = 7.0.2-10
url = https://www.imagemagick.org/download/releases/ImageMagick-${:version}.tar.xz
md5sum = 24673d00fcc8aa00313eeca6aa20fd3c
md5sum = e1cb23d9c10a8eff228ef30ee281711a
pkg_config_depends = ${fontconfig:location}/lib/pkgconfig:${fontconfig:pkg_config_depends}:${lcms2:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig
configure-options =
--disable-static
......@@ -58,8 +58,8 @@ configure-options =
--with-frozenpaths
patch-options = -p1
patches =
${:_profile_base_location_}/imagemagick-6.6.6-1-no-gsx-gsc-probe.patch#3f28ecd9f6722cf2c3238ce6ec3d7a68
${:_profile_base_location_}/safe_policy.patch#07889fefbd9b55f19b9136ed6c17aa8c
${:_profile_base_location_}/imagemagick-7.0.2-10-no-gsx-gsc-probe.patch#64898455d5175efedd1a7bef9f1f18b5
${:_profile_base_location_}/safe_policy.patch#383c0392de7257c9dff7270973342914
environment =
PATH=${freetype:location}/bin:${ghostscript:location}/bin:${inkscape:location}/bin:${libxml2:location}/bin:${patch:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${:pkg_config_depends}
......
component/imagemagick/imagemagick-6.6.6-1-no-gsx-gsc-probe.patch → component/imagemagick/imagemagick-7.0.2-10-no-gsx-gsc-probe.patch
View file @ e190f434
--- ImageMagick-6.6.6-1/configure.ac~ 2010-11-28 21:51:05.000000000 +0100
+++ ImageMagick-6.6.6-1/configure.ac 2010-12-03 14:13:14.000000000 +0100
@@ -2791,7 +2791,7 @@
--- ImageMagick-7.0.2-10/configure.ac.orig 2016-08-30 11:33:39.160279386 +0200
+++ ImageMagick-7.0.2-10/configure.ac 2016-08-30 11:35:34.753290590 +0200
@@ -3110,7 +3110,7 @@
AC_PATH_PROG(MrSIDDecodeDelegate, "$MrSIDDecodeDelegateDefault", "$MrSIDDecodeDelegateDefault")
AC_PATH_PROG(MVDelegate, "$MVDelegateDefault", "$MVDelegateDefault")
AC_PATH_PROG(PCLDelegate, "$PCLDelegateDefault", "$PCLDelegateDefault")
AC_PATH_PROG(PGPDecodeDelegate, "$PGPDecodeDelegateDefault", "$PGPDecodeDelegateDefault")
AC_PATH_PROG(POVDelegate, "$POVDelegateDefault", "$POVDelegateDefault")
-AC_PATH_PROGS(PSDelegate, gsx gsc "$PSDelegateDefault", "$PSDelegateDefault")
+AC_PATH_PROGS(PSDelegate, "$PSDelegateDefault", "$PSDelegateDefault")
AC_PATH_PROG(RLEEncodeDelegate, "$RLEEncodeDelegateDefault", "$RLEEncodeDelegateDefault")
AC_PATH_PROG(RMDelegate, "$RMDelegateDefault", "$RMDelegateDefault")
AC_PATH_PROG(RSVGDecodeDelegate, "$RSVGDecodeDelegateDefault", "$RSVGDecodeDelegateDefault")
--- ImageMagick-6.6.6-1/configure~ 2010-11-28 23:27:16.000000000 +0100
+++ ImageMagick-6.6.6-1/configure 2010-12-03 14:13:57.000000000 +0100
@@ -30931,7 +30931,7 @@
fi
-for ac_prog in gsx gsc "$PSDelegateDefault"
+for ac_prog in "$PSDelegateDefault"
do
# Extract the first word of "$ac_prog", so it can be a program name with args.
set dummy $ac_prog; ac_word=$2
AC_PATH_PROG(SVGDecodeDelegate, "$SVGDecodeDelegateDefault", "$SVGDecodeDelegateDefault")
component/imagemagick/safe_policy.patch
View file @ e190f434
--- ImageMagick-7.0.1-1/config/policy.xml.orig 2016-05-04 00:18:54.000000000 +0200
+++ ImageMagick-7.0.1-1/config/policy.xml 2016-05-06 08:22:30.002045742 +0200
@@ -50,16 +50,25 @@
--- ImageMagick-7.0.2-10/config/policy.xml.orig 2016-08-30 11:37:29.110253211 +0200
+++ ImageMagick-7.0.2-10/config/policy.xml 2016-08-30 11:40:09.719555899 +0200
@@ -50,19 +50,28 @@
-->
<policymap>
<!-- <policy domain="resource" name="temporary-path" value="/tmp"/> -->
......@@ -15,6 +15,9 @@
- <!-- <policy domain="resource" name="throttle" value="0"/> -->
- <!-- <policy domain="resource" name="time" value="3600"/> -->
- <!-- <policy domain="system" name="precision" value="6"/> -->
- <!-- <policy domain="coder" rights="none" pattern="MVG" /> -->
- <!-- <policy domain="delegate" rights="none" pattern="HTTPS" /> -->
- <!-- <policy domain="path" rights="none" pattern="@*" /> -->
+ <policy domain="resource" name="memory" value="2GiB"/>
+ <policy domain="resource" name="map" value="4GiB"/>
+ <policy domain="resource" name="width" value="10MP"/>
......@@ -26,7 +29,10 @@
+ <policy domain="resource" name="throttle" value="0"/>
+ <policy domain="resource" name="time" value="3600"/>
+ <policy domain="system" name="precision" value="6"/>
<policy domain="cache" name="shared-secret" value="passphrase"/>
+ <policy domain="coder" rights="none" pattern="MVG" />
+ <policy domain="delegate" rights="none" pattern="HTTPS" />
+ <policy domain="path" rights="none" pattern="@*" />
<policy domain="cache" name="shared-secret" value="passphrase" stealth="true"/>
+ <policy domain="coder" rights="none" pattern="EPHEMERAL" />
+ <policy domain="coder" rights="none" pattern="HTTPS" />
+ <policy domain="coder" rights="none" pattern="MSL" />
......
component/inkscape/buildout.cfg
View file @ e190f434
......@@ -14,6 +14,7 @@ extends =
../libsigc/buildout.cfg
../libxml2/buildout.cfg
../libxslt/buildout.cfg
../patch/buildout.cfg
../perl/buildout.cfg
../pkgconfig/buildout.cfg
../popt/buildout.cfg
......@@ -35,6 +36,11 @@ environment =
recipe = slapos.recipe.cmmi
url = https://inkscape.org/en/gallery/item/3860/inkscape-0.91.tar.bz2
md5sum = 278dfa4514adcde23546370ec2c84581
patch-options = -p0
# based on
# http://bazaar.launchpad.net/~inkscape.dev/inkscape/trunk/revision/15017
patches =
${:_profile_base_location_}/inkscape-0.91_gcc6.patch#ce3ddbb90f7e5e81fd322469194b6c3c
pkg_config_depends = ${gtkmm:location}/lib/pkgconfig:${gtkmm:pkg_config_depends}:${gsl:location}/lib/pkgconfig:${popt:location}/lib/pkgconfig:${garbage-collector:location}/lib/pkgconfig:${libxslt:location}/lib/pkgconfig
configure-options =
--disable-static
......@@ -45,7 +51,7 @@ configure-options =
--disable-cdr
--without-gnome-vfs
environment =
PATH=${freetype:location}/bin:${gdk-pixbuf:location}/bin:${gettext:location}/bin:${glib:location}/bin:${intltool:location}/bin:${libxml2:location}/bin:${pkgconfig:location}/bin:${pango:location}/bin:${perl:location}/bin:%(PATH)s
PATH=${freetype:location}/bin:${gdk-pixbuf:location}/bin:${gettext:location}/bin:${glib:location}/bin:${intltool:location}/bin:${libxml2:location}/bin:${pango:location}/bin:${patch:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${:pkg_config_depends}
CPPFLAGS=-I${boost-lib:location}/include -I${cairo:location}/include -I${garbage-collector:location}/include -I${libpng:location}/include -I${popt:location}/include -I${zlib:location}/include
# rpath seems not taken from pkgconfig...
......
component/inkscape/inkscape-0.91_gcc6.patch 0 → 100644
View file @ e190f434
--- src/ui/dialog/layer-properties.cpp 2014-12-21 21:58:32 +0000
+++ src/ui/dialog/layer-properties.cpp 2016-07-18 17:19:25 +0000
@@ -146,7 +146,7 @@
destroy_();
Glib::signal_idle().connect(
sigc::bind_return(
- sigc::bind(sigc::ptr_fun(&::operator delete), this),
+ sigc::bind(sigc::ptr_fun<void*, void>(&::operator delete), this),
false
)
);
component/kumo/buildout.cfg
View file @ e190f434
[buildout]
extends =
../gcc/buildout.cfg
../tokyocabinet/buildout.cfg
../messagepack/buildout.cfg
../openssl/buildout.cfg
......@@ -24,5 +25,6 @@ configure-options =
environment =
CPPFLAGS=-I${zlib:location}/include -I${openssl:location}/include
LDFLAGS=-L${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${tokyocabinet:location}/lib -Wl,-rpath=${messagepack:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib
PATH=${patch:location}/bin:%(PATH)s
LD_LIBRARY_PATH=${gcc-fortran:location}/lib:${gcc-fortran:location}/lib64
LDFLAGS=-Wl,-rpath=${gcc-fortran:location}/lib -Wl,-rpath=${gcc-fortran:location}/lib64 -L${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${tokyocabinet:location}/lib -Wl,-rpath=${messagepack:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib
PATH=${gcc-fortran:location}/bin:${patch:location}/bin:%(PATH)s
component/mariadb/buildout.cfg
View file @ e190f434
......@@ -22,9 +22,9 @@ parts =
[mariadb]
recipe = slapos.recipe.cmmi
version = 10.1.14
version = 10.1.17
url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http:/ftp.osuosl.org/pub/mariadb/?serve
md5sum = 294925531e0fd2f0461e3894496a5adc
md5sum = 036aca95257cb2948dd100605ec6d5a1
location = ${buildout:parts-directory}/${:_buildout_section_name_}
patch-options = -p0
patches =
......@@ -70,8 +70,8 @@ post-install =
# mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
# http://mroonga.github.com/
recipe = slapos.recipe.cmmi
url = http://packages.groonga.org/source/mroonga/mroonga-6.02.tar.gz
md5sum = a7c6320e273eaa407860cac11970dae8
url = http://packages.groonga.org/source/mroonga/mroonga-6.08.tar.gz
md5sum = 10c13aa9f7e520d93799be893aa44d3c
pre-configure =
mkdir fake_mariadb_source &&
ln -s ${mariadb:location}/include/mysql/private fake_mariadb_source/sql
......
component/postgresql/buildout.cfg
View file @ e190f434
......@@ -35,5 +35,5 @@ environment =
[postgresql92]
<= postgresql-common
url = http://ftp.postgresql.org/pub/source/v9.2.17/postgresql-9.2.17.tar.bz2
md5sum = a75d4a82eae1edda04eda2e60656e74c
url = http://ftp.postgresql.org/pub/source/v9.2.18/postgresql-9.2.18.tar.bz2
md5sum = fd175eb5f29557c6ef2eeaf340330f9a
component/tar/buildout.cfg
View file @ e190f434
[buildout]
extends =
../patch/buildout.cfg
../xz-utils/buildout.cfg
parts = tar
[tar]
patch-options = -p1
patches =
${:_profile_base_location_}/tar-drop.gets.patch#9352820566aa3534a04bd269c9f89f48
recipe = slapos.recipe.cmmi
url = http://ftp.gnu.org/gnu/tar/tar-1.26.tar.gz
md5sum = 00d1e769c6af702c542cca54b728920d
url = http://ftp.gnu.org/gnu/tar/tar-1.29.tar.xz
md5sum = a1802fec550baaeecff6c381629653ef
environment =
FORCE_UNSAFE_CONFIGURE=1
PATH=${patch:location}/bin:%(PATH)s
PATH=${xz-utils:location}/bin:%(PATH)s
component/tar/tar-drop.gets.patch deleted 100644 → 0
View file @ 6e7b8e62
diff -ur tar-1.26.orig/gnu/stdio.in.h tar-1.26/gnu/stdio.in.h
--- tar-1.26.orig/gnu/stdio.in.h 2011-03-12 10:14:33.000000000 +0100
+++ tar-1.26/gnu/stdio.in.h 2012-08-24 15:35:22.299190847 +0200
@@ -164,7 +164,10 @@
so any use of gets warrants an unconditional warning. Assume it is
always declared, since it is required by C89. */
#undef gets
+#if defined(__GLIBC__) && !defined(__UCLIBC__) && !__GLIBC_PREREQ(2, 16)
_GL_WARN_ON_USE (gets, "gets is a security hole - use fgets instead");
+#endif
+
#if @GNULIB_FOPEN@
# if @REPLACE_FOPEN@
component/wget/buildout.cfg
View file @ e190f434
......@@ -11,8 +11,8 @@ parts =
[wget]
recipe = slapos.recipe.cmmi
url = http://ftp.gnu.org/gnu/wget/wget-1.16.3.tar.xz
md5sum = d2e4455781a70140ae83b54ca594ce21
url = http://ftp.gnu.org/gnu/wget/wget-1.18.tar.xz
md5sum = af9ca95a4bb8ac4a9bf10aeae66fa5ec
configure-options =
--enable-ipv6
--enable-opie
......
software/agent/software.cfg
View file @ e190f434
......@@ -43,34 +43,34 @@ eggs =
[versions]
apache-libcloud = 0.18.0
ecdsa = 0.13
erp5.util = 0.4.44
erp5.util = 0.4.45
gitdb = 0.6.4
pycrypto = 2.6.1
slapos.recipe.download = 1.0
slapos.recipe.template = 2.8
slapos.toolbox = 0.56
slapos.toolbox = 0.58
smmap = 0.9.0
# Required by:
# slapos.toolbox==0.56
GitPython = 2.0.6
# slapos.toolbox = 0.58
GitPython = 2.0.8
# Required by:
# slapos.toolbox==0.56
# slapos.toolbox = 0.58
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.56
# slapos.toolbox = 0.58
feedparser = 5.2.1
# Required by:
# slapos.toolbox==0.56
# slapos.toolbox = 0.58
lockfile = 0.12.2
# Required by:
# slapos.toolbox==0.56
# slapos.toolbox = 0.58
paramiko = 2.0.1
# Required by:
# slapos.toolbox==0.56
# slapos.toolbox = 0.58
rpdb = 0.1.5
software/apache-frontend/common.cfg
View file @ e190f434
......@@ -167,7 +167,7 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/templates/trafficserver/${:filename}
md5sum = 65afeef0229430ad8a6fbc57298b787b
md5sum = 59287d2de3d948b135619edd211a5e84
location = ${buildout:parts-directory}/${:_buildout_section_name_}
filename = records.config.jinja2
download-only = true
......
software/apache-frontend/software.cfg
View file @ e190f434
......@@ -11,29 +11,29 @@ plone.recipe.command = 1.1
pycrypto = 2.6.1
rdiff-backup = 1.0.5
slapos.recipe.template = 2.8
slapos.toolbox = 0.56
slapos.toolbox = 0.58
smmap = 0.9.0
# Required by:
# slapos.toolbox==0.56
GitPython = 2.0.6
# slapos.toolbox = 0.58
GitPython = 2.0.8
# Required by:
# slapos.toolbox==0.56
# slapos.toolbox = 0.58
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.56
# slapos.toolbox = 0.58
feedparser = 5.2.1
# Required by:
# slapos.toolbox==0.56
# slapos.toolbox = 0.58
lockfile = 0.12.2
# Required by:
# slapos.toolbox==0.56
# slapos.toolbox = 0.58
paramiko = 2.0.1
# Required by:
# slapos.toolbox==0.56
# slapos.toolbox = 0.58
rpdb = 0.1.5
software/apache-frontend/templates/trafficserver/records.config.jinja2
View file @ e190f434
......@@ -184,10 +184,10 @@ CONFIG proxy.config.http.background_fill_completed_threshold FLOAT 0.5
##################################
# origin server connect attempts #
##################################
CONFIG proxy.config.http.connect_attempts_max_retries INT 6
CONFIG proxy.config.http.connect_attempts_max_retries INT 1
CONFIG proxy.config.http.connect_attempts_max_retries_dead_server INT 3
CONFIG proxy.config.http.connect_attempts_rr_retries INT 3
CONFIG proxy.config.http.connect_attempts_timeout INT 30
CONFIG proxy.config.http.connect_attempts_timeout INT 160
CONFIG proxy.config.http.post_connect_attempts_timeout INT 1800
CONFIG proxy.config.http.down_server.cache_time INT 300
CONFIG proxy.config.http.down_server.abort_threshold INT 10
......
software/cdn-me/software.cfg
View file @ e190f434
......@@ -38,6 +38,6 @@ eggs =
[versions]
cns.recipe.symlink = 0.2.3
collective.recipe.environment = 0.2.0
erp5.util = 0.4.44
erp5.util = 0.4.45
plone.recipe.command = 1.1
slapos.recipe.template = 2.8
software/erp5/instance-erp5-input-schema.json
View file @ e190f434
......@@ -38,7 +38,7 @@
"developer-list": {
"description": "List of logins which should get the Developper role (required to modify portal_components' content), defaulting to inituser-login's value",
"items": {
"pattern": "/^\\S+$/",
"pattern": "^\\S+$",
"type": "string"
},
"uniqueItems": true,
......@@ -143,28 +143,28 @@
"kumofs": {
"description": "Persistent memcached service",
"additionalProperties": {
"$ref": "./instance-kumofs-schema.json#properties"
"$ref": "./instance-kumofs-schema.json#/properties"
},
"type": "object"
},
"memcached": {
"description": "Volatile memcached service",
"additionalProperties": {
"$ref": "./instance-kumofs-schema.json#properties"
"$ref": "./instance-kumofs-schema.json#/properties"
},
"type": "object"
},
"cloudooo": {
"description": "Format conversion service",
"additionalProperties": {
"$ref": "./instance-cloudooo-schema.json#properties"
"$ref": "./instance-cloudooo-schema.json#/properties"
},
"type": "object"
},
"mariadb": {
"description": "Relational database service",
"additionalProperties": {
"$ref": "./instance-mariadb-schema.json#properties"
"$ref": "./instance-mariadb-schema.json#/properties"
},
"type": "object"
},
......
software/erp5/instance-erp5-output-schema.json
View file @ e190f434
{
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Values returned by ERP5 instantiation",
"additionalProperties": false,
"properties": {
"hosts-dict": {
"description": "Hosts mapping, including auto-generated entries",
"patternProperties": {
".*": {
"description": "IP or domain names current entry resolves to",
"description": "IP current entry resolves to",
"type": "string"
}
},
......@@ -24,31 +25,59 @@
"description": "Initial user password",
"type": "string"
},
"kumofs-url": {
"deadlock-debugger-password": {
"description": "Deadlock debugger password",
"type": "string"
},
"memcached-persistent-url": {
"description": "Persistent memcached access information",
"pattern": "^memcached://",
"type": "string"
},
"memcached-url": {
"memcached-volatile-url": {
"description": "Volatile memcached access information",
"pattern": "^memcached://",
"type": "string"
},
"cloudooo-url": {
"description": "Conversion service access information",
"description": "Conversion service access information - DEPRECATED",
"pattern": "^cloudooo://",
"type": "string"
},
"mariadb-url": {
"mariadb-database-list": {
"description": "Relational database access information",
"type": "string"
"items": {
"pattern": "^mysql://",
"type": "string"
},
"uniqueItems": true,
"type": "array"
},
"mariadb-test-database-list": {
"description": "Relational database access information",
"items": {
"pattern": "^mysql://",
"type": "string"
},
"uniqueItems": true,
"type": "array"
},
"neo-masters": {
"$ref": "../neoppod/instance-neo-output-schema.json#/properties/masters"
},
"neo-admins": {
"$ref": "../neoppod/instance-neo-output-schema.json#/properties/admins"
},
"jupyter-url": {
"description": "Jupyter notebook web UI access information",
"type": "string",
"optional": true
"pattern": "^https://",
"type": "string"
}
},
"patternProperties": {
"family-.*": {
"description": "Zope family access information",
"pattern": "^https://",
"type": "string"
}
},
......
software/erp5/software.cfg.json
View file @ e190f434
......@@ -5,14 +5,14 @@
"software-type": {
"default": {
"title": "Default",
"description": "Default deployment of ERP5 with auto-create instance.",
"description": "No automated database modification (ERP5Site is not automatically created).",
"request": "instance-erp5-input-schema.json",
"response": "instance-erp5-output-schema.json",
"index": 0
},
"create-erp5-site": {
"title": "Create ERP5 Site",
"description": "Default deployment of ERP5 with automatic creation of the instance.",
"description": "Automated ERP5Site creation on instanciation when ZODB is found empty.",
"request": "instance-erp5-input-schema.json",
"response": "instance-erp5-output-schema.json",
"index": 1
......
software/html5as/software.cfg
View file @ e190f434
......@@ -79,7 +79,7 @@ plone.recipe.command = 1.1
# Required by:
# slapos.toolbox==0.40.2
GitPython = 2.0.6
GitPython = 2.0.8
# Required by:
# slapos.toolbox==0.40.2
......
software/jstestnode/software.cfg
View file @ e190f434
......@@ -111,6 +111,6 @@ output = ${buildout:directory}/runTestSuite.in
mode = 0644
[versions]
erp5.util = 0.4.44
erp5.util = 0.4.45
slapos.recipe.template = 2.9
selenium = 2.53.1
software/kvm/software.cfg
View file @ e190f434
......@@ -15,12 +15,12 @@ plone.recipe.command = 1.1
pycrypto = 2.6.1
slapos.recipe.template = 2.7
smmap = 0.9.0
erp5.util = 0.4.44
erp5.util = 0.4.45
pycurl = 7.19.5.1
# Required by:
# slapos.toolbox==0.48
GitPython = 2.0.6
GitPython = 2.0.8
# Required by:
# slapos.toolbox==0.48
......
software/neoppod/instance-neo-storage-mysql.cfg.in
View file @ e190f434
......@@ -14,6 +14,7 @@ mysql-base-directory = {{ mariadb_location }}
[my-cnf-parameters]
socket = ${directory:var_run}/mariadb.sock
data-directory = ${mysqld:data-directory}
tmp-directory = ${directory:tmp}
pid-file = ${directory:var_run}/mariadb.pid
error-log = ${directory:log}/mariadb_error.log
slow-query-log = ${directory:log}/mariadb_slowquery.log
......
software/neoppod/my.cnf.in
View file @ e190f434
......@@ -5,6 +5,7 @@
skip_networking
socket = {{ socket }}
datadir = {{ parameter_dict['data-directory'] }}
tmpdir = {{ parameter_dict['tmp-directory'] }}
pid_file = {{ parameter_dict['pid-file'] }}
log_error = {{ parameter_dict['error-log'] }}
slow_query_log
......
software/neoppod/software-common.cfg
View file @ e190f434
......@@ -45,6 +45,7 @@ eggs = neoppod[admin, ctl, master, storage-importer, storage-mysqldb, tests]
patch-binary = ${patch:location}/bin/patch
ZODB3-patches =
${:_profile_base_location_}/../../component/egg-patch/ZODB3-3.10.5.patch#c5fe331b1e3a930446f93ab4f6e97c6e
${:_profile_base_location_}/../../component/egg-patch/ZODB3-persistent-ghostify-slots.patch#3a66e9c018d7269bd522d5b0a746f510
ZODB3-patch-options = -p1
[slapos-deps-eggs]
......@@ -95,27 +96,28 @@ md5sum = 82f3f76f54ee9db355966a7ada61f56e
[instance-neo-storage-mysql]
<= download-base-neo
md5sum = 4572f8d3f92f1b1639600d0eb7119ab5
md5sum = cd2a978a09c5686205592923866f6584
[template-neo-my-cnf]
<= download-base-neo
url = ${:_profile_base_location_}/my.cnf.in
md5sum = febd3ed58043ce1367b86cf6e4e69700
md5sum = 81ab5e842ecf8385b12d735585497cc8
[versions]
slapos.recipe.template = 2.9
# patched egg
ZODB3 = 3.10.5+SlapOSPatched001
# Required by slapos.toolbox==0.56
slapos.toolbox = 0.56
ZODB3 = 3.10.5+SlapOSPatched002
# Required by slapos.toolbox = 0.58
slapos.toolbox = 0.58
apache-libcloud = 0.20.1
atomize = 0.2.0
ecdsa = 0.13
feedparser = 5.2.1
GitPython = 2.0.6
GitPython = 2.0.8
gitdb = 0.6.4
lockfile = 0.12.2
mysqlclient = 1.3.7
paramiko = 2.0.1
paramiko = 2.0.2
passlib = 1.6.5
pycrypto = 2.6.1
smmap = 0.9.0
software/neoppod/software.cfg
View file @ e190f434
......@@ -38,8 +38,8 @@ ZODB3-patches +=
${neoppod-repository:location}/ZODB3.patch
[versions]
ZODB3 = 3.10.5+SlapOSPatched002
erp5.util = 0.4.44
ZODB3 = 3.10.5+SlapOSPatched003
erp5.util = 0.4.45
# To match ERP5
transaction = 1.1.1
ZConfig = 2.9.3
......
software/re6stnet/software.cfg
View file @ e190f434
......@@ -112,15 +112,15 @@ gitdb = 0.6.4
plone.recipe.command = 1.1
pycrypto = 2.6.1
slapos.recipe.template = 2.7
slapos.toolbox = 0.56
slapos.toolbox = 0.58
smmap = 0.9.0
# Required by:
# slapos.toolbox==0.56
GitPython = 2.0.6
# slapos.toolbox = 0.58
GitPython = 2.0.8
# Required by:
# slapos.toolbox==0.56
# slapos.toolbox = 0.58
atomize = 0.2.0
# Required by:
......@@ -128,11 +128,11 @@ atomize = 0.2.0
backports.ssl-match-hostname = 3.4.0.2
# Required by:
# slapos.toolbox==0.56
# slapos.toolbox = 0.58
feedparser = 5.1.3
# Required by:
# slapos.toolbox==0.56
# slapos.toolbox = 0.58
lockfile = 0.12.2
# Required by:
......@@ -140,10 +140,10 @@ lockfile = 0.12.2
miniupnpc = 1.9
# Required by:
# slapos.toolbox==0.56
# slapos.toolbox = 0.58
paramiko = 2.0.1
# Required by:
# slapos.toolbox==0.56
# slapos.toolbox = 0.58
rpdb = 0.1.5
software/slapos-master/software.cfg
View file @ e190f434
[buildout]
extends =
../../stack/erp5/buildout.cfg
../../software/erp5/sofware.cfg
parts +=
vifib-fix-products-paths
......@@ -10,29 +10,17 @@ parts +=
# Used to generate bt5lists.
list = ${erp5:location}/bt5 ${erp5:location}/product/ERP5/bootstrap ${vifib:location}/master/bt5
[genbt5list]
recipe = plone.recipe.command
stop-on-error = true
genbt5list = ${erp5:location}/product/ERP5/bin/genbt5list
command =
${buildout:executable} ${:genbt5list} ${local-bt5-repository:list}
update-command = ${:command}
[erp5_repository_list]
repository_id_list = erp5 vifib/master
[erp5]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/erp5.git
branch = erp5-vifib
git-executable = ${git:location}/bin/git
revision = b22077622b356aade6981957ddc6ff271c228bbb
[vifib]
recipe = slapos.recipe.build:gitclone
branch = master
<= erp5
repository = https://lab.nexedi.com/nexedi/slapos.core.git
git-executable = ${git:location}/bin/git
branch = master
revision = b3b9da6158f0dd3956e72a804c57f18b6f966fc1
[vifib-fix-products-paths]
......
software/slaprunner/common.cfg
View file @ e190f434
......@@ -101,7 +101,7 @@ recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/instance-resilient-test.cfg.jinja2
download-only = true
md5sum = fb8c45e5c35548331fb06c4633ec592a
md5sum = c31de887459dfb45fe1213f2837cadcb
filename = instance-resilient-test.cfg.jinja2
mode = 0644
......@@ -110,7 +110,7 @@ recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/nginx_conf.in
download-only = true
md5sum = 94d83ef3eb89c2d75c8c079ab12b4518
md5sum = 2b06f7eb9a1d45d250d4b92a944db925
filename = nginx_conf.in
mode = 0644
......
software/slaprunner/instance-resilient-test.cfg.jinja2
View file @ e190f434
......@@ -61,6 +61,6 @@ config-ignore-known-hosts-file = true
#config-frontend-domain = google.com
# XXX Hack to deploy Root Instance on the same computer as the type-test Instance
sla-computer_guid = ${slap-connection:computer-id}
return = backend_url
return = backend-url
[slap-parameter]
software/slaprunner/nginx_conf.in
View file @ e190f434
......@@ -35,21 +35,6 @@ http {
scgi_temp_path {{ param_tempdir['scgi_temp_path'] }};
location / {
# When no .htpasswd exist, redirect the user to account creation page
if ( !-f {{ param_nginx_frontend['etc_dir'] }}/.htpasswd ) {
# redirect URL is different wether nginx is accessed directly or behind apache.
# nginx does not support nested if or multiple conditions, so we use this well known hack.
set $test no_htpasswd;
}
if ( $host = [{{ param_nginx_frontend['global-ip'] }}] ) {
set $test "${test}_backend_access";
}
if ( $test = no_htpasswd) {
return 301 $scheme://$host/setAccount ;
}
if ( $test = no_htpasswd_backend_access) {
return 301 /setAccount ;
}
auth_basic "Restricted";
auth_basic_user_file {{ param_nginx_frontend['etc_dir'] }}/.htpasswd;
proxy_redirect off;
......
software/slaprunner/software.cfg
View file @ e190f434
......@@ -13,36 +13,36 @@ apache-libcloud = 0.20.1
cns.recipe.symlink = 0.2.3
collective.recipe.environment = 0.2.0
ecdsa = 0.13
erp5.util = 0.4.44
erp5.util = 0.4.45
futures = 3.0.5
gitdb = 0.6.4
gunicorn = 19.5.0
prettytable = 0.7.2
pycrypto = 2.6.1
slapos.recipe.template = 2.9
slapos.toolbox = 0.56
slapos.toolbox = 0.58
smmap = 0.9.0
# Required by:
# slapos.toolbox==0.56
GitPython = 2.0.6
# slapos.toolbox = 0.58
GitPython = 2.0.8
# Required by:
# slapos.toolbox==0.56
# slapos.toolbox = 0.58
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.56
# slapos.toolbox = 0.58
feedparser = 5.2.1
# Required by:
# slapos.toolbox==0.56
# slapos.toolbox = 0.58
lockfile = 0.12.2
# Required by:
# slapos.toolbox==0.56
# slapos.toolbox = 0.58
paramiko = 2.0.1
# Required by:
# slapos.toolbox==0.55
# slapos.toolbox = 0.58
passlib = 1.6.5
\ No newline at end of file
software/wendelin/software.cfg
View file @ e190f434
......@@ -52,6 +52,5 @@ revision = 9c1384dc94905f05b8bac807c59f38990668b648
revision = 4dfdf05a89445a9286dbdf364d495bedf10f9394
[versions]
scipy = 0.15.1
msgpack-python = 0.4.6
msgpack-python = 0.4.8
wendelin.core = 0.7
stack/boinc/buildout.cfg
View file @ e190f434
......@@ -75,7 +75,7 @@ smmap = 0.8.2
# Required by:
# slapos.toolbox==0.40.2
GitPython = 2.0.6
GitPython = 2.0.8
# Required by:
# slapos.toolbox==0.40.2
......
stack/cloudooo.cfg
View file @ e190f434
......@@ -102,4 +102,4 @@ PasteDeploy = 1.5.2
# Required by:
# cloudooo==1.2.5.dev0
erp5.util = 0.4.44
erp5.util = 0.4.45
stack/erp5/buildout.cfg
View file @ e190f434
......@@ -156,7 +156,7 @@ mode = 755
[template-mariadb]
<= download-base
filename = instance-mariadb.cfg.in
md5sum = ede2481d6ce60a335bde920d8cec5318
md5sum = 9312af2f9d9faf06d2f26f073ad60180
link-binary =
${coreutils:location}/bin/basename
${coreutils:location}/bin/cat
......@@ -192,7 +192,7 @@ md5sum = bc6048b85b410693e60e5a77399dd1b7
[template-my-cnf]
<= download-base
filename = my.cnf.in
md5sum = ac5c87991d95907f62bf2129ecfd42c4
md5sum = d50920c942b8ee98402f8551fef38383
[template-mariadb-initial-setup]
<= download-base
......@@ -622,8 +622,8 @@ scripts +=
Acquisition = 2.13.9+SlapOSPatched001
Products.DCWorkflow = 2.2.4+SlapOSPatched001
pysvn = 1.7.10+SlapOSPatched002
python-ldap = 2.4.25+SlapOSPatched001
python-magic = 0.4.11+SlapOSPatched001
python-ldap = 2.4.27+SlapOSPatched001
python-magic = 0.4.12+SlapOSPatched001
# specify dev version to be sure that an old released version is not used
cloudooo = 1.2.5-dev
......@@ -632,7 +632,7 @@ cloudooo = 1.2.5-dev
PasteDeploy = 1.5.2
Pygments = 2.1.3
argparse = 1.4.0
coverage = 4.0.3
coverage = 4.2
zope.dottedname = 4.1.0
# test_UserManagerInterfaces in testERP5Security fails with 1.10.0.
......@@ -664,14 +664,14 @@ zope.app.publication = 3.14.0
zope.app.testing = 3.8.1
# Pinned versions
Pillow = 3.2.0
Pillow = 3.3.1
Products.CMFActionIcons = 2.1.3
Products.DCWorkflowGraph = 0.4.1
# Products.ExternalEditor 2.0.0's dtml is not based on Zope2 OFS's one.
Products.ExternalEditor = 1.1.1
Products.GenericSetup = 1.8.3
Products.LongRequestLogger = 2.0.0
Products.MimetypesRegistry = 2.0.9
Products.MimetypesRegistry = 2.0.10
Products.PluginRegistry = 1.4
Products.TIDStorage = 5.4.9
PyPDF2 = 1.26.0
......@@ -696,19 +696,20 @@ httplib2 = 0.9.2
huBarcode = 1.0.0
interval = 1.0.0
ipdb = 0.10.1
ipykernel = 4.3.1
ipython = 4.2.1
ipykernel = 4.4.1
ipython = 5.1.0
ipywidgets = 5.1.5
logilab-common = 1.2.2
matplotlib = 1.5.1
matplotlib = 1.5.2
mistune = 0.7.3
notebook = 4.2.1
notebook = 4.2.2
numpy = 1.11.1
objgraph = 3.0.0
pandas = 0.18.1
ply = 3.8
polib = 1.0.7
pprofile = 1.9.1
prompt-toolkit = 1.0.3
ptyprocess = 0.5.1
pycountry = 1.20
pyflakes = 1.2.3
......@@ -716,12 +717,12 @@ pyflakes = 1.2.3
pylint = 1.4.4
python-memcached = 1.58
pytracemalloc = 1.2
pyzmq = 15.2.0
pyzmq = 15.4.0
qrcode = 5.3
restkit = 4.2.2
rtjp-eventlet = 0.3.2
scikit-learn = 0.17.1
scipy = 0.17.1
scipy = 0.18.0
simplegeneric = 0.8.1
socketpool = 0.5.3
spyne = 2.12.11
......@@ -729,8 +730,9 @@ suds = 0.4
terminado = 0.6
threadframe = 0.2
timerserver = 2.0.2
tornado = 4.3
urlnorm = 1.1.2
tornado = 4.4.1
traitlets = 4.2.2
urlnorm = 1.1.4
uuid = 1.30
validictory = 1.0.2
widgetsnbextension = 1.2.3
......@@ -742,23 +744,23 @@ xupdate-processor = 0.4
Products.ZSQLMethods = 2.13.4
# Required by:
# ipython==4.2.1
# ipython==5.1.0
backports.shutil-get-terminal-size = 1.0.0
# Required by:
# tornado==4.3
# tornado==4.4.1
backports.ssl-match-hostname = 3.5.0.1
# Required by:
# tornado==4.3
certifi = 2016.2.28
# tornado==4.4.1
certifi = 2016.8.8
# Required by:
# matplotlib==1.5.1
cycler = 0.10.0
# Required by:
# ipython==4.2.1
# ipython==5.1.0
# traitlets==4.2.2
decorator = 4.0.10
......@@ -767,32 +769,32 @@ decorator = 4.0.10
fpconst = 0.7.2
# Required by:
# nbformat==4.0.1
# notebook==4.2.1
# nbformat==4.1.0
# notebook==4.2.2
# traitlets==4.2.2
ipython-genutils = 0.1.0
# Required by:
# notebook==4.2.1
# notebook==4.2.2
# nbconvert 4.2.0 depends on entrypoints egg that is not available as tar/zip source.
nbconvert = 4.1.0
# Required by:
# nbconvert==4.1.0
# notebook==4.2.1
nbformat = 4.0.1
# notebook==4.2.2
nbformat = 4.1.0
# Required by:
# pickleshare==0.7.2
# ipython==5.1.0
pathlib2 = 2.1.0
# Required by:
# ipython==4.2.1
pexpect = 4.1.0
# ipython==5.1.0
pexpect = 4.2.1
# Required by:
# ipython==4.2.1
pickleshare = 0.7.2
# ipython==5.1.0
pickleshare = 0.7.4
# Required by:
# matplotlib==1.5.1
......@@ -800,13 +802,12 @@ pickleshare = 0.7.2
python-dateutil = 2.5.3
# Required by:
# tornado==4.3
# tornado==4.4.1
singledispatch = 3.4.0.3
# Required by:
# ipython==4.2.1
# notebook==4.2.1
traitlets = 4.2.2
# prompt-toolkit==1.0.3
wcwidth = 0.1.7
# Required by:
# zope.app.testing==3.8.1
......
stack/erp5/instance-mariadb.cfg.in
View file @ e190f434
......@@ -102,6 +102,7 @@ ip = {{ ip }}
port = {{ port }}
socket = ${directory:run}/mariadb.sock
data-directory = ${directory:mariadb-data}
tmp-directory = ${directory:tmp}
pid-file = ${directory:run}/mariadb.pid
error-log = ${directory:log}/mariadb_error.log
slow-query-log = ${directory:log}/mariadb_slowquery.log
......@@ -204,6 +205,7 @@ etc = ${buildout:directory}/etc
services = ${:etc}/run
promise = ${:etc}/promise
srv = ${buildout:directory}/srv
tmp = ${buildout:directory}/tmp
backup = ${:srv}/backup
mariadb-backup-full = ${:backup}/mariadb-full
mariadb-backup-incremental = ${:backup}/mariadb-incremental
......
stack/erp5/my.cnf.in
View file @ e190f434
......@@ -19,6 +19,7 @@ skip_networking
{% endif -%}
socket = {{ socket }}
datadir = {{ parameter_dict['data-directory'] }}
tmpdir = {{ parameter_dict['tmp-directory'] }}
pid_file = {{ parameter_dict['pid-file'] }}
log_error = {{ parameter_dict['error-log'] }}
slow_query_log
......
stack/lamp.cfg
View file @ e190f434
......@@ -50,7 +50,7 @@ smmap = 0.8.2
# Required by:
# slapos.toolbox==0.40.2
GitPython = 2.0.6
GitPython = 2.0.8
# Required by:
# slapos.toolbox==0.40.2
......
stack/lamp/buildout.cfg
View file @ e190f434
......@@ -194,7 +194,7 @@ smmap = 0.8.2
# Required by:
# slapos.toolbox==0.40.2
GitPython = 2.0.6
GitPython = 2.0.8
# Required by:
# slapos.toolbox==0.40.2
......
stack/lapp/buildout.cfg
View file @ e190f434
......@@ -191,7 +191,7 @@ smmap = 0.8.2
# Required by:
# slapos.toolbox==0.40.2
GitPython = 2.0.6
GitPython = 2.0.8
# Required by:
# slapos.toolbox==0.40.2
......
stack/monitor/buildout.cfg
View file @ e190f434
[buildout]
# XXX THIS STACK IS A KIND OF FORK OF `stack/monitor`. THIS ONE WAS
# CREATED AS A REDESIGNED ONE TO REMOVE UNWANTED FEATURES AND
# TO GO FURTHER TO THE GOOD DESIGN DIRECTION. SEE THE README FOR
# MORE INFORMATION.
extends =
../../component/apache/buildout.cfg
......@@ -43,6 +39,7 @@ eggs =
plone.recipe.command
collective.recipe.template
cns.recipe.symlink
slapos.toolbox
[extra-eggs]
<= monitor-eggs
......@@ -90,17 +87,19 @@ recipe = slapos.recipe.template:jinja2
filename = template-monitor.cfg
template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in
rendered = ${buildout:directory}/template-monitor.cfg
md5sum = 7ac78495de73cadafea87f97428e487f
md5sum = 84998b1ca3c29445dca70b495515c35b
context =
key apache_location apache:location
key gzip_location gzip:location
raw monitor_bin ${monitor2-bin:location}/${monitor2-bin:filename}
raw monitor_collect ${monitor-collect:location}/${monitor-collect:filename}
raw monitor_bin ${buildout:directory}/bin/monitor.bootstrap
raw monitor_collect ${buildout:directory}/bin/monitor.collect
raw monitor_runpromise ${buildout:directory}/bin/monitor.runpromise
raw monitor_genstatus ${buildout:directory}/bin/monitor.genstatus
raw monitor_genrss ${buildout:directory}/bin/monitor.genrss
raw monitor_configwrite ${buildout:directory}/bin/monitor.configwrite
raw monitor_conf_template ${monitor-conf:location}/${monitor-conf:filename}
raw monitor_document_edit ${monitor-document-edit:location}/${monitor-document-edit:filename}
raw monitor_https_cors ${monitor-httpd-cors:location}/${monitor-httpd-cors:filename}
raw monitor_instance_info ${monitor-instance-info:location}/${monitor-instance-info:filename}
raw monitor_globalstate ${monitor-globalstate:location}/${monitor-globalstate:filename}
raw curl_executable_location ${curl:location}/bin/curl
raw dash_executable_location ${dash:location}/bin/dash
raw dcron_executable_location ${dcron:location}/sbin/crond
......@@ -109,48 +108,12 @@ context =
raw openssl_executable_location ${openssl:location}/bin/openssl
raw python_executable ${buildout:executable}
raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter}
raw promise_executor_py ${run-promise-py:rendered}
raw template_wrapper ${monitor-template-wrapper:location}/${monitor-template-wrapper:filename}
raw status2rss_executable_path ${status2rss-executable:location}/${status2rss-executable:filename}
depends =
${monitor-eggs:eggs}
[monitor2-bin]
<= monitor-template-script
filename = monitor.py
md5sum = 5525e7445dab16fd03f4eeccf069b74b
[run-promise-py]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/scripts/run-promise.py
rendered = ${buildout:parts-directory}/monitor-scripts/run-promise.py
md5sum = 97148dfbb730cc4f55ed54513ce823e0
mode = 0755
context =
raw python ${buildout:directory}/bin/${extra-eggs:interpreter}
[status2rss-executable]
<= monitor-template-script
filename = status2rss.py
md5sum = 88e3bf955e1e4eac76a444d50fa4f020
[monitor-globalstate]
<= monitor-template-script
filename = globalstate.py
md5sum = 3377e325baa4ecfcd6eee06945fb69fc
[monitor-collect]
<= monitor-template-script
filename = collect.py
md5sum = 78fbcb56761315bde354fe7914d3c54f
[monitor-document-edit]
<= monitor-template-script
filename = monitor-document.py
md5sum = 399ff4939b55ff74e6d48bec5a495981
[versions]
PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
slapos.toolbox = 0.58
stack/monitor/instance-monitor.cfg.jinja2.in
View file @ e190f434
......@@ -135,7 +135,7 @@ monitor-hal-json = ${monitor-directory:public}/monitor.hal.json
service-pid-folder = ${monitor-directory:pids}
crond-folder = ${logrotate-directory:cron-entries}
logrotate-folder = ${logrotate:logrotate-entries}
promise-runner = {{ promise_executor_py }}
promise-runner = {{ monitor_runpromise }}
promise-folder-list =
${directory:promises}
${directory:monitor-promise}
......@@ -159,6 +159,8 @@ collector-db = ${monitor-instance-parameter:collector-db}
collect-script = {{ monitor_collect }}
python = {{ python_with_eggs }}
promise-output-file = ${directory:monitor}/monitor-bootstrap-status
[monitor-conf]
recipe = slapos.recipe.template:jinja2
template = {{ monitor_conf_template }}
......@@ -272,7 +274,7 @@ command = kill -USR1 $(cat ${monitor-httpd-conf-parameter:pid-file})
[monitor-status2rss-wrapper]
recipe = slapos.cookbook:wrapper
# XXX - hard-coded Urls
command-line = {{ python_with_eggs }} {{ status2rss_executable_path }} --output '${monitor-directory:public}/feed' --items_folder '${monitor-directory:public}' --feed_url '${monitor-conf-parameters:base-url}/public/feed' --public_url '${monitor-conf-parameters:base-url}/share/jio_public/' --private_url '${monitor-conf-parameters:base-url}/share/jio_private/' --instance_name '${monitor-conf-parameters:title}' --hosting_name '${monitor-conf-parameters:root-title}'
command-line = {{ monitor_genrss }} --output '${monitor-directory:public}/feed' --items_folder '${monitor-directory:public}' --feed_url '${monitor-conf-parameters:base-url}/public/feed' --public_url '${monitor-conf-parameters:base-url}/share/jio_public/' --private_url '${monitor-conf-parameters:base-url}/share/jio_private/' --instance_name '${monitor-conf-parameters:title}' --hosting_name '${monitor-conf-parameters:root-title}'
wrapper-path = ${directory:bin}/monitor-status2rss.py
......@@ -285,13 +287,13 @@ command = ${monitor-status2rss-wrapper:wrapper-path}
[monitor-globalstate-wrapper]
recipe = slapos.cookbook:wrapper
command-line = {{ python_with_eggs }} {{ monitor_globalstate }} '${monitor-conf:rendered}' '${monitor-instance-info:rendered}'
command-line = {{ monitor_genstatus }} '${monitor-conf:rendered}' '${monitor-instance-info:rendered}'
wrapper-path = ${directory:bin}/monitor-globalstate
[monitor-configurator-wrapper]
recipe = slapos.cookbook:wrapper
# XXX - hard coded path
command-line = {{ python_with_eggs }} {{ monitor_document_edit }} --config_folder '${monitor-conf-parameters:private-folder}/config/.jio_documents' --output_cfg_file '${monitor-instance-parameter:configuration-file-path}' --htpasswd_bin '{{ apache_location }}/bin/htpasswd'
command-line = {{ monitor_configwrite }} --config_folder '${monitor-conf-parameters:private-folder}/config/.jio_documents' --output_cfg_file '${monitor-instance-parameter:configuration-file-path}' --htpasswd_bin '{{ apache_location }}/bin/htpasswd'
wrapper-path = ${directory:bin}/monitor-configurator
[monitor-globalstate-cron-entry]
......@@ -330,7 +332,7 @@ monitor-httpd-ipv6 = ${slap-configuration:ipv6-random}
monitor-httpd-port = 8196
# XXX - Set monitor-base-url = ${monitor-httpd-conf-parameter:url} => https://[ipv6]:port
monitor-base-url = ${monitor-frontend-promise:url}
# monitor-base-url = ${monitor-httpd-conf-parameter:url}
#monitor-base-url = ${monitor-httpd-conf-parameter:url}
root-instance-title = ${slap-configuration:root-instance-title}
monitor-url-list =
cors-domains = monitor.app.officejs.com
......@@ -367,6 +369,17 @@ dash_path = {{ dash_executable_location }}
curl_path = {{ curl_executable_location }}
check-secure = 1
[monitor-bootstrap-promise]
recipe = slapos.recipe.template:jinja2
template = {{ template_wrapper }}
rendered = ${directory:promises}/monitor-bootstrap-status
file = ${monitor-conf-parameters:promise-output-file}
command = if [ ! -f "${:file}" ]; then echo "Monitor bootstrap exited with error." && exit 2; else echo "Bootstrap OK"; fi
mode = 0700
context =
key content :command
raw dash_binary {{ dash_executable_location }}
[monitor-base]
# create dependencies between required monitor parts
recipe = plone.recipe.command
......@@ -383,6 +396,7 @@ depends =
${ca-httpd:wrapper}
${monitor-httpd-promise:filename}
${monitor-status2rss-cron-entry:name}
${monitor-bootstrap-promise:file}
[monitor-publish]
monitor-base-url = ${publish:monitor-base-url}
......
stack/monitor/scripts/collect.py deleted 100644 → 0
View file @ 6e7b8e62
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2010-2014 Vifib SARL and Contributors.
# All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public License
# as published by the Free Software Foundation; either version 2.1
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import sqlite3
import os
import pwd
import time
import json
import argparse
import psutil
from time import strftime
from datetime import datetime, timedelta
def parseArguments():
"""
Parse arguments for monitor collector instance.
"""
parser = argparse.ArgumentParser()
parser.add_argument('--output_folder',
help='Path of the folder where output files should be written.')
parser.add_argument('--partition_id',
help='ID of the computer partition to collect data from.')
parser.add_argument('--collector_db',
help='The path of slapos collect database.')
return parser.parse_args()
class RessourceCollect:
def __init__(self, db_path = None):
assert os.path.exists(db_path) and os.path.isfile(db_path)
self.uri = db_path
self.connection = None
self.cursor = None
def connect(self):
self.connection = sqlite3.connect(self.uri)
self.cursor = self.connection.cursor()
def close(self):
assert self.connection is not None
self.cursor.close()
self.connection.close()
def _execute(self, sql):
assert self.connection is not None
return self.cursor.execute(sql)
def select(self, table, date=None, columns="*", where=None):
""" Query database for a full table information """
if date is not None:
where_clause = " WHERE date = '%s' " % date
else:
where_clause = ""
if where is not None:
if where_clause == "":
where_clause += " WHERE 1 = 1 "
where_clause += " AND %s " % where
select_sql = "SELECT %s FROM %s %s " % (columns, table, where_clause)
return self._execute(select_sql)
def has_table(self, name):
self.connect()
check_result_cursor = self.select(
table="sqlite_master",
columns='name',
where="type='table' AND name='%s'" % name)
table_exists_result = zip(*check_result_cursor)
if not len(table_exists_result) or table_exists_result[0][0] is None:
return False
return True
def getPartitionCPULoadAverage(self, partition_id, date_scope):
self.connect()
query_result_cursor = self.select("user", date_scope,
columns="SUM(cpu_percent)",
where="partition = '%s'" % partition_id)
cpu_percent_sum = zip(*query_result_cursor)
if len(cpu_percent_sum) and cpu_percent_sum[0][0] is None:
return
query_result_cursor = self.select("user", date_scope,
columns="COUNT(DISTINCT time)",
where="partition = '%s'" % partition_id)
sample_amount = zip(*query_result_cursor)
self.close()
if len(sample_amount) and len(cpu_percent_sum):
return round(cpu_percent_sum[0][0]/sample_amount[0][0], 2)
def getPartitionUsedMemoryAverage(self, partition_id, date_scope):
self.connect()
query_result_cursor = self.select("user", date_scope,
columns="SUM(memory_rss)",
where="partition = '%s'" % partition_id)
memory_sum = zip(*query_result_cursor)
if len(memory_sum) and memory_sum[0][0] is None:
return
query_result_cursor = self.select("user", date_scope,
columns="COUNT(DISTINCT time)",
where="partition = '%s'" % partition_id)
sample_amount = zip(*query_result_cursor)
self.close()
if len(sample_amount) and len(memory_sum):
return round(memory_sum[0][0]/(sample_amount[0][0]*1024*1024.0), 2)
def getPartitionDiskUsedAverage(self, partition_id, date_scope):
if not self.has_table('folder'):
return
self.db.connect()
query_result_cursor = self.select("folder", date_scope,
columns="SUM(disk_used)",
where="partition = '%s'" % partition_id)
disk_used_sum = zip(*query_result_cursor)
if len(disk_used_sum) and disk_used_sum[0][0] is None:
return
query_result_cursor = self.select("folder", date_scope,
columns="COUNT(DISTINCT time)",
where="partition = '%s'" % partition_id)
collect_amount = zip(*query_result_cursor)
self.db.close()
if len(collect_amount) and len(disk_used_sum):
return round(disk_used_sum[0][0]/(collect_amount[0][0]*1024.0), 2)
def getPartitionConsumption(self, partition_id, where=""):
"""
Query collector db to get consumed ressource for last minute
"""
self.connect()
comsumption_list = []
if where != "":
where = "and %s" % where
date_scope = datetime.now().strftime('%Y-%m-%d')
min_time = (datetime.now() - timedelta(minutes=1)).strftime('%H:%M:00')
max_time = (datetime.now() - timedelta(minutes=1)).strftime('%H:%M:59')
sql_query = """select count(pid), SUM(cpu_percent) as cpu_result, SUM(cpu_time),
MAX(cpu_num_threads), SUM(memory_percent), SUM(memory_rss), pid, SUM(io_rw_counter),
SUM(io_cycles_counter) from user
where date='%s' and partition='%s' and (time between '%s' and '%s') %s
group by pid order by cpu_result desc""" % (
date_scope, partition_id, min_time, max_time, where)
query_result = self._execute(sql_query)
for result in query_result:
count = int(result[0])
if not count > 0:
continue
resource_dict = {
'pid': result[6],
'cpu_percent': round(result[1]/count, 2),
'cpu_time': round((result[2] or 0)/(60.0), 2),
'cpu_num_threads': round(result[3]/count, 2),
'memory_percent': round(result[4]/count, 2),
'memory_rss': round((result[5] or 0)/(1024*1024.0), 2),
'io_rw_counter': round(result[7]/count, 2),
'io_cycles_counter': round(result[8]/count, 2)
}
try:
pprocess = psutil.Process(int(result[6]))
except psutil.NoSuchProcess:
pass
else:
resource_dict['name'] = pprocess.name()
resource_dict['command'] = pprocess.cmdline()
resource_dict['user'] = pprocess.username()
resource_dict['date'] = datetime.fromtimestamp(pprocess.create_time()).strftime("%Y-%m-%d %H:%M:%S")
comsumption_list.append(resource_dict)
self.close()
return comsumption_list
def getPartitionComsumptionStatus(self, partition_id, where=""):
self.connect()
if where != "":
where = " and %s" % where
date_scope = datetime.now().strftime('%Y-%m-%d')
min_time = (datetime.now() - timedelta(minutes=1)).strftime('%H:%M:00')
max_time = (datetime.now() - timedelta(minutes=1)).strftime('%H:%M:59')
sql_query = """select count(pid), SUM(cpu_percent), SUM(cpu_time),
SUM(cpu_num_threads), SUM(memory_percent), SUM(memory_rss), SUM(io_rw_counter),
SUM(io_cycles_counter) from user where
date='%s' and partition='%s' and (time between '%s' and '%s') %s""" % (
date_scope, partition_id, min_time, max_time, where)
query_result = self._execute(sql_query)
result_list = zip(*query_result)
process_dict = memory_dict = io_dict = {}
if len(result_list):
result = result_list
process_dict = {'total_process': result[0][0],
'cpu_percent': round((result[1][0] or 0), 2),
'cpu_time': round((result[2][0] or 0)/(60.0), 2),
'cpu_num_threads': round((result[3][0] or 0), 2),
'date': '%s %s' % (date_scope, min_time)
}
memory_dict = {'memory_percent': round((result[4][0] or 0), 2),
'memory_rss': round((result[5][0] or 0)/(1024*1024.0), 2),
'date': '%s %s' % (date_scope, min_time)
}
io_dict = {'io_rw_counter': round((result[6][0] or 0), 2),
'io_cycles_counter': round((result[7][0] or 0), 2),
'disk_used': 0,
'date': '%s %s' % (date_scope, min_time)
}
if self.has_table('folder'):
disk_result_cursor = self.select(
"folder", date_scope,
columns="SUM(disk_used)",
where="partition='%s' and (time between '%s' and '%s') %s" % (
partition_id, min_time, max_time, where
)
)
disk_used_sum = zip(*disk_result_cursor)
if len(disk_used_sum) and disk_used_sum[0][0] is not None:
io_dict['disk_used'] = round(disk_used_sum[0][0]/1024.0, 2)
self.close()
return (process_dict, memory_dict, io_dict)
def appendToJsonFile(file_path, content, stepback=2):
with open (file_path, mode="r+") as jfile:
jfile.seek(0, 2)
position = jfile.tell() - stepback
jfile.seek(position)
jfile.write('%s}' % ',"{}"]'.format(content))
def initProcessDataFile(file_path):
with open(process_file, 'w') as fprocess:
data_dict = {
"date": time.time(),
"data": ["date, total process, CPU percent, CPU time, CPU threads"]
}
fprocess.write(json.dumps(data_dict))
def initMemoryDataFile(file_path):
with open(mem_file, 'w') as fmem:
data_dict = {
"date": time.time(),
"data": ["date, memory used percent, memory used"]
}
fmem.write(json.dumps(data_dict))
def initIODataFile(file_path):
with open(io_file, 'w') as fio:
data_dict = {
"date": time.time(),
"data": ["date, io rw counter, io cycles counter, disk used"]
}
fio.write(json.dumps(data_dict))
if __name__ == "__main__":
parser = parseArguments()
if not os.path.exists(parser.output_folder) and os.path.isdir(parser.output_folder):
raise Exception("Invalid ouput folder: %s" % parser.output_folder)
# Consumption global status
process_file = os.path.join(parser.output_folder, 'monitor_resource_process.data.json')
mem_file = os.path.join(parser.output_folder, 'monitor_resource_memory.data.json')
io_file = os.path.join(parser.output_folder, 'monitor_resource_io.data.json')
resource_file = os.path.join(parser.output_folder, 'monitor_process_resource.status.json')
status_file = os.path.join(parser.output_folder, 'monitor_resource.status.json')
if not os.path.exists(parser.collector_db):
print "Collector database not found..."
initProcessDataFile(process_file)
initMemoryDataFile(mem_file)
initIODataFile(io_file)
with open(status_file, "w") as status_file:
status_file.write('{"cpu_time": 0, "cpu_percent": 0, "memory_rss": 0, "memory_percent": 0, "io_rw_counter": 0, "date": "", "total_process": 0, "disk_used": 0, "io_cycles_counter": 0, "cpu_num_threads": 0}')
with open(resource_file, "w") as resource_file:
resource_file.write('[]')
exit(1)
collector = RessourceCollect(parser.collector_db)
date_scope = datetime.now().strftime('%Y-%m-%d')
stat_info = os.stat(parser.output_folder)
partition_user = pwd.getpwuid(stat_info.st_uid)[0]
process_result, memory_result, io_result = collector.getPartitionComsumptionStatus(partition_user)
label_list = ['date', 'total_process', 'cpu_percent', 'cpu_time', 'cpu_num_threads',
'memory_percent', 'memory_rss', 'io_rw_counter', 'io_cycles_counter',
'disk_used']
resource_status_dict = {}
if not os.path.exists(process_file):
initProcessDataFile(process_file)
if not os.path.exists(mem_file):
initMemoryDataFile(mem_file)
if not os.path.exists(io_file):
initIODataFile(io_file)
if process_result and process_result['total_process'] != 0.0:
appendToJsonFile(process_file, ", ".join(
[str(process_result[key]) for key in label_list if process_result.has_key(key)])
)
resource_status_dict.update(process_result)
if memory_result and memory_result['memory_rss'] != 0.0:
appendToJsonFile(mem_file, ", ".join(
[str(memory_result[key]) for key in label_list if memory_result.has_key(key)])
)
resource_status_dict.update(memory_result)
if io_result and io_result['io_rw_counter'] != 0.0:
appendToJsonFile(io_file, ", ".join(
[str(io_result[key]) for key in label_list if io_result.has_key(key)])
)
resource_status_dict.update(io_result)
with open(status_file, 'w') as fp:
fp.write(json.dumps(resource_status_dict))
# Consumption Ressource
resource_process_status_list = collector.getPartitionConsumption(partition_user)
if resource_process_status_list:
with open(resource_file, 'w') as rf:
rf.write(json.dumps(resource_process_status_list))
stack/monitor/scripts/globalstate.py deleted 100644 → 0
View file @ 6e7b8e62
#!/usr/bin/env python
import sys
import os
import glob
import json
import ConfigParser
import time
from datetime import datetime
def softConfigGet(config, *args, **kwargs):
try:
return config.get(*args, **kwargs)
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
return ""
def generateStatisticsData(stat_file_path, content):
# csv document for statictics
if not os.path.exists(stat_file_path):
with open(stat_file_path, 'w') as fstat:
data_dict = {
"date": time.time(),
"data": ["Date, Success, Error, Warning"]
}
fstat.write(json.dumps(data_dict))
current_state = ''
if content.has_key('state'):
current_state = '%s, %s, %s, %s' % (
content['date'],
content['state']['success'],
content['state']['error'],
content['state']['warning'])
# append to file
if current_state:
with open (stat_file_path, mode="r+") as fstat:
fstat.seek(0,2)
position = fstat.tell() -2
fstat.seek(position)
fstat.write('%s}' % ',"{}"]'.format(current_state))
def main(args_list):
monitor_file, instance_file = args_list
monitor_config = ConfigParser.ConfigParser()
monitor_config.read(monitor_file)
base_folder = monitor_config.get('monitor', 'private-folder')
status_folder = monitor_config.get('monitor', 'public-folder')
base_url = monitor_config.get('monitor', 'base-url')
related_monitor_list = monitor_config.get("monitor", "monitor-url-list").split()
statistic_folder = os.path.join(base_folder, 'data', '.jio_documents')
parameter_file = os.path.join(base_folder, 'config', '.jio_documents', 'config.json')
report_date = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
if not os.path.exists(statistic_folder):
try:
os.makedirs(statistic_folder)
except OSError, e:
if e.errno == os.errno.EEXIST and os.path.isdir(statistic_folder):
pass
else: raise
# search for all status files
file_list = filter(os.path.isfile,
glob.glob("%s/*.status.json" % status_folder)
)
error = warning = success = 0
status = 'OK'
promise_list = []
global_state_file = os.path.join(base_folder, 'monitor.global.json')
public_state_file = os.path.join(status_folder, 'monitor.global.json')
for file in file_list:
try:
with open(file, 'r') as temp_file:
tmp_json = json.loads(temp_file.read())
except ValueError:
# bad json file ?
continue
if tmp_json['status'] == 'ERROR':
error += 1
elif tmp_json['status'] == 'OK':
success += 1
elif tmp_json['status'] == 'WARNING':
warning += 1
tmp_json['time'] = tmp_json['start-date'].split(' ')[1]
promise_list.append(tmp_json)
if error:
status = 'ERROR'
elif warning:
status = 'WARNING'
global_state_dict = dict(
status=status,
state={
'error': error,
'success': success,
'warning': warning,
},
type='global',
date=report_date,
_links={"rss_url": {"href": "%s/public/feed" % base_url},
"public_url": {"href": "%s/share/jio_public/" % base_url},
"private_url": {"href": "%s/share/jio_private/" % base_url}
},
data={'state': 'monitor_state.data',
'process_state': 'monitor_process_resource.status',
'process_resource': 'monitor_resource_process.data',
'memory_resource': 'monitor_resource_memory.data',
'io_resource': 'monitor_resource_io.data',
'monitor_process_state': 'monitor_resource.status'}
)
global_state_dict['_embedded'] = {'promises': promise_list}
if os.path.exists(instance_file):
config = ConfigParser.ConfigParser()
config.read(instance_file)
if 'instance' in config.sections():
instance_dict = {}
global_state_dict['title'] = config.get('instance', 'name')
global_state_dict['hosting-title'] = config.get('instance', 'root-name')
if not global_state_dict['title']:
global_state_dict['title'] = 'Instance Monitoring'
instance_dict['computer'] = config.get('instance', 'computer')
instance_dict['ipv4'] = config.get('instance', 'ipv4')
instance_dict['ipv6'] = config.get('instance', 'ipv6')
instance_dict['software-release'] = config.get('instance', 'software-release')
instance_dict['software-type'] = config.get('instance', 'software-type')
instance_dict['partition'] = config.get('instance', 'partition')
global_state_dict['_embedded'].update({'instance' : instance_dict})
if related_monitor_list:
global_state_dict['_links']['related_monitor'] = [{'href': "%s/share/jio_public" % url}
for url in related_monitor_list]
if os.path.exists(parameter_file):
with open(parameter_file) as cfile:
global_state_dict['parameters'] = json.loads(cfile.read())
# Public information with the link to private folder
public_state_dict = dict(
status=status,
date=report_date,
_links={'monitor': {'href': '%s/share/jio_private/' % base_url}},
title=global_state_dict.get('title', '')
)
public_state_dict['hosting-title'] = global_state_dict.get('hosting-title', '')
public_state_dict['_links']['related_monitor'] = global_state_dict['_links'].get('related_monitor', [])
with open(global_state_file, 'w') as fglobal:
fglobal.write(json.dumps(global_state_dict))
with open(public_state_file, 'w') as fpglobal:
fpglobal.write(json.dumps(public_state_dict))
generateStatisticsData(
os.path.join(statistic_folder, 'monitor_state.data.json'),
global_state_dict)
return 0
if __name__ == "__main__":
if len(sys.argv) < 3:
print("Usage: %s <monitor_conf_path> <instance_conf_path>" % sys.argv[0])
sys.exit(2)
sys.exit(main(sys.argv[1:]))
stack/monitor/scripts/monitor-document.py deleted 100644 → 0
View file @ 6e7b8e62
#!/usr/bin/env python
import sys
import os
import re
import json
import argparse
import subprocess
from datetime import datetime
import time
def parseArguments():
"""
Parse arguments for monitor instance.
"""
parser = argparse.ArgumentParser()
parser.add_argument('--config_folder',
help='Path where json configuration/document will be read and write')
parser.add_argument('--htpasswd_bin',
help='Path apache htpasswd binary. Needed to write htpasswd file.')
parser.add_argument('--output_cfg_file',
help='Ouput parameters in cfg file.')
return parser.parse_args()
def fileWrite(file_path, content):
if os.path.exists(file_path):
try:
with open(file_path, 'w') as wf:
wf.write(content)
return True
except OSError, e:
print "ERROR while writing changes to %s.\n %s" % (file_path, str(e))
return False
def htpasswdWrite(htpasswd_bin, parameter_dict, value):
if not os.path.exists(parameter_dict['file']):
return False
command = [htpasswd_bin, '-cb', parameter_dict['htpasswd'], parameter_dict['user'], value]
process = subprocess.Popen(
command,
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
result = process.communicate()[0]
if process.returncode != 0:
print result
return False
with open(parameter_dict['file'], 'w') as pfile:
pfile.write(value)
return True
def httpdCorsDomainWrite(httpd_cors_file, httpd_gracefull_bin, cors_domain):
cors_string = ""
cors_domain_list = cors_domain.split()
old_httpd_cors_file = os.path.join(
os.path.dirname(httpd_cors_file),
'prev_%s' % os.path.basename(httpd_cors_file)
)
if os.path.exists(old_httpd_cors_file) and os.path.isfile(old_httpd_cors_file):
try:
with open(old_httpd_cors_file, 'r') as cors_file:
if cors_file.read() == cors_domain:
if os.path.exists(httpd_cors_file) and (os.stat(httpd_cors_file).st_size > 0
or (cors_domain == "" and os.stat(httpd_cors_file).st_size == 0)):
# Skip if cors file is not empty
return True
except OSError, e:
print "Failed to open file at %s. \n%s" % (old_httpd_cors_file, str(e))
for domain in cors_domain_list:
if cors_string:
cors_string += '|'
cors_string += re.escape(domain)
try:
with open(httpd_cors_file, 'w') as file:
file.write('SetEnvIf Origin "^http(s)?://(.+\.)?(%s)$" origin_is=$0\n' % cors_string)
file.write('Header always set Access-Control-Allow-Origin %{origin_is}e env=origin_is')
except OSError, e:
print "ERROR while writing CORS changes to %s.\n %s" % (httpd_cors_file, str(e))
return False
# Save current cors domain list
try:
with open(old_httpd_cors_file, 'w') as cors_file:
cors_file.write(cors_domain)
except OSError, e:
print "Failed to open file at %s. \n%s" % (old_httpd_cors_file, str(e))
return False
# Restart httpd process
try:
subprocess.call(httpd_gracefull_bin)
except OSError, e:
print "Failed to execute command %s.\n %s" % (httpd_gracefull_bin, str(e))
return False
def applyEditChage(parser):
parameter_tmp_file = os.path.join(parser.config_folder, 'config.tmp.json')
config_file = os.path.join(parser.config_folder, 'config.json')
parameter_config_file = os.path.join(parser.config_folder, 'config.parameters.json')
if not os.path.exists(parameter_tmp_file) or not os.path.isfile(parameter_tmp_file):
return {}
if not os.path.exists(config_file):
print "ERROR: Config file doesn't exist... Exiting"
return {}
new_parameter_list = []
parameter_list = []
description_dict = {}
result_dict = {}
try:
with open(parameter_tmp_file) as tmpfile:
new_parameter_list = json.loads(tmpfile.read())
except ValueError:
print "Error: Couldn't parse json file %s" % parameter_tmp_file
with open(parameter_config_file) as tmpfile:
description_dict = json.loads(tmpfile.read())
for i in range(0, len(new_parameter_list)):
key = new_parameter_list[i]['key']
if key != '':
description_entry = description_dict[key]
if description_entry['type'] == 'file':
result_dict[key] = fileWrite(description_entry['file'], new_parameter_list[i]['value'])
elif description_entry['type'] == 'htpasswd':
result_dict[key] = htpasswdWrite(parser.htpasswd_bin, description_entry, new_parameter_list[i]['value'])
elif description_entry['type'] == 'httpdcors':
result_dict[key] = httpdCorsDomainWrite(description_entry['cors_file'], description_entry['gracefull_bin'], new_parameter_list[i]['value'])
if (parser.output_cfg_file):
try:
with open(parser.output_cfg_file, 'w') as pfile:
pfile.write('[public]\n')
for parameter in new_parameter_list:
if parameter['key']:
pfile.write('%s = %s\n' % (parameter['key'], parameter['value']))
except OSError, e:
print "Error failed to create file %s" % parser.output_cfg_file
pass
return result_dict
if __name__ == "__main__":
parser = parseArguments()
parameter_tmp_file = os.path.join(parser.config_folder, 'config.tmp.json')
config_file = os.path.join(parser.config_folder, 'config.json')
# Run 4 times with sleep
run_counter = 1
max_runn = 4
sleep_time = 15
while True:
result_dict = applyEditChage(parser)
if result_dict != {}:
status = True
for key in result_dict:
if not result_dict[key]:
status = False
if status and os.path.exists(parameter_tmp_file):
try:
os.unlink(config_file)
except OSError, e:
print "ERROR cannot remove file: %s" % parameter_tmp_file
else:
os.rename(parameter_tmp_file, config_file)
if run_counter == max_runn:
break
else:
run_counter += 1
time.sleep(sleep_time)
stack/monitor/scripts/monitor.py deleted 100644 → 0
View file @ 6e7b8e62
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
import os
import stat
import json
import ConfigParser
import traceback
import argparse
import urllib2
import ssl
import glob
from datetime import datetime
OPML_START = """<?xml version="1.0" encoding="UTF-8"?>
<!-- OPML generated by SlapOS -->
<opml version="1.1">
<head>
<title>%(root_title)s</title>
<dateCreated>%(creation_date)s</dateCreated>
<dateModified>%(modification_date)s</dateModified>
</head>
<body>
<outline text="%(outline_title)s">"""
OPML_END = """ </outline>
</body>
</opml>"""
OPML_OUTLINE_FEED = '<outline text="%(title)s" title="%(title)s" type="rss" version="RSS" htmlUrl="%(html_url)s" xmlUrl="%(xml_url)s" url="%(global_url)s" />'
def parseArguments():
"""
Parse arguments for monitor instance.
"""
parser = argparse.ArgumentParser()
parser.add_argument('--config_file',
default='monitor.cfg',
help='Monitor Configuration file')
return parser.parse_args()
def mkdirAll(path):
try:
os.makedirs(path)
except OSError, e:
if e.errno == os.errno.EEXIST and os.path.isdir(path):
pass
else: raise
def softConfigGet(config, *args, **kwargs):
try:
return config.get(*args, **kwargs)
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
return None
def createSymlink(source, destination):
try:
os.symlink(source, destination)
except OSError, e:
if e.errno != os.errno.EEXIST:
raise
class Monitoring(object):
def __init__(self, configuration_file):
config = self.loadConfig([configuration_file])
# Set Monitor variables
self.monitor_hal_json = config.get("monitor", "monitor-hal-json")
self.title = config.get("monitor", "title")
self.root_title = config.get("monitor", "root-title")
self.service_pid_folder = config.get("monitor", "service-pid-folder")
self.crond_folder = config.get("monitor", "crond-folder")
self.logrotate_d = config.get("monitor", "logrotate-folder")
self.promise_runner = config.get("monitor", "promise-runner")
self.promise_folder_list = config.get("monitor", "promise-folder-list").split()
self.public_folder = config.get("monitor", "public-folder")
self.private_folder = config.get("monitor", "private-folder")
self.collector_db = config.get("monitor", "collector-db")
self.collect_script = config.get("monitor", "collect-script")
self.webdav_folder = config.get("monitor", "webdav-folder")
self.report_script_folder = config.get("monitor", "report-folder")
self.webdav_url = '%s/share' % config.get("monitor", "base-url")
self.public_url = '%s/public' % config.get("monitor", "base-url")
self.python = config.get("monitor", "python") or "python"
self.public_path_list = config.get("monitor", "public-path-list").split()
self.private_path_list = config.get("monitor", "private-path-list").split()
self.monitor_url_list = config.get("monitor", "monitor-url-list").split()
self.parameter_list = [param.strip() for param in config.get("monitor", "parameter-list").split('\n') if param]
# Use this file to write knowledge0_cfg required by webrunner
self.parameter_cfg_file = config.get("monitor", "parameter-file-path").strip()
self.config_folder = os.path.join(self.private_folder, 'config')
self.report_folder = self.private_folder
self.promise_dict = {}
for promise_folder in self.promise_folder_list:
self.setupPromiseDictFromFolder(promise_folder)
def loadConfig(self, pathes, config=None):
if config is None:
config = ConfigParser.ConfigParser()
try:
config.read(pathes)
except ConfigParser.MissingSectionHeaderError:
traceback.print_exc()
return config
def readInstanceConfiguration(self):
type_list = ['raw', 'file', 'htpasswd', 'httpdcors']
configuration_list = []
if not self.parameter_list:
return []
for config in self.parameter_list:
config_list = config.strip().split(' ')
# type: config_list[0]
if len(config_list) >= 3 and config_list[0] in type_list:
if config_list[0] == 'raw':
configuration_list.append(dict(
key='',
title=config_list[1],
value=' '.join(config_list[2:])
))
elif (config_list[0] == 'file' or config_list[0] == 'htpasswd') and \
os.path.exists(config_list[2]) and os.path.isfile(config_list[2]):
try:
with open(config_list[2]) as cfile:
parameter = dict(
key=config_list[1],
title=config_list[1],
value=cfile.read(),
description={
"type": config_list[0],
"file": config_list[2]
}
)
if config_list[0] == 'htpasswd':
if len(config_list) != 5 or not os.path.exists(config_list[4]):
print 'htpasswd file is not specified: %s' % str(config_list)
continue
parameter['description']['user'] = config_list[3]
parameter['description']['htpasswd'] = config_list[4]
configuration_list.append(parameter)
except OSError, e:
print 'Cannot read file %s, Error is: %s' % (config_list[2], str(e))
pass
elif config_list[0] == 'httpdcors' and os.path.exists(config_list[2]) and \
os.path.exists(config_list[3]):
old_cors_file = os.path.join(
os.path.dirname(config_list[2]),
'prev_%s' % os.path.basename(config_list[2])
)
try:
cors_content = ""
if os.path.exists(old_cors_file):
with open(old_cors_file) as cfile:
cors_content = cfile.read()
else:
# Create empty file
with open(old_cors_file, 'w') as cfile:
cfile.write("")
parameter = dict(
key=config_list[1],
title=config_list[1],
value=cors_content,
description={
"type": config_list[0],
"cors_file": config_list[2],
"gracefull_bin": config_list[3]
}
)
configuration_list.append(parameter)
except OSError, e:
print 'Cannot read file at %s, Error is: %s' % (old_cors_file, str(e))
pass
return configuration_list
def setupPromiseDictFromFolder(self, folder):
for filename in os.listdir(folder):
path = os.path.join(folder, filename)
if os.path.isfile(path) and os.access(path, os.X_OK):
self.promise_dict[filename] = {"path": path,
"configuration": ConfigParser.ConfigParser()}
def createSymlinksFromConfig(self, destination_folder, source_path_list, name=""):
if destination_folder:
if source_path_list:
for path in source_path_list:
path = path.rstrip('/')
dirname = os.path.join(destination_folder, name)
try:
mkdirAll(dirname) # could also raise OSError
os.symlink(path, os.path.join(dirname, os.path.basename(path)))
except OSError, e:
if e.errno != os.errno.EEXIST:
raise
def getMonitorTitleFromUrl(self, monitor_url):
# This file should be generated
if not monitor_url.startswith('https://') and not monitor_url.startswith('http://'):
return 'Unknown Instance'
if not monitor_url.endswith('/'):
monitor_url = monitor_url + '/'
url = monitor_url + '/.jio_documents/monitor.global.json' # XXX Hard Coded path
try:
# XXX - working here with public url
if hasattr(ssl, '_create_unverified_context'):
context = ssl._create_unverified_context()
response = urllib2.urlopen(url, context=context)
else:
response = urllib2.urlopen(url)
except urllib2.HTTPError:
return 'Unknown Instance'
else:
try:
monitor_dict = json.loads(response.read())
return monitor_dict.get('title', 'Unknown Instance')
except ValueError, e:
print "Bad Json file at %s" % url
return 'Unknown Instance'
def getReportInfoFromFilename(self, filename):
splited_filename = filename.split('_every_')
possible_time_list = ['hour', 'minute']
if len(splited_filename) == 1:
return (filename, "* * * * *")
run_time = splited_filename[1].split('_')
report_name = splited_filename[0]
if len(run_time) != 2 or not run_time[1] in possible_time_list:
return (report_name, "* * * * *")
try:
value = int(run_time[0])
except ValueError:
print "Warning: Bad report filename: %s" % filename
return (report_name, "* * * * *")
if run_time[1] == 'hour':
return (report_name, "* */%s * * *" % value)
if run_time[1] == 'minute':
return (report_name, "*/%s * * * *" % value)
def configureFolders(self):
# configure public and private folder
self.createSymlinksFromConfig(self.webdav_folder, [self.public_folder])
self.createSymlinksFromConfig(self.webdav_folder, [self.private_folder])
#configure jio_documents folder
jio_public = os.path.join(self.webdav_folder, 'jio_public')
jio_private = os.path.join(self.webdav_folder, 'jio_private')
mkdirAll(jio_public)
mkdirAll(jio_private)
createSymlink(self.public_folder,
os.path.join(jio_public, '.jio_documents'))
createSymlink(self.private_folder,
os.path.join(jio_private, '.jio_documents'))
self.data_folder = os.path.join(self.private_folder, 'data', '.jio_documents')
self.document_folder = os.path.join(self.private_folder, 'documents')
config_folder = os.path.join(self.config_folder, '.jio_documents')
mkdirAll(self.data_folder)
mkdirAll(config_folder)
createSymlink(os.path.join(self.private_folder, 'data'),
os.path.join(jio_private, 'data'))
createSymlink(self.config_folder, os.path.join(jio_private, 'config'))
createSymlink(self.data_folder, self.document_folder)
# Cleanup private folder
for file in glob.glob("%s/*.history.json" % self.private_folder):
try:
os.unlink(file)
except OSError:
print "failed to remove file %s. Ignoring..." % file
def makeConfigurationFiles(self):
config_folder = os.path.join(self.config_folder, '.jio_documents')
parameter_config_file = os.path.join(config_folder, 'config.parameters.json')
parameter_file = os.path.join(config_folder, 'config.json')
#mkdirAll(config_folder)
parameter_list = self.readInstanceConfiguration()
description_dict = {}
if parameter_list:
for i in range(0, len(parameter_list)):
key = parameter_list[i]['key']
if key:
description_dict[key] = parameter_list[i].pop('description')
with open(parameter_config_file, 'w') as config_file:
config_file.write(json.dumps(description_dict))
with open(parameter_file, 'w') as config_file:
config_file.write(json.dumps(parameter_list))
try:
with open(self.parameter_cfg_file, 'w') as pfile:
pfile.write('[public]\n')
for parameter in parameter_list:
if parameter['key']:
pfile.write('%s = %s\n' % (parameter['key'], parameter['value']))
except OSError, e:
print "Error failed to create file %s" % self.parameter_cfg_file
pass
def generateOpmlFile(self, feed_url_list, output_file):
if os.path.exists(output_file):
creation_date = datetime.fromtimestamp(os.path.getctime(output_file)).utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
modification_date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
else:
creation_date = modification_date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
opml_content = OPML_START % {'creation_date': creation_date,
'modification_date': modification_date,
'outline_title': 'Monitoring RSS Feed list',
'root_title': self.root_title}
opml_content += OPML_OUTLINE_FEED % {'title': self.title,
'html_url': self.public_url + '/feed',
'xml_url': self.public_url + '/feed',
'global_url': "%s/jio_private/" % self.webdav_url}
for feed_url in feed_url_list:
opml_content += OPML_OUTLINE_FEED % {'title': self.getMonitorTitleFromUrl(feed_url + "/share/jio_public/"),
'html_url': feed_url + '/public/feed',
'xml_url': feed_url + '/public/feed',
'global_url': "%s/share/jio_private/" % feed_url}
opml_content += OPML_END
with open(output_file, 'w') as wfile:
wfile.write(opml_content)
def generateLogrotateEntry(self, name, file_list, option_list):
"""
Will add a new entry in logrotate.d folder. This can help to rotate data file daily
"""
content = "%(logfiles)s {\n%(options)s\n}\n" % {
'logfiles': ' '.join(file_list),
'options': '\n'.join(option_list)
}
file_path = os.path.join(self.logrotate_d, name)
with open(file_path, 'w') as flog:
flog.write(content)
def generateReportCronEntries(self):
cron_line_list = []
report_name_list = [name.replace('.report.json', '')
for name in os.listdir(self.report_folder) if name.endswith('.report.json')]
for filename in os.listdir(self.report_script_folder):
report_script = os.path.join(self.report_script_folder, filename)
if os.path.isfile(report_script) and os.access(report_script, os.X_OK):
report_name, frequency = self.getReportInfoFromFilename(filename)
# report_name = os.path.splitext(filename)[0]
report_json_path = "%s.report.json" % report_name
report_cmd_line = [
frequency,
self.promise_runner,
'--pid_path "%s"' % os.path.join(self.service_pid_folder,
"%s.pid" % filename),
'--output "%s"' % os.path.join(self.report_folder,report_json_path),
'--promise_script "%s"' % report_script,
'--promise_name "%s"' % report_name,
'--monitor_url "%s/jio_private/"' % self.webdav_url, # XXX hardcoded,
'--history_folder "%s"' % self.data_folder,
'--instance_name "%s"' % self.title,
'--hosting_name "%s"' % self.root_title,
'--promise_type "report"']
cron_line_list.append(' '.join(report_cmd_line))
if report_name in report_name_list:
report_name_list.pop(report_name_list.index(report_name))
# cleanup removed report json result
if report_name_list != []:
for report_name in report_name_list:
result_path = os.path.join(self.public_folder, '%s.report.json' % report_name)
if os.path.exists(result_path):
try:
os.unlink(result_path)
except OSError, e:
print "Error: Failed to delete %s" % result_path, str(e)
pass
with open(self.crond_folder + "/monitor-reports", "w") as freport:
freport.write("\n".join(cron_line_list))
def generateServiceCronEntries(self):
# XXX only if at least one configuration file is modified, then write in the cron
#cron_line_list = ['PATH=%s\n' % os.environ['PATH']]
cron_line_list = []
service_name_list = [name.replace('.status.json', '')
for name in os.listdir(self.public_folder) if name.endswith('.status.json')]
for service_name, promise in self.promise_items:
service_config = promise["configuration"]
service_status_path = "%s/%s.status.json" % (self.public_folder, service_name)
mkdirAll(os.path.dirname(service_status_path))
promise_cmd_line = [
softConfigGet(service_config, "service", "frequency") or "* * * * *",
self.promise_runner,
'--pid_path "%s"' % os.path.join(self.service_pid_folder,
"%s.pid" % service_name),
'--output "%s"' % service_status_path,
'--promise_script "%s"' % promise["path"],
'--promise_name "%s"' % service_name,
'--monitor_url "%s/jio_private/"' % self.webdav_url, # XXX hardcoded,
'--history_folder "%s"' % self.public_folder,
'--instance_name "%s"' % self.title,
'--hosting_name "%s"' % self.root_title]
cron_line_list.append(' '.join(promise_cmd_line))
if service_name in service_name_list:
service_name_list.pop(service_name_list.index(service_name))
if service_name_list != []:
# XXX Some service was removed, delete his status file so monitor will not consider his status anymore
for service_name in service_name_list:
status_path = os.path.join(self.public_folder, '%s.status.json' % service_name)
if os.path.exists(status_path):
try:
os.unlink(status_path)
except OSError, e:
print "Error: Failed to delete %s" % status_path, str(e)
pass
with open(self.crond_folder + "/monitor-promises", "w") as fp:
fp.write("\n".join(cron_line_list))
def addCronEntry(self, name, frequency, command):
entry_line = '%s %s' % (frequency, command)
cron_entry_file = os.path.join(self.crond_folder, name)
with open(cron_entry_file, "w") as cronf:
cronf.write(entry_line)
def bootstrapMonitor(self):
# create symlinks from monitor.conf
self.createSymlinksFromConfig(self.public_folder, self.public_path_list)
self.createSymlinksFromConfig(self.private_folder, self.private_path_list)
self.configureFolders()
# create symlinks from service configurations
self.promise_items = self.promise_dict.items()
for service_name, promise in self.promise_items:
service_config = promise["configuration"]
public_path_list = softConfigGet(service_config, "service", "public-path-list")
private_path_list = softConfigGet(service_config, "service", "private-path-list")
if public_path_list:
self.createSymlinksFromConfig(self.public_folder,
public_path_list.split(),
service_name)
if private_path_list:
self.createSymlinksFromConfig(self.private_folder,
private_path_list.split(),
service_name)
# Generate OPML file
self.generateOpmlFile(self.monitor_url_list,
os.path.join(self.public_folder, 'feeds'))
# put promises to a cron file
self.generateServiceCronEntries()
# put report script to cron
self.generateReportCronEntries()
# Generate parameters files and scripts
self.makeConfigurationFiles()
# Rotate monitor data files
option_list = [
'daily', 'nocreate', 'olddir %s' % self.data_folder, 'rotate 5',
'nocompress', 'extension .json', 'dateext',
'dateformat -%Y-%m-%d', 'notifempty'
]
file_list = [
"%s/*.data.json" % self.private_folder,
"%s/*.data.json" % self.data_folder]
self.generateLogrotateEntry('monitor.data', file_list, option_list)
# Rotate public history status file, delete data of previous days
option_list = [
'daily', 'nocreate', 'rotate 0',
'nocompress', 'notifempty'
]
file_list = ["%s/*.history.json" % self.public_folder]
self.generateLogrotateEntry('monitor.service.status', file_list, option_list)
# Add cron entry for SlapOS Collect
command = "%s %s --output_folder %s --collector_db %s" % (self.python,
self.collect_script, self.data_folder, self.collector_db)
self.addCronEntry('monitor_collect', '* * * * *', command)
return 0
if __name__ == "__main__":
parser = parseArguments()
monitor = Monitoring(parser.config_file)
sys.exit(monitor.bootstrapMonitor())
stack/monitor/scripts/run-promise.py deleted 100644 → 0
View file @ 6e7b8e62
#!{{ python }}
# -*- coding: utf-8 -*-
import sys
import os
import subprocess
import json
import psutil
import time
from shutil import copyfile
import glob
import argparse
import traceback
def parseArguments():
"""
Parse arguments for monitor collector instance.
"""
parser = argparse.ArgumentParser()
parser.add_argument('--pid_path',
help='Path where the pid of this process will be writen.')
parser.add_argument('--output',
help='The Path of file where Json result of this promise will be saved.')
parser.add_argument('--promise_script',
help='Promise script to execute.')
parser.add_argument('--promise_name',
help='Title to give to this promise.')
parser.add_argument('--promise_type',
default='status',
help='Type of promise to execute. [status, report].')
parser.add_argument('--monitor_url',
help='Monitor Instance website URL.')
parser.add_argument('--history_folder',
help='Path where old result file will be placed before generate a new json result file.')
parser.add_argument('--instance_name',
default='UNKNOWN Software Instance',
help='Software Instance name.')
parser.add_argument('--hosting_name',
default='UNKNOWN Hosting Subscription',
help='Hosting Subscription name.')
return parser.parse_args()
def main():
parser = parseArguments()
if os.path.exists(parser.pid_path):
with open(parser.pid_path, "r") as pidfile:
try:
pid = int(pidfile.read(6))
except ValueError:
pid = None
if pid and os.path.exists("/proc/" + str(pid)):
print("A process is already running with pid " + str(pid))
return 1
start_date = ""
with open(parser.pid_path, "w") as pidfile:
process = executeCommand(parser.promise_script)
ps_process = psutil.Process(process.pid)
start_date = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ps_process.create_time()))
pidfile.write(str(process.pid))
status_json = generateStatusJsonFromProcess(process, start_date=start_date)
status_json['_links'] = {"monitor": {"href": parser.monitor_url}}
status_json['title'] = parser.promise_name
status_json['instance'] = parser.instance_name
status_json['hosting_subscription'] = parser.hosting_name
status_json['type'] = parser.promise_type
# Save the lastest status change date (needed for rss)
status_json['change-time'] = ps_process.create_time()
if os.path.exists(parser.output):
with open(parser.output) as f:
try:
last_result = json.loads(f.read())
if status_json['status'] == last_result['status'] and last_result.has_key('change-time'):
status_json['change-time'] = last_result['change-time']
except ValueError:
pass
updateStatusHistoryFolder(
parser.promise_name,
parser.output,
parser.history_folder,
parser.promise_type
)
with open(parser.output, "w") as outputfile:
json.dump(status_json, outputfile)
os.remove(parser.pid_path)
def updateStatusHistoryFolder(name, status_file, history_folder, promise_type):
history_path = os.path.join(history_folder)
if not os.path.exists(status_file):
return
if not os.path.exists(history_folder):
return
if not os.path.exists(history_path):
try:
os.makedirs(history_path)
except OSError, e:
if e.errno == os.errno.EEXIST and os.path.isdir(history_path):
pass
else: raise
with open(status_file, 'r') as sf:
try:
status_dict = json.loads(sf.read())
except ValueError:
traceback.print_exc()
return
if promise_type == 'status':
filename = '%s.history.json' % name
history_file = os.path.join(history_path, filename)
# Remove links from history (not needed)
status_dict.pop('_links', None)
if not os.path.exists(history_file):
with open(history_file, 'w') as f_history:
data_dict = {
"date": time.time(),
"data": [status_dict]
}
f_history.write(json.dumps(data_dict))
else:
# Remove useless informations
status_dict.pop('hosting_subscription', '')
status_dict.pop('title', '')
status_dict.pop('instance', '')
status_dict.pop('type', '')
with open (history_file, mode="r+") as f_history:
f_history.seek(0,2)
position = f_history.tell() -2
f_history.seek(position)
#f_history.write(',%s]}' % str(status_dict))
f_history.write('%s}' % ',{}]'.format(json.dumps(status_dict)))
elif promise_type == 'report':
# keep_item_amount = 3
filename = '%s.history.json' % (
name)
copyfile(status_file, os.path.join(history_path, filename))
"""# Don't let history foler grow too much, keep xx files
file_list = filter(os.path.isfile,
glob.glob("%s/*.%s.history.json" % (history_path, promise_type))
)
file_count = len(file_list)
if file_count > keep_item_amount:
file_list.sort(key=lambda x: os.path.getmtime(x))
while file_count > keep_item_amount:
to_delete = file_list.pop(0)
try:
os.unlink(to_delete)
file_count -= 1
except OSError:
raise"""
def generateStatusJsonFromProcess(process, start_date=None, title=None):
stdout, stderr = process.communicate()
try:
status_json = json.loads(stdout)
except ValueError:
status_json = {}
if process.returncode != 0:
status_json["status"] = "ERROR"
elif not status_json.get("status"):
status_json["status"] = "OK"
if stderr:
status_json["message"] = stderr
if start_date:
status_json["start-date"] = start_date
if title:
status_json["title"] = title
return status_json
def executeCommand(args):
return subprocess.Popen(
args,
#cwd=instance_path,
#env=None if sys.platform == 'cygwin' else {},
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
if __name__ == "__main__":
sys.exit(main())
stack/monitor/scripts/status2rss.py deleted 100644 → 0
View file @ 6e7b8e62
import sys
import os
import json
from datetime import datetime
import base64
import hashlib
import PyRSS2Gen
import argparse
def parseArguments():
"""
Parse arguments for monitor Rss Generator.
"""
parser = argparse.ArgumentParser()
parser.add_argument('--items_folder',
help='Path where to get *.status.json files which contain result of promises.')
parser.add_argument('--output',
help='The Path of file where feed file will be saved.')
parser.add_argument('--feed_url',
help='Url of this feed file.')
parser.add_argument('--public_url',
help='Monitor Instance public URL.')
parser.add_argument('--private_url',
help='Monitor Instance private URL.')
parser.add_argument('--instance_name',
default='UNKNOW Software Instance',
help='Software Instance name.')
parser.add_argument('--hosting_name',
default='',
help='Hosting Subscription name.')
return parser.parse_args()
def getKey(item):
return item.pubDate
def main():
parser = parseArguments()
rss_item_list = []
report_date = datetime.utcnow()
for filename in os.listdir(parser.items_folder):
if filename.endswith(".status.json"):
filepath = os.path.join(parser.items_folder, filename)
result_dict = None
try:
result_dict = json.load(open(filepath, "r"))
except ValueError:
print "Failed to load json file: %s" % filepath
continue
description = result_dict.get('message', '')
event_time = datetime.fromtimestamp(result_dict['change-time'])
rss_item = PyRSS2Gen.RSSItem(
categories = [result_dict['status']],
source = PyRSS2Gen.Source(result_dict['title'], parser.public_url),
title = '[%s] %s' % (result_dict['status'], result_dict['title']),
comments = description,
description = "%s: %s\n%s" % (event_time, result_dict['status'], description),
link = parser.private_url,
pubDate = event_time,
guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (parser.hosting_name, result_dict['title'])))
)
rss_item_list.append(rss_item)
### Build the rss feed
sorted(rss_item_list, key=getKey)
rss_feed = PyRSS2Gen.RSS2 (
title = parser.instance_name,
link = parser.feed_url,
description = parser.hosting_name,
lastBuildDate = report_date,
items = rss_item_list
)
with open(parser.output, 'w') as frss:
frss.write(rss_feed.to_xml())
if __name__ == "__main__":
exit(main())
stack/slapos.cfg
View file @ e190f434
......@@ -108,33 +108,32 @@ zc.recipe.egg = 1.3.2.post5
hexagonit.recipe.download = 1.7.post4
Jinja2 = 2.8
PyYAML = 3.11
PyYAML = 3.12
Werkzeug = 0.11.10
buildout-versions = 1.7
cffi = 1.7.0
click = 6.6
cliff = 2.1.0
cliff = 2.2.0
cmd2 = 0.6.8
collective.recipe.template = 1.13
cryptography = 1.4
cryptography = 1.5
decorator = 4.0.10
idna = 2.1
inotifyx = 0.2.2
itsdangerous = 0.24
lxml = 3.6.0
lxml = 3.6.4
meld3 = 1.0.2
netaddr = 0.7.18
pbr = 1.10.0
plone.recipe.command = 1.1
prettytable = 0.7.2
psutil = 4.3.0
pyOpenSSL = 16.0.0
pyOpenSSL = 16.1.0
pyasn1 = 0.1.9
pyparsing = 2.1.5
pytz = 2016.4
requests = 2.10.0
pyparsing = 2.1.8
pytz = 2016.6.1
requests = 2.11.1
setuptools = 19.6.2
simplejson = 3.8.2
six = 1.10.0
slapos.cookbook = 1.0.31
slapos.core = 1.3.15
......@@ -142,7 +141,7 @@ slapos.extension.strip = 0.1
slapos.libnetworkcache = 0.14.5
slapos.recipe.build = 0.23
slapos.recipe.cmmi = 0.2
stevedore = 1.15.0
stevedore = 1.17.1
unicodecsv = 0.14.1
xml-marshaller = 0.9.7
......@@ -155,7 +154,7 @@ Flask = 0.11.1
MarkupSafe = 0.23
# Required by:
# cryptography==1.4
# cryptography==1.5
enum34 = 1.1.6
# Required by:
......@@ -163,7 +162,7 @@ enum34 = 1.1.6
functools32 = 3.2.3.post2
# Required by:
# cryptography==1.4
# cryptography==1.5
ipaddress = 1.0.16
# Required by:
......@@ -176,6 +175,7 @@ lock-file = 2.0
# Required by:
# slapos.core==1.3.15
# XXX 'slapos node format' raises an exception with netifaces 0.10.5.
netifaces = 0.10.4
# Required by:
......@@ -184,11 +184,11 @@ pycparser = 2.14
# Required by:
# slapos.core==1.3.15
supervisor = 3.3.0
supervisor = 3.3.1
# Required by:
# slapos.core==1.3.15
uritemplate = 0.6
uritemplate = 3.0.0
# Required by:
# slapos.core==1.3.15
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7