Commit 7be9ec30 authored by Alain Takoudjou's avatar Alain Takoudjou

Update Release Candidate

parents 29a41407 8b64fd3a
Changes Changes
======= =======
1.0.62 (2018-04-10)
-------------------
* promise.plugin: new recipe for python promises plugin script generation
1.0.59 (2018-03-15) 1.0.59 (2018-03-15)
------------------- -------------------
* librecipe.execute: fix convert process arguments to string formatting. * librecipe.execute: fix convert process arguments to string formatting.
......
...@@ -34,9 +34,9 @@ md5sum = 2202b18f269ad606d70e1864857ed93c ...@@ -34,9 +34,9 @@ md5sum = 2202b18f269ad606d70e1864857ed93c
[apache] [apache]
# inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/ # inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
version = 2.4.29 version = 2.4.33
url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2 url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
md5sum = 0c599404ef6b69eee95bcd9fcd094407 md5sum = 6ef469d3f16fffeb688bc6e0346823e5
pre-configure = pre-configure =
cp -ar ${apr:location}/apr-${apr:version} srclib/apr/ && cp -ar ${apr:location}/apr-${apr:version} srclib/apr/ &&
cp -ar ${apr-util:location}/apr-util-${apr-util:version} srclib/apr-util cp -ar ${apr-util:location}/apr-util-${apr-util:version} srclib/apr-util
......
diff -uNr Acquisition-2.13.8/src/Acquisition/_Acquisition.c Acquisition-2.13.8nxd001/src/Acquisition/_Acquisition.c diff -uNr Acquisition-2.13.12.orig/src/Acquisition/_Acquisition.c Acquisition-2.13.12/src/Acquisition/_Acquisition.c
--- Acquisition-2.13.8/src/Acquisition/_Acquisition.c 2011-06-11 17:19:14.000000000 +0200 --- Acquisition-2.13.12.orig/src/Acquisition/_Acquisition.c 2017-12-01 12:01:34.000000000 +0100
+++ Acquisition-2.13.8nxd001/src/Acquisition/_Acquisition.c 2013-10-31 16:24:55.665085888 +0100 +++ Acquisition-2.13.12/src/Acquisition/_Acquisition.c 2018-04-09 17:10:15.394836944 +0200
@@ -448,6 +448,64 @@ @@ -449,6 +449,64 @@
} }
static PyObject * static PyObject *
...@@ -66,7 +66,7 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/_Acquisition.c Acquisition-2.13.8nx ...@@ -66,7 +66,7 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/_Acquisition.c Acquisition-2.13.8nx
Wrapper_acquire(Wrapper *self, PyObject *oname, Wrapper_acquire(Wrapper *self, PyObject *oname,
PyObject *filter, PyObject *extra, PyObject *orig, PyObject *filter, PyObject *extra, PyObject *orig,
int explicit, int containment); int explicit, int containment);
@@ -545,8 +603,8 @@ @@ -589,8 +647,8 @@
Py_XDECREF(r); Py_XDECREF(v); Py_XDECREF(tb); Py_XDECREF(r); Py_XDECREF(v); Py_XDECREF(tb);
r=NULL; r=NULL;
} }
...@@ -77,7 +77,7 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/_Acquisition.c Acquisition-2.13.8nx ...@@ -77,7 +77,7 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/_Acquisition.c Acquisition-2.13.8nx
{ {
if (r==Acquired) if (r==Acquired)
{ {
@@ -670,7 +728,7 @@ @@ -714,7 +772,7 @@
Py_XDECREF(r); Py_XDECREF(v); Py_XDECREF(tb); Py_XDECREF(r); Py_XDECREF(v); Py_XDECREF(tb);
r=NULL; r=NULL;
...@@ -86,7 +86,7 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/_Acquisition.c Acquisition-2.13.8nx ...@@ -86,7 +86,7 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/_Acquisition.c Acquisition-2.13.8nx
if (r == Acquired) { if (r == Acquired) {
Py_DECREF(r); Py_DECREF(r);
} }
@@ -707,7 +765,7 @@ @@ -751,7 +809,7 @@
Wrapper_getattro(Wrapper *self, PyObject *oname) Wrapper_getattro(Wrapper *self, PyObject *oname)
{ {
if (self->obj || self->container) if (self->obj || self->container)
...@@ -95,18 +95,18 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/_Acquisition.c Acquisition-2.13.8nx ...@@ -95,18 +95,18 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/_Acquisition.c Acquisition-2.13.8nx
/* Maybe we are getting initialized? */ /* Maybe we are getting initialized? */
return Py_FindAttr(OBJECT(self),oname); return Py_FindAttr(OBJECT(self),oname);
@@ -724,7 +782,7 @@ @@ -776,7 +834,7 @@
return Py_FindAttr(OBJECT(self),oname); result = Py_FindAttr(OBJECT(self),oname);
if (self->obj || self->container) else if (self->obj || self->container)
- return Wrapper_findattr(self, oname, NULL, NULL, NULL, 1, 0, 0, 0); - result = Wrapper_findattr(self, oname, NULL, NULL, NULL, 1, 0, 0, 0);
+ return Wrapper_findattr(self, oname, NULL, NULL, OBJECT(self), 1, 0, 0, 0); + result = Wrapper_findattr(self, oname, NULL, NULL, OBJECT(self), 1, 0, 0, 0);
/* Maybe we are getting initialized? */ /* Maybe we are getting initialized? */
return Py_FindAttr(OBJECT(self),oname); else result = Py_FindAttr(OBJECT(self),oname);
diff -uNr Acquisition-2.13.8/src/Acquisition/test_dynamic_acquisition.py Acquisition-2.13.8nxd001/src/Acquisition/test_dynamic_acquisition.py diff -uNr Acquisition-2.13.12.orig/src/Acquisition/test_dynamic_acquisition.py Acquisition-2.13.12/src/Acquisition/test_dynamic_acquisition.py
--- Acquisition-2.13.8/src/Acquisition/test_dynamic_acquisition.py 1970-01-01 01:00:00.000000000 +0100 --- Acquisition-2.13.12.orig/src/Acquisition/test_dynamic_acquisition.py 1970-01-01 01:00:00.000000000 +0100
+++ Acquisition-2.13.8nxd001/src/Acquisition/test_dynamic_acquisition.py 2013-10-31 16:24:55.665085888 +0100 +++ Acquisition-2.13.12/src/Acquisition/test_dynamic_acquisition.py 2018-04-09 17:07:34.863985305 +0200
@@ -0,0 +1,160 @@ @@ -0,0 +1,160 @@
+############################################################################## +##############################################################################
+# +#
...@@ -268,10 +268,10 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/test_dynamic_acquisition.py Acquisi ...@@ -268,10 +268,10 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/test_dynamic_acquisition.py Acquisi
+ >>> assert not b.a.aq_inContextOf('somestring') + >>> assert not b.a.aq_inContextOf('somestring')
+''' +'''
+ +
diff -uNr Acquisition-2.13.8/src/Acquisition/tests.py Acquisition-2.13.8nxd001/src/Acquisition/tests.py diff -uNr Acquisition-2.13.12.orig/src/Acquisition/tests.py Acquisition-2.13.12/src/Acquisition/tests.py
--- Acquisition-2.13.8/src/Acquisition/tests.py 2011-06-11 17:09:38.000000000 +0200 --- Acquisition-2.13.12.orig/src/Acquisition/tests.py 2017-12-01 12:01:34.000000000 +0100
+++ Acquisition-2.13.8nxd001/src/Acquisition/tests.py 2013-10-31 16:24:55.669085888 +0100 +++ Acquisition-2.13.12/src/Acquisition/tests.py 2018-04-09 17:07:34.867985476 +0200
@@ -2552,6 +2552,7 @@ @@ -2588,6 +2588,7 @@
def test_suite(): def test_suite():
return unittest.TestSuite(( return unittest.TestSuite((
DocTestSuite(), DocTestSuite(),
...@@ -279,10 +279,10 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/tests.py Acquisition-2.13.8nxd001/s ...@@ -279,10 +279,10 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/tests.py Acquisition-2.13.8nxd001/s
DocFileSuite('README.txt', package='Acquisition'), DocFileSuite('README.txt', package='Acquisition'),
unittest.makeSuite(TestParent), unittest.makeSuite(TestParent),
unittest.makeSuite(TestAcquire), unittest.makeSuite(TestAcquire),
diff -uNr Acquisition-2.13.8/src/Acquisition.egg-info/SOURCES.txt Acquisition-2.13.8nxd001/src/Acquisition.egg-info/SOURCES.txt diff -uNr Acquisition-2.13.12.orig/src/Acquisition.egg-info/SOURCES.txt Acquisition-2.13.12/src/Acquisition.egg-info/SOURCES.txt
--- Acquisition-2.13.8/src/Acquisition.egg-info/SOURCES.txt 2011-06-11 17:21:18.000000000 +0200 --- Acquisition-2.13.12.orig/src/Acquisition.egg-info/SOURCES.txt 2017-12-01 12:08:29.000000000 +0100
+++ Acquisition-2.13.8nxd001/src/Acquisition.egg-info/SOURCES.txt 2013-10-31 16:24:55.669085888 +0100 +++ Acquisition-2.13.12/src/Acquisition.egg-info/SOURCES.txt 2018-04-09 17:07:34.867985476 +0200
@@ -15,6 +15,7 @@ @@ -10,6 +10,7 @@
src/Acquisition/_Acquisition.c src/Acquisition/_Acquisition.c
src/Acquisition/__init__.py src/Acquisition/__init__.py
src/Acquisition/interfaces.py src/Acquisition/interfaces.py
......
From 8b31ccec54584a287cc61501948283d7d6ee7073 Mon Sep 17 00:00:00 2001
From: Julien Muchembled <jm@nexedi.com>
Date: Mon, 26 Mar 2018 20:39:07 +0200
Subject: [PATCH] Enable TCP_NODELAY for inet(6) sockets
See commit 3d886d426243655b9f5a2528636e42b5c7662c19.
---
src/ZEO/zrpc/client.py | 2 ++
src/ZEO/zrpc/server.py | 1 +
2 files changed, 3 insertions(+)
diff --git a/src/ZEO/zrpc/client.py b/src/ZEO/zrpc/client.py
index 32a7a877..669f5962 100644
--- a/src/ZEO/zrpc/client.py
+++ b/src/ZEO/zrpc/client.py
@@ -568,6 +568,8 @@ def __init__(self, domain, addr, mgr, client):
self.close()
return
self.sock.setblocking(0)
+ if domain in (socket.AF_INET, socket.AF_INET6):
+ self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.state = "opened"
def connect_procedure(self):
diff --git a/src/ZEO/zrpc/server.py b/src/ZEO/zrpc/server.py
index b83cc004..af91e3e4 100644
--- a/src/ZEO/zrpc/server.py
+++ b/src/ZEO/zrpc/server.py
@@ -66,6 +66,7 @@ def _open_socket(self):
socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, True)
else:
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+ self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
else:
self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.set_reuse_addr()
--
2.14.1
...@@ -14,8 +14,8 @@ parts = ...@@ -14,8 +14,8 @@ parts =
[gcc-common] [gcc-common]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://ftp.gnu.org/gnu/gcc/gcc-5.4.0/gcc-5.4.0.tar.bz2 url = http://ftp.gnu.org/gnu/gcc/gcc-5.5.0/gcc-5.5.0.tar.gz
md5sum = 4c626ac2a83ef30dfb9260e6f59c2b30 md5sum = 781bc0195edeb0ceaace8428f63ae63d
# make install does not work when several core are used # make install does not work when several core are used
make-targets = install -j1 make-targets = install -j1
......
...@@ -3,6 +3,7 @@ ...@@ -3,6 +3,7 @@
extends = extends =
../findutils/buildout.cfg ../findutils/buildout.cfg
../git/buildout.cfg ../git/buildout.cfg
../pkgconfig/buildout.cfg
parts = gowork parts = gowork
...@@ -64,7 +65,15 @@ environment-extra = ...@@ -64,7 +65,15 @@ environment-extra =
# github.com/pkg/profile \ # github.com/pkg/profile \
# golang.org/x/perf/cmd/benchstat # golang.org/x/perf/cmd/benchstat
# #
# it is possible to specify Go build flags used for compilation e.g. this way: # For Cgo support pkg-config is made pre-available by gowork, and users
# should list paths where to search for pkg-config files, e.g. this way:
#
# [gowork]
# cpkgpath =
# ${sqlite3:location}/lib/pkgconfig
# ${zlib:location}/lib/pkgconfig
#
# It is also possible to specify Go build flags used for compilation e.g. this way:
# #
# [gowork] # [gowork]
# buildflags = -race # buildflags = -race
...@@ -80,6 +89,9 @@ golang = ${golang19:location} ...@@ -80,6 +89,9 @@ golang = ${golang19:location}
# no special build flags by default # no special build flags by default
buildflags = buildflags =
# empty pkg-config path by default
cpkgpath =
# everything is done by dependent parts # everything is done by dependent parts
recipe = plone.recipe.command recipe = plone.recipe.command
command = : command = :
...@@ -93,7 +105,7 @@ recipe = slapos.recipe.template ...@@ -93,7 +105,7 @@ recipe = slapos.recipe.template
url = ${:_profile_base_location_}/goenv.sh.in url = ${:_profile_base_location_}/goenv.sh.in
output = ${gowork:directory}/env.sh output = ${gowork:directory}/env.sh
depends = ${gowork.mkdir:recipe} depends = ${gowork.mkdir:recipe}
md5sum = a9a265135931b3da53f4392870748264 md5sum = 7a067a3974c446c3eaa0e82818ba1adb
[gowork.mkdir] [gowork.mkdir]
# NOTE do not use slapos.cookbook:mkdirectory here - if anything in software (not instance) # NOTE do not use slapos.cookbook:mkdirectory here - if anything in software (not instance)
......
...@@ -3,13 +3,16 @@ ...@@ -3,13 +3,16 @@
# ---- 8< ---- (buildout substitution here) # ---- 8< ---- (buildout substitution here)
# PATH so that go & friends work out of the box # PATH so that go & friends work out of the box
export PATH=${gowork:golang}/bin:${git:location}/bin:${buildout:bin-directory}:$PATH export PATH=${gowork:golang}/bin:${git:location}/bin:${pkgconfig:location}/bin:${buildout:bin-directory}:$PATH
X=${gowork:directory} X=${gowork:directory}
export PKG_CONFIG_PATH=$(echo -n "${gowork:cpkgpath}" |tr '\n' ':'):$PKG_CONFIG_PATH
# ---- 8< ---- # ---- 8< ----
export GOPATH=$X:$GOPATH export GOPATH=$X:$GOPATH
export PATH=$X/bin:$PATH export PATH=$X/bin:$PATH
export PS1="(`basename $X`) $PS1" export PS1="(`basename $X`) $PS1"
# strip trailing : from $GOPATH # strip trailing : from $GOPATH, $PKG_CONFIG_PATH
GOPATH=$${GOPATH%:} GOPATH=$${GOPATH%:}
PKG_CONFIG_PATH=$${PKG_CONFIG_PATH%:}
...@@ -14,8 +14,8 @@ extends = ...@@ -14,8 +14,8 @@ extends =
[groonga] [groonga]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://packages.groonga.org/source/groonga/groonga-7.0.5.tar.gz url = http://packages.groonga.org/source/groonga/groonga-8.0.0.tar.gz
md5sum = d42dd98c5272ed2b9ab50ded54eb818b md5sum = f4641e68a5301e641399f2d112a494dd
# temporary patch to respect more tokens in natural language mode. # temporary patch to respect more tokens in natural language mode.
patches = patches =
${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b ${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b
......
...@@ -14,6 +14,8 @@ configure-options = ...@@ -14,6 +14,8 @@ configure-options =
--prefix=${:location} --prefix=${:location}
--disable-static --disable-static
--enable-rpath --enable-rpath
patches =
${:_profile_base_location_}/fix-glibc2.26-ftbfs.patch#f1622be16964029fc66a70b8f9e1693c
[icu4c-slaposgcc] [icu4c-slaposgcc]
# need for onlyoffice-core # need for onlyoffice-core
......
--- source/i18n/digitlst.cpp 2018-03-29 10:59:45.706746334 +0900
+++ source/i18n/digitlst.cpp 2018-03-29 11:01:15.933534983 +0900
@@ -61,11 +61,7 @@
#endif
#if U_USE_STRTOD_L
-# if U_PLATFORM_USES_ONLY_WIN32_API || U_PLATFORM == U_PF_CYGWIN
-# include <locale.h>
-# else
-# include <xlocale.h>
-# endif
+# include <locale.h>
#endif
// ***************************************************************************
diff --git a/mrn_mysql_compat.h b/mrn_mysql_compat.h
index d2b227ff..337de968 100644
--- a/mrn_mysql_compat.h
+++ b/mrn_mysql_compat.h
@@ -362,6 +362,7 @@
#if defined(MRN_MARIADB_P) && \
((MYSQL_VERSION_ID >= 100207) || \
+ ((MYSQL_VERSION_ID >= 100126) && (MYSQL_VERSION_ID < 100200)) || \
((MYSQL_VERSION_ID >= 50557) && (MYSQL_VERSION_ID < 100000)))
# define mrn_create_partition_name(out, \
out_length, \
...@@ -28,8 +28,8 @@ parts = ...@@ -28,8 +28,8 @@ parts =
[mariadb] [mariadb]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
version = 10.1.31 version = 10.1.32
md5sum = 14ab0398c019eb531bc29f2c437ccb51 md5sum = 389ce891cf00957748ba98b09f433c14
patch-options = -p0 patch-options = -p0
patches = patches =
${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3 ${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3
...@@ -84,8 +84,8 @@ post-install = ...@@ -84,8 +84,8 @@ post-install =
# mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users. # mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
# http://mroonga.github.com/ # http://mroonga.github.com/
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://packages.groonga.org/source/mroonga/mroonga-7.05.tar.gz url = http://packages.groonga.org/source/mroonga/mroonga-8.00.tar.gz
md5sum = d289667a60a6aa78d8bc276ab61b2996 md5sum = 8b906705be5fc14785207ca6752b96e9
pre-configure = set -e pre-configure = set -e
rm -rf fake_mariadb_source rm -rf fake_mariadb_source
mkdir -p fake_mariadb_source mkdir -p fake_mariadb_source
...@@ -106,7 +106,6 @@ configure-options = ...@@ -106,7 +106,6 @@ configure-options =
patch-options = -p1 patch-options = -p1
patches = patches =
${:_profile_base_location_}/mroonga_boolean.patch#c818568fe35ca6a4298f18e575d962a0 ${:_profile_base_location_}/mroonga_boolean.patch#c818568fe35ca6a4298f18e575d962a0
${:_profile_base_location_}/1a910088ec55c61434029d8e1cc3c6192ac508d7.diff#a8b8df4dd870a1a269dfd9dc6941467b
pre-build = pre-build =
sed -i -e "s,${mariadb:location}/include,$(pwd)/fake_mariadb_source/include,g" Makefile */Makefile sed -i -e "s,${mariadb:location}/include,$(pwd)/fake_mariadb_source/include,g" Makefile */Makefile
environment = environment =
......
# Do not extend any file that touch buildout:parts. # Do not extend any file that touch buildout:parts.
[mariadb] [mariadb]
version = 10.2.13 version = 10.2.14
md5sum = 20c61bd4059ba287e54cfb2862bae81d md5sum = d98cce6f3c0e2971afa061fc67183b91
stable-patches = stable-patches =
configure-options += configure-options +=
-DCMAKE_C_COMPILER=${gcc:location}/bin/gcc -DCMAKE_C_COMPILER=${gcc:location}/bin/gcc
......
...@@ -5,7 +5,7 @@ parts = ...@@ -5,7 +5,7 @@ parts =
[noVNC] [noVNC]
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
ignore-existing = true ignore-existing = true
# version-0.6.1 release from 04 Jul 2016 # version-1.0.0 release on Feb 22, 2018
url = https://github.com/kanaka/noVNC/archive/v0.6.1.tar.gz url = https://github.com/novnc/noVNC/archive/v1.0.0.tar.gz
md5sum = d153c6aa69a9178081768fecbace1932 md5sum = d63c2944abd12fecec6ec504e82f27c8
strip-top-level-dir = true strip-top-level-dir = true
[buildout] [buildout]
extends = extends =
../binutils/buildout.cfg
../gcc/buildout.cfg ../gcc/buildout.cfg
../libxml2/buildout.cfg ../libxml2/buildout.cfg
../zlib/buildout.cfg ../zlib/buildout.cfg
...@@ -15,12 +16,12 @@ recipe = slapos.recipe.cmmi ...@@ -15,12 +16,12 @@ recipe = slapos.recipe.cmmi
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
# This url contains the hash provided by the DocumentServer core submodule hash. # This url contains the hash provided by the DocumentServer core submodule hash.
# https://github.com/ONLYOFFICE/DocumentServer/ # https://github.com/ONLYOFFICE/DocumentServer/
url = https://lab.nexedi.com/bk/onlyoffice_core/repository/archive.tar.bz2?ref=b051e75b179b3599c09937668fbbd2d7e2c50683 url = https://lab.nexedi.com/bk/onlyoffice_core/repository/archive.tar.bz2?ref=8a40eb47bd80a40ecde14c223525b21852d2fc9f
md5sum = b2713373d687dd1c7121c286fa156626 md5sum = 9cd1cd731202511e475971eee58ba7b6
configure-command = true configure-command = true
make-targets = lib bin make-targets = lib bin
environment = environment =
PATH=${gcc:location}/bin:${qt5-qmake:location}/bin:%(PATH)s PATH=${binutils:location}/bin:${gcc:location}/bin:${qt5-qmake:location}/bin:%(PATH)s
CXXFLAGS=-I${libxml2:location}/include -I${zlib:location}/include -I${icu4c-slaposgcc:location}/include -I${boost-lib:location}/include -Wno-comment -Wno-deprecated-declarations -Wno-endif-labels -Wno-parentheses -Wno-reorder -Wno-sign-compare -Wno-switch -Wno-unknown-pragmas -Wno-unused CXXFLAGS=-I${libxml2:location}/include -I${zlib:location}/include -I${icu4c-slaposgcc:location}/include -I${boost-lib:location}/include -Wno-comment -Wno-deprecated-declarations -Wno-endif-labels -Wno-parentheses -Wno-reorder -Wno-sign-compare -Wno-switch -Wno-unknown-pragmas -Wno-unused
LDFLAGS=-L${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib -L${gcc:location}/lib64 -Wl,-rpath=${gcc:location}/lib64 -L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${icu4c-slaposgcc:location}/lib -Wl,-rpath=${icu4c-slaposgcc:location}/lib -L${boost-lib:location}/lib -Wl,-rpath=${boost-lib:location}/lib -Wl,-rpath=${:location}/lib LDFLAGS=-L${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib -L${gcc:location}/lib64 -Wl,-rpath=${gcc:location}/lib64 -L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${icu4c-slaposgcc:location}/lib -Wl,-rpath=${icu4c-slaposgcc:location}/lib -L${boost-lib:location}/lib -Wl,-rpath=${boost-lib:location}/lib -Wl,-rpath=${:location}/lib
post-install = post-install =
......
...@@ -16,8 +16,8 @@ parts = ...@@ -16,8 +16,8 @@ parts =
[openssl] [openssl]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = https://www.openssl.org/source/openssl-1.0.2n.tar.gz url = https://www.openssl.org/source/openssl-1.0.2o.tar.gz
md5sum = 13bdc1b1d1ff39b6fd42a255e74676a4 md5sum = 44279b8557c3247cbe324e2322ecd114
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
# 'prefix' option to override --openssldir/--prefix (which is useful # 'prefix' option to override --openssldir/--prefix (which is useful
# when combined with INSTALL_PREFIX). Used by slapos.package.git/obs # when combined with INSTALL_PREFIX). Used by slapos.package.git/obs
......
...@@ -64,7 +64,6 @@ md5sum = 096c1c18b44c269808bd815d58c53c8f ...@@ -64,7 +64,6 @@ md5sum = 096c1c18b44c269808bd815d58c53c8f
[debian-amd64-jessie-netinst.iso] [debian-amd64-jessie-netinst.iso]
<= debian-amd64-netinst-base <= debian-amd64-netinst-base
release = archive
version = 8.10.0 version = 8.10.0
md5sum = 19dcfc381bd3e609c6056216d203f5bc md5sum = 19dcfc381bd3e609c6056216d203f5bc
......
...@@ -5,8 +5,8 @@ parts = ...@@ -5,8 +5,8 @@ parts =
[sqlite3] [sqlite3]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://sqlite.org/2017/sqlite-autoconf-3190000.tar.gz url = https://sqlite.org/2018/sqlite-autoconf-3220000.tar.gz
md5sum = 2426883b4dd3a9fd6aeb28a16b81a72b md5sum = 96b5648d542e8afa6ab7ffb8db8ddc3d
configure-options = configure-options =
--disable-static --disable-static
--enable-readline --enable-readline
......
...@@ -22,7 +22,7 @@ dist = ${vm-debian:dists} ...@@ -22,7 +22,7 @@ dist = ${vm-debian:dists}
[vm-debian] [vm-debian]
recipe = slapos.recipe.build:vm.install-debian recipe = slapos.recipe.build:vm.install-debian
environment = vm-install-environment environment = vm-install-environment
dists = debian-jessie dists = debian-stretch
size = 1Gi size = 1Gi
late-command = late-command =
# rdnssd causes too much trouble with recent QEMU, because the latter acts as # rdnssd causes too much trouble with recent QEMU, because the latter acts as
...@@ -55,18 +55,22 @@ preseed.recommends = false ...@@ -55,18 +55,22 @@ preseed.recommends = false
preseed.tasks = preseed.tasks =
[debian-squeeze] [debian-squeeze]
<= debian-jessie <= debian-stretch
x86_64.iso = debian-amd64-squeeze-netinst.iso x86_64.iso = debian-amd64-squeeze-netinst.iso
[debian-wheezy] [debian-wheezy]
<= debian-jessie <= debian-stretch
x86_64.iso = debian-amd64-wheezy-netinst.iso x86_64.iso = debian-amd64-wheezy-netinst.iso
[debian-jessie] [debian-jessie]
<= debian-stretch
x86_64.iso = debian-amd64-jessie-netinst.iso
[debian-stretch]
x86_64.iso = debian-amd64-netinst.iso x86_64.iso = debian-amd64-netinst.iso
x86_64.kernel = install.amd/vmlinuz x86_64.kernel = install.amd/vmlinuz
x86_64.initrd = install.amd/initrd.gz x86_64.initrd = install.amd/initrd.gz
[debian-stretch] [debian-buster]
<= debian-jessie <= debian-stretch
x86_64.iso = debian-amd64-testing-netinst.iso x86_64.iso = debian-amd64-testing-netinst.iso
...@@ -28,7 +28,7 @@ from setuptools import setup, find_packages ...@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob import glob
import os import os
version = '1.0.60.dev0' version = '1.0.62'
name = 'slapos.cookbook' name = 'slapos.cookbook'
long_description = open("README.rst").read() + "\n" + \ long_description = open("README.rst").read() + "\n" + \
open("CHANGES.rst").read() + "\n" open("CHANGES.rst").read() + "\n"
...@@ -99,7 +99,6 @@ setup(name=name, ...@@ -99,7 +99,6 @@ setup(name=name,
'dropbear.add_authorized_key = slapos.recipe.dropbear:AddAuthorizedKey', 'dropbear.add_authorized_key = slapos.recipe.dropbear:AddAuthorizedKey',
'dropbear.client = slapos.recipe.dropbear:Client', 'dropbear.client = slapos.recipe.dropbear:Client',
'duplicity = slapos.recipe.duplicity:Recipe', 'duplicity = slapos.recipe.duplicity:Recipe',
'egg_test = slapos.recipe.erp5_test:EggTestRecipe',
'equeue = slapos.recipe.equeue:Recipe', 'equeue = slapos.recipe.equeue:Recipe',
'erp5.promise = slapos.recipe.erp5_promise:Recipe', 'erp5.promise = slapos.recipe.erp5_promise:Recipe',
'erp5.test = slapos.recipe.erp5_test:Recipe', 'erp5.test = slapos.recipe.erp5_test:Recipe',
...@@ -151,6 +150,7 @@ setup(name=name, ...@@ -151,6 +150,7 @@ setup(name=name,
'postgres.export = slapos.recipe.postgres.backup:ExportRecipe', 'postgres.export = slapos.recipe.postgres.backup:ExportRecipe',
'postgres.import = slapos.recipe.postgres.backup:ImportRecipe', 'postgres.import = slapos.recipe.postgres.backup:ImportRecipe',
'proactive = slapos.recipe.proactive:Recipe', 'proactive = slapos.recipe.proactive:Recipe',
'promise.plugin= slapos.recipe.promise_plugin:Recipe',
'publish = slapos.recipe.publish:Recipe', 'publish = slapos.recipe.publish:Recipe',
'publish.serialised = slapos.recipe.publish:Serialised', 'publish.serialised = slapos.recipe.publish:Serialised',
'publish-early = slapos.recipe.publish_early:Recipe', 'publish-early = slapos.recipe.publish_early:Recipe',
......
...@@ -113,26 +113,3 @@ class CloudoooRecipe(GenericBaseRecipe): ...@@ -113,26 +113,3 @@ class CloudoooRecipe(GenericBaseRecipe):
return path_list return path_list
class EggTestRecipe(GenericBaseRecipe):
"""
Recipe used to create wrapper used to run test suite (python setup.py test)
off a list of Python eggs.
"""
def install(self):
test_list = self.options['test-list'].strip().replace('\n', ',')
common_dict = {}
if self.options.get('environment'):
environment_part = self.buildout.get(self.options['environment'])
if environment_part:
common_dict['environment'] = dict(environment_part)
if 'prepend-path' in self.options:
common_dict['prepend_path'] = self.options['prepend-path']
return self.createPythonScript(
self.options['run-test-suite'], __name__ + '.test.runTestSuite',
((self.options['run-test-suite-binary'],
"--source_code_path_list", test_list),
common_dict)
)
##############################################################################
#
# Copyright (c) 2018 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import logging, os, sys
import zc.buildout.easy_install
from slapos.recipe.librecipe import GenericBaseRecipe
script_template = '''# This script is auto generated by slapgrid, do not edit!
import sys
sys.path[0:0] = [
%(path)s
]
extra_config_dict = {
%(config)s
}
# We want to cleanup all imported modules from slapos namespace, because
# they will conflict with slapos.core.
# In fact as slapos.grid is already imported, the promise cannot reimport
# his own slapos.grid from an updated sys.path. Then all module imports which
# are not in slapos.core will fail.
#
# call reload(slapos) only solve a part of problem because not all modules
# will be reloaded, and some new modules won't be added.
# The solution is to delete all cached 'slapos' modules as well as all cached
# 'pkg_resources' modules which is responsible of namespace declaration.
# They will be re-imported again using the updated sys.path
for module in sys.modules.keys():
if 'slapos' in module or 'pkg_resources' in module:
del sys.modules[module]
import slapos.grid.promise
%(content)s
'''
class Recipe(GenericBaseRecipe):
_WORKING_SET_CACHE_NAME = "slapos.cookbook_pplugin_ws_cache"
def __init__(self, buildout, name, options):
buildout_section = buildout['buildout']
options['eggs-directory'] = buildout_section['eggs-directory']
options['develop-eggs-directory'] = buildout_section['develop-eggs-directory']
super(Recipe, self).__init__(buildout, name, options)
def _get_cache_storage(self):
"""Return a mapping where to store generated working sets.
from https://github.com/buildout/buildout/blob/master/zc.recipe.egg_/src/zc/recipe/egg/egg.py#L170
"""
cache_storage = getattr(
self.buildout,
self._WORKING_SET_CACHE_NAME,
None
)
if cache_storage is None:
cache_storage = {}
setattr(
self.buildout,
self._WORKING_SET_CACHE_NAME,
cache_storage
)
return cache_storage
def install(self):
develop_eggs_dir = self.options['develop-eggs-directory']
eggs_dir = self.options['eggs-directory']
egg_list = [
egg.strip()
for egg in self.options['eggs'].split('\n')
if egg.strip()
]
cache_storage = self._get_cache_storage()
cache_key = (
tuple(egg_list),
eggs_dir,
develop_eggs_dir,
)
if cache_key not in cache_storage:
working_set = zc.buildout.easy_install.working_set(
egg_list,
[develop_eggs_dir, eggs_dir]
)
cache_storage[cache_key] = working_set
else:
working_set = cache_storage[cache_key]
content = self.options['content'].strip()
output = self.options['output']
mode = self.options.get('mode', '0600')
path_list_string = ""
for dist in working_set:
path_list_string += ' "%s",\n' % dist.location
content_string = '\n'.join([line.lstrip() for line in content.split('\n')])
config_string = ""
for key in self.options:
if key.startswith('config-'):
config_string += " '%s': '%s',\n" % (key[7:], self.options[key])
option_dict = dict(path=path_list_string.strip(),
content=content_string,
config=config_string.strip())
with open(output, 'w') as f:
f.write(script_template % option_dict)
os.chmod(output, int(mode, 8))
return (output,)
update = install
...@@ -4,8 +4,6 @@ import unittest ...@@ -4,8 +4,6 @@ import unittest
from mock import patch from mock import patch
from slapos.recipe import free_port
class SocketMock(): class SocketMock():
def __init__(self, *args, **kw): def __init__(self, *args, **kw):
self.args = args self.args = args
...@@ -28,31 +26,16 @@ class FreePortTest(unittest.TestCase): ...@@ -28,31 +26,16 @@ class FreePortTest(unittest.TestCase):
SocketMock.bind = SocketMock.close = SocketMock.nothing_happen SocketMock.bind = SocketMock.close = SocketMock.nothing_happen
def new_recipe(self, **kw): def new_recipe(self, **kw):
buildout = { from slapos.recipe import free_port
'buildout': { from slapos.test.utils import makeRecipe
'bin-directory': '',
'find-links': '',
'allow-hosts': '',
'develop-eggs-directory': '',
'eggs-directory': '',
'python': 'testpython',
'installed': '.installed.cfg',
},
'testpython': {
'executable': sys.executable,
},
'slap-connection': {
'computer-id': '',
'partition-id': '',
'server-url': '',
'software-release-url': '',
}
}
options = { options = {
'ip': '127.0.0.1', 'ip': '127.0.0.1',
} }
options.update(kw) options.update(kw)
return free_port.Recipe(buildout=buildout, name='free_port', options=options) return makeRecipe(
free_port.Recipe,
options=options,
name='free_port')
@useMock @useMock
def test_ifNoBusyPortThenMinPortIsAlwaysReturned(self): def test_ifNoBusyPortThenMinPortIsAlwaysReturned(self):
......
...@@ -4,30 +4,15 @@ import unittest ...@@ -4,30 +4,15 @@ import unittest
from tempfile import mkdtemp from tempfile import mkdtemp
from shutil import rmtree from shutil import rmtree
from slapos.recipe import generic_cloudooo
class TestGenericCloudooo(unittest.TestCase): class TestGenericCloudooo(unittest.TestCase):
def new_recipe(self, options): def new_recipe(self, options):
buildout = { from slapos.recipe import generic_cloudooo
'buildout': { from slapos.test.utils import makeRecipe
'bin-directory': '', return makeRecipe(
'find-links': '', generic_cloudooo.Recipe,
'allow-hosts': '', options=options,
'develop-eggs-directory': '', name='generic_cloudooo')
'eggs-directory': '',
'python': 'testpython',
},
'testpython': {
'executable': sys.executable,
},
'slap-connection': {
'computer-id': '',
'partition-id': '',
'server-url': '',
'software-release-url': '',
}
}
return generic_cloudooo.Recipe(buildout=buildout, name='generic_cloudooo', options=options)
def setUp(self): def setUp(self):
self.test_dir = mkdtemp() self.test_dir = mkdtemp()
......
...@@ -5,37 +5,19 @@ import sys ...@@ -5,37 +5,19 @@ import sys
import tempfile import tempfile
import unittest import unittest
from slapos.recipe import pbs
class PBSTest(unittest.TestCase): class PBSTest(unittest.TestCase):
def new_recipe(self): def new_recipe(self):
buildout = { from slapos.recipe import pbs
'buildout': { from slapos.test.utils import makeRecipe
'bin-directory': '',
'find-links': '',
'allow-hosts': '',
'develop-eggs-directory': '',
'eggs-directory': '',
'python': 'testpython',
},
'testpython': {
'executable': sys.executable,
},
'slap-connection': {
'computer-id': '',
'partition-id': '',
'server-url': '',
'software-release-url': '',
}
}
options = { options = {
'rdiffbackup-binary': '' 'rdiffbackup-binary': ''
} }
return makeRecipe(
return pbs.Recipe(buildout=buildout, name='pbs', options=options) pbs.Recipe,
options=options,
name='pbs')
def test_push(self): def test_push(self):
recipe = self.new_recipe() recipe = self.new_recipe()
......
...@@ -6,8 +6,6 @@ import tempfile ...@@ -6,8 +6,6 @@ import tempfile
import unittest import unittest
from slapos.slap.slap import NotFoundError, ConnectionError from slapos.slap.slap import NotFoundError, ConnectionError
from slapos.recipe import re6stnet
class Re6stnetTest(unittest.TestCase): class Re6stnetTest(unittest.TestCase):
...@@ -47,31 +45,21 @@ class Re6stnetTest(unittest.TestCase): ...@@ -47,31 +45,21 @@ class Re6stnetTest(unittest.TestCase):
shutil.rmtree(path) shutil.rmtree(path)
def new_recipe(self): def new_recipe(self):
buildout = { from slapos.recipe import re6stnet
'buildout': { from slapos.test.utils import makeRecipe
'bin-directory': '', return makeRecipe(
'find-links': '', re6stnet.Recipe,
'allow-hosts': '', options=self.options,
'develop-eggs-directory': '', slap_connection={
'eggs-directory': '',
'python': 'testpython',
},
'testpython': {
'executable': sys.executable,
},
'slap-connection': {
'computer-id': 'comp-test', 'computer-id': 'comp-test',
'partition-id': 'slappart0', 'partition-id': 'slappart0',
'server-url': 'http://server.com', 'server-url': 'http://server.com',
'software-release-url': 'http://software.com', 'software-release-url': 'http://software.com',
'key-file': '/path/to/key', 'key-file': '/path/to/key',
'cert-file': '/path/to/cert' 'cert-file': '/path/to/cert'
} },
} name='re6stnet')
options = self.options
return re6stnet.Recipe(buildout=buildout, name='re6stnet', options=options)
def checkWrapper(self, path): def checkWrapper(self, path):
self.assertTrue(os.path.exists(path)) self.assertTrue(os.path.exists(path))
......
"""Test helpers
"""
import sys
import os.path
from ConfigParser import ConfigParser
import logging
def makeRecipe(recipe_class, options, name='test', slap_connection=None):
"""Instanciate a recipe of `recipe_class` with `options` with a buildout
mapping containing a python and an empty `slapos-connection` mapping, unless
provided as `slap_connection`.
If running tests in a buildout folder, the test recipe will reuse the
`eggs-directory` and `develop-eggs-directory` from this buildout so that the
test recipe does not need to install eggs again when using working set.
To prevent test accidentally writing to the buildout's eggs repositories, we
set `newest` to false and `offline` to true in this case.
"""
buildout = {
'buildout': {
'bin-directory': '',
'find-links': '',
'allow-hosts': '',
'develop-eggs-directory': '',
'eggs-directory': '',
'python': 'testpython',
},
'testpython': {
'executable': sys.executable,
},
'slap-connection': {
'computer-id': '',
'partition-id': '',
'server-url': '',
'software-release-url': '',
}
}
if slap_connection is not None:
buildout['slap-connection'] = slap_connection
# are we in buildout folder ?
# the usual layout is
# ${buildout:directory}/parts/slapos-repository/slapos/test/utils.py , so try
# to find a buildout relative to this file.
buildout_cfg = os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', 'buildout.cfg')
if os.path.exists(buildout_cfg):
parser = ConfigParser()
parser.readfp(open(buildout_cfg))
eggs_directory = parser.get('buildout', 'eggs-directory')
develop_eggs_directory = parser.get('buildout', 'develop-eggs-directory')
logging.getLogger(__name__).info(
'Using eggs-directory (%s) and develop-eggs-directory (%s) from buildout at %s',
eggs_directory,
develop_eggs_directory,
buildout_cfg)
buildout['buildout']['eggs-directory'] = eggs_directory
buildout['buildout']['develop-eggs-directory'] = develop_eggs_directory
buildout['buildout']['newest'] = False
buildout['buildout']['offline'] = True
return recipe_class(buildout=buildout, name=name, options=options)
...@@ -51,7 +51,7 @@ pycrypto = 2.6.1 ...@@ -51,7 +51,7 @@ pycrypto = 2.6.1
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.download = 1.0 slapos.recipe.download = 1.0
slapos.recipe.template = 4.3 slapos.recipe.template = 4.3
slapos.toolbox = 0.74 slapos.toolbox = 0.76
smmap = 0.9.0 smmap = 0.9.0
# Required by: # Required by:
......
...@@ -55,7 +55,7 @@ md5sum = d98a01182f38868612948c87d5231428 ...@@ -55,7 +55,7 @@ md5sum = d98a01182f38868612948c87d5231428
[template-default-slave-virtualhost] [template-default-slave-virtualhost]
filename = templates/default-virtualhost.conf.in filename = templates/default-virtualhost.conf.in
md5sum = 5344bff68a3f7dead633a35771ad1a19 md5sum = 7f38084af107034bedefba971abe165c
[template-cached-slave-virtualhost] [template-cached-slave-virtualhost]
filename = templates/cached-virtualhost.conf.in filename = templates/cached-virtualhost.conf.in
......
...@@ -11,7 +11,7 @@ plone.recipe.command = 1.1 ...@@ -11,7 +11,7 @@ plone.recipe.command = 1.1
pycrypto = 2.6.1 pycrypto = 2.6.1
rdiff-backup = 1.0.5+SlapOSPatched001 rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.3 slapos.recipe.template = 4.3
slapos.toolbox = 0.74 slapos.toolbox = 0.76
smmap = 0.9.0 smmap = 0.9.0
numpy = 1.11.2 numpy = 1.11.2
pyasn1 = 0.2.3 pyasn1 = 0.2.3
......
...@@ -69,7 +69,7 @@ def main(): ...@@ -69,7 +69,7 @@ def main():
test_title = args.test_suite_title or args.test_suite test_title = args.test_suite_title or args.test_suite
if args.master_url: if args.master_url:
tool = taskdistribution.TaskDistributionTool(args.master_url) tool = taskdistribution.TaskDistributor(args.master_url)
test_result = tool.createTestResult(args.revision, test_result = tool.createTestResult(args.revision,
dist_list, dist_list,
args.test_node_title, args.test_node_title,
......
...@@ -66,6 +66,7 @@ sparse-checkout = /playbook/roles/rina ...@@ -66,6 +66,7 @@ sparse-checkout = /playbook/roles/rina
[vm-debian] [vm-debian]
# building a generic Debian kernel uses a lot of space # building a generic Debian kernel uses a lot of space
size = 16Gi size = 16Gi
dists = debian-jessie
packages += packages +=
# generic (another SR that build packages automatically would use the same list) # generic (another SR that build packages automatically would use the same list)
apt-utils build-essential devscripts equivs lsb-release apt-utils build-essential devscripts equivs lsb-release
......
...@@ -10,7 +10,7 @@ extends = ...@@ -10,7 +10,7 @@ extends =
../../component/netcat/buildout.cfg ../../component/netcat/buildout.cfg
../../component/pycurl/buildout.cfg ../../component/pycurl/buildout.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
../../stack/nodejs.cfg ../../component/nodejs/buildout.cfg
../../stack/resilient/buildout.cfg ../../stack/resilient/buildout.cfg
# stacks are listed from most generic to most specific, # stacks are listed from most generic to most specific,
...@@ -99,7 +99,7 @@ recipe = hexagonit.recipe.download ...@@ -99,7 +99,7 @@ recipe = hexagonit.recipe.download
ignore-existing = true ignore-existing = true
url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2 url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
mode = 644 mode = 644
md5sum = 68b66fb3e9020642e57f4a4ee266f2b3 md5sum = c348188a6dcb15430bea0bad51c249fc
download-only = true download-only = true
on-update = true on-update = true
...@@ -186,7 +186,7 @@ ignore-existing = true ...@@ -186,7 +186,7 @@ ignore-existing = true
url = ${:_profile_base_location_}/template/template-kvm-run.in url = ${:_profile_base_location_}/template/template-kvm-run.in
mode = 644 mode = 644
filename = template-kvm-run.in filename = template-kvm-run.in
md5sum = bd238397af6236b6b24b693012feeece md5sum = c6f1536a3502102dadbfb9d82496cc36
download-only = true download-only = true
on-update = true on-update = true
......
...@@ -381,8 +381,8 @@ interface-url = {{ slapparameter_dict.get('monitor-interface-url', 'https://moni ...@@ -381,8 +381,8 @@ interface-url = {{ slapparameter_dict.get('monitor-interface-url', 'https://moni
<= monitor-publish <= monitor-publish
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish
ipv6 = ${slap-network-information:global-ipv6} ipv6 = ${slap-network-information:global-ipv6}
backend-url = https://[${novnc-instance:ip}]:${novnc-instance:port}/vnc_auto.html?host=[${novnc-instance:ip}]&port=${novnc-instance:port}&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd} backend-url = https://[${novnc-instance:ip}]:${novnc-instance:port}/vnc.html?host=[${novnc-instance:ip}]&port=${novnc-instance:port}&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}
url = ${request-slave-frontend:connection-url}/vnc_auto.html?host=${request-slave-frontend:connection-domainname}&port=${request-slave-frontend:connection-port}&encrypt=1&path=${request-slave-frontend:connection-resource}&password=${kvm-controller-parameter-dict:vnc-passwd} url = ${request-slave-frontend:connection-url}/vnc.html?host=${request-slave-frontend:connection-domainname}&port=${request-slave-frontend:connection-port}&encrypt=1&path=${request-slave-frontend:connection-resource}&password=${kvm-controller-parameter-dict:vnc-passwd}
{% set disk_number = len(storage_dict) -%} {% set disk_number = len(storage_dict) -%}
maximum-extra-disk-amount = {{ disk_number }} maximum-extra-disk-amount = {{ disk_number }}
{% set iface = 'eth0' -%} {% set iface = 'eth0' -%}
......
...@@ -5,7 +5,7 @@ extends = common.cfg ...@@ -5,7 +5,7 @@ extends = common.cfg
# XXX - use websockify = 0.5.1 for compatibility with kvm frontend # XXX - use websockify = 0.5.1 for compatibility with kvm frontend
websockify = 0.5.1 websockify = 0.5.1
slapos.toolbox = 0.74 slapos.toolbox = 0.76
erp5.util = 0.4.49 erp5.util = 0.4.49
apache-libcloud = 1.1.0 apache-libcloud = 1.1.0
collective.recipe.environment = 0.2.0 collective.recipe.environment = 0.2.0
......
...@@ -264,7 +264,7 @@ if enable_device_hotplug != 'true': ...@@ -264,7 +264,7 @@ if enable_device_hotplug != 'true':
ram = '%sM,slots=128,maxmem=%sM' % (ram_size, ram_max_size) ram = '%sM,slots=128,maxmem=%sM' % (ram_size, ram_max_size)
else: else:
smp = '1,maxcpus=%s' % smp_max_count smp = '1,maxcpus=%s' % smp_max_count
ram = '%sM,slots=128,maxmem=%s' % (init_ram_size, ram_max_size) ram = '%sM,slots=128,maxmem=%sM' % (init_ram_size, ram_max_size)
kvm_argument_list = [qemu_path, kvm_argument_list = [qemu_path,
'-enable-kvm', '-smp', smp, '-name', vm_name, '-m', ram, '-vga', 'std', '-enable-kvm', '-smp', smp, '-name', vm_name, '-m', ram, '-vga', 'std',
'-drive', 'file=%s,if=%s,cache=%s,aio=%s' % (disk_path, disk_type, disk_cache, disk_aio), '-drive', 'file=%s,if=%s,cache=%s,aio=%s' % (disk_path, disk_type, disk_cache, disk_aio),
......
# DESTINATION
<match td.*.*>
type tdlog
apikey YOUR_API_KEY
auto_create_table
buffer_type file
buffer_path {{ fluentd_log_directory }}/td-agent/buffer/td
<secondary>
type file
path {{ fluentd_log_directory }}/failed_records
</secondary>
</match>
{% set wendelin_streamtool_uri = slapparameter_dict.get('wendelin-streamtool-uri', '') -%}
{% set wendelin_password = slapparameter_dict.get('wendelin-password', '') -%}
{% set wendelin_user = slapparameter_dict.get('wendelin-user', '') -%}
{% set wendelin_uid = slapparameter_dict.get('wendelin-uid', 'UNKNOWN') -%}
{% if wendelin_streamtool_uri and wendelin_password and wendelin_user -%}
<source>
type syslog
port 42185
bind 127.0.0.1
tag {{ wendelin_uid }}.system
</source>
<match {{ wendelin_uid }}.system.**>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}_syslog
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
<source>
type tail
path {{ fluentd_log_directory }}/consumption/*/dump_user.csv
pos_file {{ fluentd_log_directory }}/consumption/dump_user.pos
tag slapos.monitor.usage.user.{{ wendelin_uid }}
format /^(?<partition>[^,]*),(?<pid>[^,]*),(?<process>[^,]*),(?<cpu_percent>[^,]*),(?<cpu_time>[^,]*),(?<cpu_num_threads>[^,]*),(?<memory_percent>[^,]*),(?<memory_rss>[^,]*),(?<io_rw_counter>[^,]*),(?<io_cycles_counter>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
read_from_head true
</source>
<match slapos.monitor.usage.user.{{ wendelin_uid }}>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}_user
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
<source>
type tail
path {{ fluentd_log_directory }}/consumption/*/dump_system.csv
pos_file {{ fluentd_log_directory }}/consumption/dump_system.pos
tag slapos.monitor.usage.system.{{ wendelin_uid }}
format /^(?<loadavg>[^,]*),(?<cpu_percent>[^,]*),(?<memory_used>[^,]*),(?<memory_free>[^,]*),(?<net_in_bytes>[^,]*),(?<net_in_errors>[^,]*),(?<net_in_dropped>[^,]*),(?<net_out_bytes>[^,]*),(?<net_out_errors>[^,]*),(?<net_out_dropped>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
read_from_head true
</source>
<match slapos.monitor.usage.system.{{ wendelin_uid }}>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}_system
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
<source>
type tail
path {{ fluentd_log_directory }}/consumption/*/dump_computer.csv
pos_file {{ fluentd_log_directory }}/consumption/dump_computer.pos
tag slapos.monitor.usage.computer.{{ wendelin_uid }}
format /^(?<cpu_num_core>[^,]*),(?<cpu_frequency>[^,]*),(?<cpu_type>[^,]*),(?<memory_size>[^,]*),(?<memory_type>[^,]*),(?<partition_list>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
read_from_head true
</source>
<match slapos.monitor.usage.computer.{{ wendelin_uid }}>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}_computer
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
<source>
type tail
path {{ fluentd_log_directory }}/consumption/*/dump_disk.csv
pos_file {{ fluentd_log_directory }}/consumption/dump_disk.pos
tag slapos.monitor.usage.disk.{{ wendelin_uid }}
format /^(?<partition>[^,]*),(?<used>[^,]*),(?<free>[^,]*),(?<mountpoint>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
read_from_head true
</source>
<match slapos.monitor.usage.disk.{{ wendelin_uid }}>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}_disk
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
<source>
type tail
path {{ fluentd_log_directory }}/consumption/*/dump_temperature.csv
pos_file {{ fluentd_log_directory }}/consumption/dump_temperature.pos
tag slapos.monitor.usage.temperature.{{ wendelin_uid }}
format /^(?<sensor_id>[^,]*),(?<temperature>[^,]*),(?<alarm>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
read_from_head true
</source>
<match slapos.monitor.usage.temperature.{{ wendelin_uid }}>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}_temperature
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
{% endif -%}
# DESTINATION
<match td.*.*>
type tdlog
apikey YOUR_API_KEY
auto_create_table
buffer_type file
buffer_path {{ fluentd_log_directory }}/td-agent/buffer/td
<secondary>
type file
path {{ fluentd_log_directory }}/failed_records
</secondary>
</match>
## match tag=debug.** and dump to console
<match debug.**>
type stdout
</match>
<source>
type tail
path {{ crawl_log_directory }}/*/*ping.log.20*
pos_file {{ crawl_log_directory }}/tail_in_ping.pos
tag slapos.monitor.networktest.ping.ipv4
format /^(?<time>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/
read_from_head true
</source>
<source>
type tail
path {{ crawl_log_directory }}/*/*ping6.log.20*
pos_file {{ crawl_log_directory }}/tail_in_ping6.pos
tag slapos.monitor.networktest.ping.ipv6
format /^(?<time>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/
read_from_head true
</source>
<match slapos.monitor.networktest.ping.*>
type copy
{% for slave_instance in slave_instance_list -%}
{% if slave_instance.get("ping_ip_list") -%}
<store>
type grep
regexp1 name_or_ip ^{{ slave_instance.get("ping_ip_list") }}$
add_tag_prefix {{ slave_instance.get("slave_reference") }}
</store>
{% endif -%}
{% if slave_instance.get("ping6_ip_list") -%}
<store>
type grep
regexp1 name_or_ip ^{{ slave_instance.get("ping6_ip_list") }}$
add_tag_prefix {{ slave_instance.get("slave_reference") }}
</store>
{% endif -%}
{% endfor -%}
</match>
{% for slave_instance in slave_instance_list -%}
<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv6>
type file
path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping6/log
append true
include_time_key true
include_tag_key true
format json
</match>
<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv4>
type file
path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping/log
append true
include_time_key true
include_tag_key true
format json
</match>
{% endfor -%}
<source>
type tail
path {{ network_user_logs }}/*/ping*/log/*.log
pos_file {{ crawl_log_directory }}/tail_in_ping6.pos
tag slapos.wendelin.networktest
format json
read_from_head true
</source>
{% set wendelin_streamtool_uri = slapparameter_dict.get('wendelin-streamtool-uri', '') -%}
{% set wendelin_password = slapparameter_dict.get('wendelin-password', '') -%}
{% set wendelin_user = slapparameter_dict.get('wendelin-user', '') -%}
{% if wendelin_streamtool_uri and wendelin_password and wendelin_user -%}
<match slapos.wendelin.networktest>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
{% endif -%}
## SOURCE
<source>
type forward
</source>
## live debugging agent
<source>
type debug_agent
bind 127.0.0.1
port 24230
</source>
...@@ -72,26 +72,7 @@ wrapper-path = ${monitor-directory:bin}/log-crawler ...@@ -72,26 +72,7 @@ wrapper-path = ${monitor-directory:bin}/log-crawler
extends = {{ instance_base_monitor }} extends = {{ instance_base_monitor }}
parts += parts +=
slave-test-configuration slave-test-configuration
fluentd-distributor-wrapper
{% for part in part_list %} {% for part in part_list %}
{{ ' %s' % part }} {{ ' %s' % part }}
{% endfor %} {% endfor %}
[fluentd-distributor-wrapper]
<=fluentd-wrapper
command-line = {{ fluentd_location }}/bin/fluentd -l ${monitor-directory:log}/fluend-distributor.log -c ${fluentd-distributor-conf-configuration:rendered}
wrapper-path = ${monitor-directory:service}/fluentd-distributor
[fluentd-distributor-conf-configuration]
recipe = slapos.recipe.template:jinja2
template = {{ fluent_distributor_conf_output }}
rendered = ${monitor-directory:etc}/fluentd-distributor.cfg
mode = 0744
context =
key slapparameter_dict slap-configuration:configuration
key slave_instance_list slap-configuration:slave-instance-list
key fluentd_log_directory monitor-directory:fluentd-log
key crawl_log_directory monitor-directory:crawl-log
key network_user_logs monitor-directory:network-user-logs
key computer_id slap-configuration:computer
...@@ -8,7 +8,6 @@ parts = ...@@ -8,7 +8,6 @@ parts =
symlink-re6st-logs symlink-re6st-logs
symlink-collected-logs symlink-collected-logs
monitor-collect-csv-wrapper monitor-collect-csv-wrapper
fluentd-wrapper
monitor-base monitor-base
monitor-check-memory-usage monitor-check-memory-usage
monitor-check-cpu-usage monitor-check-cpu-usage
...@@ -24,7 +23,10 @@ offline = true ...@@ -24,7 +23,10 @@ offline = true
recipe = slapos.cookbook:cron.d recipe = slapos.cookbook:cron.d
name = network-bench-test name = network-bench-test
frequency = */10 * * * * frequency = */10 * * * *
command = {{ buildout }}/networkbench ${network-bench-configuration:rendered} ${monitor-directory:monitor-log} # skip to not fill cron log file.
# command = {{ buildout_bin }}/networkbench ${network-bench-configuration:rendered} ${monitor-directory:monitor-log}
command = true
[symlink-re6st-logs] [symlink-re6st-logs]
recipe = cns.recipe.symlink recipe = cns.recipe.symlink
...@@ -63,8 +65,7 @@ monitor-log = ${:private}/monitor-log ...@@ -63,8 +65,7 @@ monitor-log = ${:private}/monitor-log
cache = ${:var}/cache cache = ${:var}/cache
mod-ssl = ${:cache}/httpd_mod_ssl mod-ssl = ${:cache}/httpd_mod_ssl
system-log = ${:private}/system-log system-log = ${:private}/system-log
fluentd-log = ${:log}/fluentd consumption = ${:log}/consumption
consumption = ${:fluentd-log}/consumption
[monitor-httpd-configuration-file] [monitor-httpd-configuration-file]
context = context =
...@@ -76,60 +77,37 @@ context = ...@@ -76,60 +77,37 @@ context =
section slave_information slap-configuration section slave_information slap-configuration
key monitor_private_hash slap-configuration:private-hash key monitor_private_hash slap-configuration:private-hash
[fluentd-wrapper]
recipe = slapos.cookbook:wrapper
command-line = {{ fluentd_location }}/bin/fluentd -qq -l ${monitor-directory:log}/fluend.log -c ${fluentd-conf-configuration:rendered}
wrapper-path = ${monitor-directory:service}/fluentd
environment =
GEM_PATH={{ fluentd_location }}/lib/ruby/gems/1.8/
[fluentd-conf-configuration]
recipe = slapos.recipe.template:jinja2
template = {{ fluent_conf_output }}
rendered = ${monitor-directory:etc}/fluentd.cfg
mode = 0744
context =
key slapparameter_dict slap-configuration:configuration
key slave_instance_list slap-configuration:slave-instance-list
key fluentd_log_directory monitor-directory:fluentd-log
key computer_id slap-connection:computer-id
[monitor-collect-csv-wrapper] [monitor-collect-csv-wrapper]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
command-line = command-line =
${monitor-directory:bin}/python {{ monitor_collect_csv_dump }} --output_folder ${monitor-directory:fluentd-log}/consumption/ ${monitor-directory:bin}/python {{ monitor_collect_csv_dump }} --output_folder ${monitor-directory:consumption}
wrapper-path = ${monitor-directory:reports}/monitor-collect-csv-dump wrapper-path = ${monitor-directory:reports}/monitor-collect-csv-dump
[monitor-check-cpu-usage] [monitor-check-cpu-usage]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
command-line = ${monitor-directory:bin}/python {{ monitor_check_system_health }} cpu ${init-monitor-parameters:cpu-load-file} command-line = ${monitor-directory:bin}/python {{ monitor_check_system_health }} cpu ${init-monitor-parameters:cpu-load-file}
wrapper-path = ${monitor-directory:promises}/system-CPU-load-check wrapper-path = ${directory:promises}/system-CPU-load-check
[monitor-check-memory-usage] [monitor-check-memory-usage]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
command-line = ${monitor-directory:bin}/python {{ monitor_check_system_health }} mem ${init-monitor-parameters:mem-free-file} ${directory:monitor} command-line = {{ buildout_bin}}/check-computer-memory
wrapper-path = ${monitor-directory:promises}/system-MEMORY-usage-check -db ${monitor-instance-parameter:collector-db}
--threshold ${slap-parameter:memory-percent-threshold}
--unit percent
wrapper-path = ${directory:promises}/check-computer-memory-usage
[publish-connection-information] [publish-connection-information]
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish
monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password} monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
server_log_url = ${monitor-publish-parameters:monitor-base-url}/${slap-configuration:private-hash}/ server_log_url = ${monitor-publish-parameters:monitor-base-url}/${slap-configuration:private-hash}/
[monitor-instance-parameter]
instance-configuration =
file max-cpu-load-per-core ${init-monitor-parameters:cpu-load-file}
file min-free-mem-percent ${init-monitor-parameters:mem-free-file}
[init-monitor-parameters] [init-monitor-parameters]
recipe = plone.recipe.command recipe = plone.recipe.command
cpu-load-file = ${directory:monitor}/cpu-load-tolerance cpu-load-file = ${directory:monitor}/cpu-load-tolerance
mem-free-file = ${directory:monitor}/mem-free-limit mem-free-file = ${directory:monitor}/mem-free-limit
command = command =
if [ ! -s "${:cpu-load-file}" ]; then if [ ! -s "${:cpu-load-file}" ]; then
echo "1.5" > ${:cpu-load-file} echo ${slap-parameter:cpu-load-threshold} > ${:cpu-load-file}
fi
if [ ! -s "${:mem-free-file}" ]; then
echo "7.0" > ${:mem-free-file}
fi fi
[slap-configuration] [slap-configuration]
...@@ -141,3 +119,7 @@ key = ${slap-connection:key-file} ...@@ -141,3 +119,7 @@ key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file} cert = ${slap-connection:cert-file}
private-hash = ${pwgen:passwd}${pwgen32:passwd} private-hash = ${pwgen:passwd}${pwgen32:passwd}
frontend-domain = frontend-domain =
[slap-parameter]
cpu-load-threshold = 2.0
memory-percent-threshold = 96
...@@ -15,14 +15,12 @@ recipe = slapos.recipe.template:jinja2 ...@@ -15,14 +15,12 @@ recipe = slapos.recipe.template:jinja2
template = ${template-monitor:destination} template = ${template-monitor:destination}
rendered = $${buildout:directory}/template-base-monitor.cfg rendered = $${buildout:directory}/template-base-monitor.cfg
extensions = jinja2.ext.do extensions = jinja2.ext.do
context = key buildout buildout:bin-directory context = key develop_eggs_directory buildout:develop-eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration key slapparameter_dict slap-configuration:configuration
raw buildout_bin ${buildout:bin-directory}
raw monitor_template_output ${monitor-template:output} raw monitor_template_output ${monitor-template:output}
raw network_benck_cfg_output ${network-bench-cfg:output} raw network_benck_cfg_output ${network-bench-cfg:output}
raw fluentd_location ${fluentd:location}
raw fluent_conf_output ${fluentd-agent-conf:output}
raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output} raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
raw monitor_check_system_health ${monitor-system-health:output} raw monitor_check_system_health ${monitor-system-health:output}
mode = 0644 mode = 0644
...@@ -33,15 +31,13 @@ template = ${template-monitor-distributor:destination} ...@@ -33,15 +31,13 @@ template = ${template-monitor-distributor:destination}
rendered = $${buildout:directory}/template-monitor-base-distributor.cfg rendered = $${buildout:directory}/template-monitor-base-distributor.cfg
extensions = jinja2.ext.do extensions = jinja2.ext.do
context = import json_module json context = import json_module json
key buildout buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration key slapparameter_dict slap-configuration:configuration
key instance_base_monitor instance-base-monitor:rendered key instance_base_monitor instance-base-monitor:rendered
key slave_instance_list slap-configuration:slave-instance-list key slave_instance_list slap-configuration:slave-instance-list
raw buildout_bin ${buildout:bin-directory}
raw template_json_distributor_test ${json-test-template:destination} raw template_json_distributor_test ${json-test-template:destination}
raw fluentd_location ${fluentd:location}
raw fluent_distributor_conf_output ${fluentd-agent-distributor-conf:output}
raw wget_bin ${wget:location}/bin/wget raw wget_bin ${wget:location}/bin/wget
mode = 0644 mode = 0644
......
...@@ -5,7 +5,6 @@ extends = ...@@ -5,7 +5,6 @@ extends =
../../component/python-cryptography/buildout.cfg ../../component/python-cryptography/buildout.cfg
../../component/wget/buildout.cfg ../../component/wget/buildout.cfg
../../stack/monitor/buildout.cfg ../../stack/monitor/buildout.cfg
../../component/fluentd/buildout.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
parts = parts =
...@@ -16,30 +15,27 @@ parts = ...@@ -16,30 +15,27 @@ parts =
template template
template-monitor-distributor template-monitor-distributor
template-monitor template-monitor
fluentd
fluentd-agent-conf
fluentd-agent-distributor-conf
monitor-collect-csv-dump monitor-collect-csv-dump
[template] [template]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg output = ${buildout:directory}/template.cfg
md5sum = 3ff5fb2710bf0ea84632c6d6d3894dd9 md5sum = 641c5916739f78171c616af00fe974a2
mode = 0644 mode = 0644
[template-monitor] [template-monitor]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-monitor.cfg.jinja2 url = ${:_profile_base_location_}/instance-monitor.cfg.jinja2
destination = ${buildout:directory}/template-base-monitor.cfg destination = ${buildout:directory}/template-base-monitor.cfg
md5sum = 5def53c8faa0dfca313e53ebcc3229fa md5sum = db36ed784c690be892bf1834444c1fe7
mode = 0644 mode = 0644
[template-monitor-distributor] [template-monitor-distributor]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-monitor-distributor.cfg.jinja2 url = ${:_profile_base_location_}/instance-monitor-distributor.cfg.jinja2
destination = ${buildout:directory}/template-monitor-base-distributor.cfg destination = ${buildout:directory}/template-monitor-base-distributor.cfg
md5sum = 0ab9a86df18125335ed365bb673bbcf3 md5sum = 61c0bfdfc0a2b51ba15fe4a49baf6091
mode = 0644 mode = 0644
[json-test-template] [json-test-template]
...@@ -56,20 +52,6 @@ md5sum = cfcbf2002b8eff5153e2bf68ed24b720 ...@@ -56,20 +52,6 @@ md5sum = cfcbf2002b8eff5153e2bf68ed24b720
output = ${buildout:directory}/template-network-bench-cfg.in output = ${buildout:directory}/template-network-bench-cfg.in
mode = 0644 mode = 0644
[fluentd-agent-conf]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/fluentd-agent.conf.jinja2.in
md5sum = 3ea59906937eab7aeef78f46c4994ecd
output = ${buildout:directory}/fluentd-agent.conf.jinja2.in
mode = 0644
[fluentd-agent-distributor-conf]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/fluentd-distributor-agent.conf.jinja2.in
md5sum = 4b81ddcbe2f16d23013caac37151d396
output = ${buildout:directory}/fluentd-agent-distributor.conf.jinja2.in
mode = 0644
[monitor-collect-csv-dump] [monitor-collect-csv-dump]
<= monitor-template-script <= monitor-template-script
url = ${:_profile_base_location_}/script/${:filename} url = ${:_profile_base_location_}/script/${:filename}
...@@ -103,12 +85,8 @@ scripts = ...@@ -103,12 +85,8 @@ scripts =
monitor.genstatus monitor.genstatus
monitor.configwrite monitor.configwrite
is-process-older-than-dependency-set is-process-older-than-dependency-set
check-free-disk
[fluentd] check-computer-memory
gems +=
fluent-plugin-wendelin==0.1.alpha1
fluent-plugin-grep==0.3.4
fluent-plugin-bin==0.1
[monitor-eggs] [monitor-eggs]
eggs += eggs +=
......
...@@ -48,6 +48,11 @@ eggs = neoppod[admin, ctl, master, storage-mysqldb] ...@@ -48,6 +48,11 @@ eggs = neoppod[admin, ctl, master, storage-mysqldb]
ZODB ZODB
zope.testing zope.testing
zodbtools zodbtools
patch-binary = ${patch:location}/bin/patch
ZEO-patch-options = -p1
ZEO-patches =
${:_profile_base_location_}/../../component/egg-patch/ZEO4/TCP_NODELAY.patch#b07288522d5c6857738240d948321df6
[slapos-deps-eggs] [slapos-deps-eggs]
recipe = zc.recipe.egg recipe = zc.recipe.egg
...@@ -117,7 +122,7 @@ persistent = 4.2.3 ...@@ -117,7 +122,7 @@ persistent = 4.2.3
pycrypto = 2.6.1 pycrypto = 2.6.1
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 4.3 slapos.recipe.template = 4.3
slapos.toolbox = 0.74 slapos.toolbox = 0.76
smmap2 = 2.0.1 smmap2 = 2.0.1
transaction = 1.7.0 transaction = 1.7.0
zodbpickle = 0.6.0 zodbpickle = 0.6.0
...@@ -161,7 +166,7 @@ zodburi = 2.3.0 ...@@ -161,7 +166,7 @@ zodburi = 2.3.0
# Required by: # Required by:
# zodburi==2.0 # zodburi==2.0
# ZEO 5 requires transaction >= 2 # ZEO 5 requires transaction >= 2
ZEO = 4.3.1 ZEO = 4.3.1+SlapOSPatched001
# Required by: # Required by:
# zodburi==2.0 # zodburi==2.0
......
[buildout] [buildout]
extends = software.cfg extends = software.cfg
[neoppod]
ZEO-patches =
[versions] [versions]
ZODB = 5.3.0 ZODB = 5.3.0
ZEO = 5.1.0 ZEO = 5.1.0
......
...@@ -3,21 +3,35 @@ ...@@ -3,21 +3,35 @@
# list of go git repositories to fetch # list of go git repositories to fetch
[gowork.goinstall] [gowork.goinstall]
depends_gitfetch = depends_gitfetch =
${go_github.com_DataDog_czlib:recipe}
${go_github.com_cznic_strutil:recipe} ${go_github.com_cznic_strutil:recipe}
${go_github.com_golang_glog:recipe} ${go_github.com_golang_glog:recipe}
${go_github.com_gwenn_gosqlite:recipe}
${go_github.com_gwenn_yacr:recipe}
${go_github.com_kisielk_og-rek:recipe} ${go_github.com_kisielk_og-rek:recipe}
${go_github.com_kylelemons_godebug:recipe} ${go_github.com_kylelemons_godebug:recipe}
${go_github.com_pkg_errors:recipe} ${go_github.com_pkg_errors:recipe}
${go_github.com_pkg_profile:recipe} ${go_github.com_pkg_profile:recipe}
${go_github.com_soheilhy_cmux:recipe}
${go_github.com_someonegg_gocontainer:recipe} ${go_github.com_someonegg_gocontainer:recipe}
${go_github.com_someonegg_gox:recipe}
${go_golang.org_x_crypto:recipe}
${go_golang.org_x_net:recipe} ${go_golang.org_x_net:recipe}
${go_golang.org_x_perf:recipe} ${go_golang.org_x_perf:recipe}
${go_golang.org_x_sync:recipe} ${go_golang.org_x_sync:recipe}
${go_golang.org_x_sys:recipe}
${go_golang.org_x_text:recipe}
${go_golang.org_x_tools:recipe} ${go_golang.org_x_tools:recipe}
${go_lab.nexedi.com_kirr_go123:recipe} ${go_lab.nexedi.com_kirr_go123:recipe}
${go_lab.nexedi.com_kirr_neo:recipe} ${go_lab.nexedi.com_kirr_neo:recipe}
[go_github.com_DataDog_czlib]
<= go-git-package
go.importpath = github.com/DataDog/czlib
repository = https://github.com/DataDog/czlib
revision = 4bc9a24e37
[go_github.com_cznic_strutil] [go_github.com_cznic_strutil]
<= go-git-package <= go-git-package
go.importpath = github.com/cznic/strutil go.importpath = github.com/cznic/strutil
...@@ -30,6 +44,18 @@ go.importpath = github.com/golang/glog ...@@ -30,6 +44,18 @@ go.importpath = github.com/golang/glog
repository = https://github.com/golang/glog repository = https://github.com/golang/glog
revision = 23def4e6c1 revision = 23def4e6c1
[go_github.com_gwenn_gosqlite]
<= go-git-package
go.importpath = github.com/gwenn/gosqlite
repository = https://github.com/gwenn/gosqlite
revision = 29cd841087
[go_github.com_gwenn_yacr]
<= go-git-package
go.importpath = github.com/gwenn/yacr
repository = https://github.com/gwenn/yacr
revision = 77093bdc7e
[go_github.com_kisielk_og-rek] [go_github.com_kisielk_og-rek]
<= go-git-package <= go-git-package
go.importpath = github.com/kisielk/og-rek go.importpath = github.com/kisielk/og-rek
...@@ -46,7 +72,7 @@ revision = d65d576e93 ...@@ -46,7 +72,7 @@ revision = d65d576e93
<= go-git-package <= go-git-package
go.importpath = github.com/pkg/errors go.importpath = github.com/pkg/errors
repository = https://github.com/pkg/errors repository = https://github.com/pkg/errors
revision = v0.8.0-6-g2b3a18b5f0 revision = v0.8.0-12-g816c908556
[go_github.com_pkg_profile] [go_github.com_pkg_profile]
<= go-git-package <= go-git-package
...@@ -54,44 +80,74 @@ go.importpath = github.com/pkg/profile ...@@ -54,44 +80,74 @@ go.importpath = github.com/pkg/profile
repository = https://github.com/pkg/profile repository = https://github.com/pkg/profile
revision = v1.2.1-0-g5b67d42886 revision = v1.2.1-0-g5b67d42886
[go_github.com_soheilhy_cmux]
<= go-git-package
go.importpath = github.com/soheilhy/cmux
repository = https://github.com/soheilhy/cmux
revision = e09e9389d8
[go_github.com_someonegg_gocontainer] [go_github.com_someonegg_gocontainer]
<= go-git-package <= go-git-package
go.importpath = github.com/someonegg/gocontainer go.importpath = github.com/someonegg/gocontainer
repository = https://github.com/someonegg/gocontainer repository = https://github.com/someonegg/gocontainer
revision = fc2c7e84b5 revision = fc2c7e84b5
[go_github.com_someonegg_gox]
<= go-git-package
go.importpath = github.com/someonegg/gox
repository = https://github.com/someonegg/gox
revision = 4915b7fd7c
[go_golang.org_x_crypto]
<= go-git-package
go.importpath = golang.org/x/crypto
repository = https://go.googlesource.com/crypto
revision = 88942b9c40
[go_golang.org_x_net] [go_golang.org_x_net]
<= go-git-package <= go-git-package
go.importpath = golang.org/x/net go.importpath = golang.org/x/net
repository = https://go.googlesource.com/net repository = https://go.googlesource.com/net
revision = 1087133bc4 revision = 6078986fec
[go_golang.org_x_perf] [go_golang.org_x_perf]
<= go-git-package <= go-git-package
go.importpath = golang.org/x/perf go.importpath = golang.org/x/perf
repository = https://go.googlesource.com/perf repository = https://go.googlesource.com/perf
revision = 4469e6ce8c revision = 8c788eb673
[go_golang.org_x_sync] [go_golang.org_x_sync]
<= go-git-package <= go-git-package
go.importpath = golang.org/x/sync go.importpath = golang.org/x/sync
repository = https://go.googlesource.com/sync repository = https://go.googlesource.com/sync
revision = 8e0aa688b6 revision = 1d60e4601c
[go_golang.org_x_sys]
<= go-git-package
go.importpath = golang.org/x/sys
repository = https://go.googlesource.com/sys
revision = 91ee8cde43
[go_golang.org_x_text]
<= go-git-package
go.importpath = golang.org/x/text
repository = https://go.googlesource.com/text
revision = v0.3.0-42-gab48842968
[go_golang.org_x_tools] [go_golang.org_x_tools]
<= go-git-package <= go-git-package
go.importpath = golang.org/x/tools go.importpath = golang.org/x/tools
repository = https://go.googlesource.com/tools repository = https://go.googlesource.com/tools
revision = 9bd2f44268 revision = 77106db15f
[go_lab.nexedi.com_kirr_go123] [go_lab.nexedi.com_kirr_go123]
<= go-git-package <= go-git-package
go.importpath = lab.nexedi.com/kirr/go123 go.importpath = lab.nexedi.com/kirr/go123
repository = https://lab.nexedi.com/kirr/go123.git repository = https://lab.nexedi.com/kirr/go123.git
revision = 2578d58311 revision = 76f667ba6c
[go_lab.nexedi.com_kirr_neo] [go_lab.nexedi.com_kirr_neo]
<= go-git-package <= go-git-package
go.importpath = lab.nexedi.com/kirr/neo go.importpath = lab.nexedi.com/kirr/neo
repository = https://lab.nexedi.com/kirr/neo.git repository = https://lab.nexedi.com/kirr/neo.git
revision = v1.8-1326-g4d0cd89484 revision = v1.8.1-1634-g4000df14e4
...@@ -8,6 +8,8 @@ extends = ...@@ -8,6 +8,8 @@ extends =
../neoppod/software-common.cfg ../neoppod/software-common.cfg
../../component/golang/buildout.cfg ../../component/golang/buildout.cfg
gowork.cfg gowork.cfg
../../component/sqlite3/buildout.cfg
../../component/zlib/buildout.cfg
../../component/wendelin.core/buildout.cfg ../../component/wendelin.core/buildout.cfg
../../component/ethtool/buildout.cfg ../../component/ethtool/buildout.cfg
../../component/ioping/buildout.cfg ../../component/ioping/buildout.cfg
...@@ -48,6 +50,10 @@ install = ...@@ -48,6 +50,10 @@ install =
github.com/pkg/profile \ github.com/pkg/profile \
golang.org/x/perf/cmd/benchstat golang.org/x/perf/cmd/benchstat
cpkgpath =
${sqlite3:location}/lib/pkgconfig
${zlib:location}/lib/pkgconfig
# dev-install neo from go checkout # dev-install neo from go checkout
[neoppod-develop] [neoppod-develop]
setup = ${go_lab.nexedi.com_kirr_neo:location} setup = ${go_lab.nexedi.com_kirr_neo:location}
...@@ -108,13 +114,13 @@ eggs = ...@@ -108,13 +114,13 @@ eggs =
# wendelin.core: latest not yet released # wendelin.core: latest not yet released
[wendelin.core-repository] [wendelin.core-repository]
revision= v0.11-4-g38fbc83ceb revision= v0.11-15-gf785ac079b
# ping eggs versions # ping eggs versions
[versions] [versions]
pyasn1 = 0.3.7 pyasn1 = 0.3.7
ZODB3 = 3.11.0 ZODB3 = 3.11.0
numpy = 1.13.3 numpy = 1.14.2
zope.testing = 4.6.2 zope.testing = 4.6.2
erp5.util = 0.4.50 erp5.util = 0.4.50
...@@ -128,3 +134,7 @@ ZConfig = 3.2.0 ...@@ -128,3 +134,7 @@ ZConfig = 3.2.0
# ZEO==4.3.1 # ZEO==4.3.1
# ZODB==4.4.5 # ZODB==4.4.5
zc.lockfile = 1.2.1 zc.lockfile = 1.2.1
# Required by:
# neoppod==1.8.1
python-dateutil = 2.7.1
...@@ -46,7 +46,7 @@ mode = 0644 ...@@ -46,7 +46,7 @@ mode = 0644
[versions] [versions]
slapos.recipe.template = 4.3 slapos.recipe.template = 4.3
slapos.toolbox = 0.74 slapos.toolbox = 0.76
dnspython = 1.15.0 dnspython = 1.15.0
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
erp5.util = 0.4.50 erp5.util = 0.4.50
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"properties": {
}
}
...@@ -104,6 +104,7 @@ filename = registry-run.in ...@@ -104,6 +104,7 @@ filename = registry-run.in
md5sum = 0bf4f2c03e06b55c6c6cc55fa33e65d6 md5sum = 0bf4f2c03e06b55c6c6cc55fa33e65d6
[versions] [versions]
re6stnet = 0.494
apache-libcloud = 0.17.0 apache-libcloud = 0.17.0
ecdsa = 0.13 ecdsa = 0.13
gitdb = 0.6.4 gitdb = 0.6.4
...@@ -111,7 +112,7 @@ plone.recipe.command = 1.1 ...@@ -111,7 +112,7 @@ plone.recipe.command = 1.1
pycrypto = 2.6.1 pycrypto = 2.6.1
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 4.3 slapos.recipe.template = 4.3
slapos.toolbox = 0.74 slapos.toolbox = 0.76
smmap = 0.9.0 smmap = 0.9.0
# Required by: # Required by:
......
...@@ -10,11 +10,13 @@ ...@@ -10,11 +10,13 @@
"response": "instance-re6stnet-output-schema.json", "response": "instance-re6stnet-output-schema.json",
"index": 0 "index": 0
}, },
"registry": { "default-slave": {
"title": "registry", "title": "Re6st Token",
"description": "Re6st registry", "description": "Re6st registry",
"request": "instance-re6stnet-resilient-input-schema.json", "software-type": "default",
"request": "instance-re6stnet-slave-input-schema.json",
"response": "instance-re6stnet-output-schema.json", "response": "instance-re6stnet-output-schema.json",
"shared": true,
"index": 1 "index": 1
} }
} }
......
# Slapos egg tests
This software release is used to run unit test of slapos eggs.
The approach is to use setuptools' integrated test runner, `python setup.py test`, to run tests.
The `python` used in this command will be a `zc.recipe.egg` interpreter with
all eggs pre-installed by this software release.
Nexedi staff can see the results of this test from the test suite
`SLAPOS-EGG-TEST` in test result module.
Here's an example session of how a developer could use this software release in
slaprunner to develop a slapos egg, in the example `slapos.core`, to make
changes to the code, run tests and publish changes.
```bash
# install this software release
SR=https://lab.nexedi.com/nexedi/slapos/raw/master/software/slapos-testing/software.cfg
COMP=slaprunner
INSTANCE_NAME=$COMP
slapos supply $SR $COMP
slapos node software
slapos request --node=node=$COMP $INSTANCE_NAME $SR
slapos node instance
# The source code is a git clone working copy on the instance
cd ~/srv/runner/instance/slappart0/parts/slapos.core/
# make some changes to the code
vim slapos/tests/client.py
# run tests, using bundled python intepreter with pre-installed eggs dependencies
~/srv/runner/instance/slappart0/software_release/bin/python_for_test setup.py build
# when satified, commit changes
git add -p && git commit
# add developer's fork remote (this is only needed the first time)
git remote add my_remote https://lab.nexedi.com/your_username/slapos.core.git/
# push the changes
git push my_remote HEAD:feature_branch_name
# then submit merge request
```
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# But avoid directories, they are not portable.
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[template]
filename = instance.cfg
md5sum = 9dece9d12dc94bf5c35d307cc8aa4d6b
[buildout] [buildout]
parts = parts =
slapos.core-setup
erp5.util-setup
phantomjs-wrapper phantomjs-wrapper
slapos-test-runner slapos-test-runner
sh-environment
eggs-directory = ${buildout:eggs-directory} eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
...@@ -24,33 +21,26 @@ bin = $${buildout:directory}/bin ...@@ -24,33 +21,26 @@ bin = $${buildout:directory}/bin
etc = $${buildout:directory}/etc etc = $${buildout:directory}/etc
services = $${:etc}/run services = $${:etc}/run
srv = $${buildout:directory}/srv srv = $${buildout:directory}/srv
source-code = $${:srv}/eggs-source-code
[download-source] [download-source]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
# Local development
[slapos.core]
<= download-source
repository = ${slapos.core-repository:location}
[slapos.core-setup]
recipe = plone.recipe.command
command = echo "Updating setup...";cd $${slapos.core:location}; export PATH="$${slapos-test-runner:prepend-path}:$PATH"; export CPPFLAGS="$${environment:CPPFLAGS}"; export LDFLAGS="$${environment:LDFLAGS}"; export PYTHONPATH="$${environment:PYTHONPATH}"; export LOCAL_IPV4="$${environment:LOCAL_IPV4}"; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n
update-command = $${:command}
[caucase] [caucase]
<= download-source <= download-source
repository = ${caucase-repository:location} repository = ${caucase-repository:location}
[erp5.util]
<= download-source
repository = ${erp5.util-repository:location}
[slapos.cookbook] [slapos.cookbook]
<= download-source <= download-source
repository = ${slapos.cookbook-repository:location} repository = ${slapos.cookbook-repository:location}
[slapos.recipe.template] [slapos.core]
<= download-source <= download-source
repository = ${slapos.recipe.template-repository:location} repository = ${slapos.core-repository:location}
[slapos.recipe.build] [slapos.recipe.build]
<= download-source <= download-source
...@@ -60,57 +50,33 @@ repository = ${slapos.recipe.build-repository:location} ...@@ -60,57 +50,33 @@ repository = ${slapos.recipe.build-repository:location}
<= download-source <= download-source
repository = ${slapos.recipe.cmmi-repository:location} repository = ${slapos.recipe.cmmi-repository:location}
[slapos.toolbox] [slapos.recipe.template]
<= download-source <= download-source
repository = ${slapos.toolbox-repository:location} repository = ${slapos.recipe.template-repository:location}
[erp5-util] [slapos.toolbox]
<= download-source <= download-source
repository = ${erp5-util-repository:location} repository = ${slapos.toolbox-repository:location}
[erp5.util-setup]
recipe = plone.recipe.command
command = echo "Updating setup...";cd $${erp5-util:location}; export PATH="$${slapos-test-runner:prepend-path}:$PATH"; export CPPFLAGS="$${environment:CPPFLAGS}"; export LDFLAGS="$${environment:LDFLAGS}"; export PYTHONPATH="$${environment:PYTHONPATH}"; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n
update-command = $${:command}
[slapos-test-runner] [slapos-test-runner]
recipe = slapos.cookbook:egg_test recipe = slapos.cookbook:wrapper
run-test-suite = $${create-directory:bin}/runTestSuite wrapper-path = $${create-directory:bin}/runTestSuite
run-test-suite-binary = ${buildout:bin-directory}/runTestSuite command-line =
# The list of executables should be defined here and a combination ${buildout:bin-directory}/runTestSuite
# of tests should dynamically generated. --python_interpreter=${buildout:bin-directory}/${eggs:interpreter}
#python-list = $${} --source_code_path_list=$${caucase:location},$${erp5.util:location},$${slapos.cookbook:location},$${slapos.core:location},$${slapos.recipe.build:location},$${slapos.recipe.cmmi:location},$${slapos.recipe.template:location},$${slapos.toolbox:location}
test-list =
$${slapos.cookbook:location} # Notes about environment:
$${slapos.core:location} # * slapos.cookbook:wrapper does not seem to allow "extending" PATH. Tests
$${slapos.recipe.template:location} # needs ping, which is a setuid binary that cannot be installed via slapos
$${slapos.recipe.build:location} # way of building software without root access, so we keep "standard"
$${slapos.recipe.cmmi:location} # /usr/bin and /bin in $PATH
$${slapos.toolbox:location} # * LOCAL_IPV4 is needed for some slapos.core tests
$${erp5-util:location} environment =
$${caucase:location} PATH=${coreutils:location}/bin:${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:/usr/bin/:/bin/
prepend-path = ${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:${python2.7:location}/bin LOCAL_IPV4=$${slap-configuration:ipv4-random}
environment = environment
[environment]
CPPFLAGS = -I${python2.7:location}/include/python2.7 -I${libxml2:location}/include -I${libxslt:location}/include
LDFLAGS = -L${python2.7:location}/lib -L${libxml2:location}/lib -L${libxslt:location}/lib -L${libxslt:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${python2.7:location}/lib -Wl,-rpath=${libxml2:location}/lib -Wl,-rpath=${libxslt:location}/lib -Wl,-rpath=${zlib:location}/lib
LD_LIBRARY_PATH = ${python2.7:location}/lib:${libxml2:location}/lib:${libxslt:location}/lib:${libxslt:location}/lib:${zlib:location}/lib
PYTHONPATH = ${python-setuptools:pythonpath}:${buildout:eggs-directory}:${buildout:develop-eggs-directory}
LOCAL_IPV4 = $${slap-configuration:ipv4-random}
[sh-environment]
# Section exposes testing default environment as sh file. It is thus easy
# to directly develop and test the egg inside of this instance.
recipe = collective.recipe.template
input = inline:
export PATH="$${slapos-test-runner:prepend-path}:$PATH"
export CPPFLAGS="$${environment:CPPFLAGS}"
export LDFLAGS="$${environment:LDFLAGS}"
export PYTHONPATH="$${environment:PYTHONPATH}"
export PS1="[slapos-testing env Active] $PS1"
output = $${create-directory:bin}/environment.sh
mode = 755
[phantomjs-wrapper] [phantomjs-wrapper]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
......
...@@ -6,26 +6,78 @@ extends = ...@@ -6,26 +6,78 @@ extends =
../../component/libxml2/buildout.cfg ../../component/libxml2/buildout.cfg
../../component/libxslt/buildout.cfg ../../component/libxslt/buildout.cfg
../../component/bcrypt/buildout.cfg ../../component/bcrypt/buildout.cfg
../../component/python-2.7/buildout.cfg
../../component/python-setuptools/buildout.cfg
../../component/zlib/buildout.cfg ../../component/zlib/buildout.cfg
../../component/phantomjs/buildout.cfg ../../component/phantomjs/buildout.cfg
../../component/pycurl/buildout.cfg ../../component/pycurl/buildout.cfg
../../component/coreutils/buildout.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
./buildout.hash.cfg
parts = parts =
caucase-repository bootstrap-slapos.recipe.cmmi
slapos.cookbook-repository
slapos.core-repository
slapos.recipe.template-repository
slapos.recipe.build-repository
slapos.recipe.cmmi-repository
slapos.toolbox-repository
erp5-util-repository
eggs eggs
phantomjs phantomjs
template template
[bootstrap-slapos.recipe.cmmi]
# install our develop version of slapos.recipe.cmmi before anything else,
# otherwise it will be installed from pypi by dependencies.
recipe = zc.recipe.egg
eggs = ${slapos.recipe.cmmi-setup:egg}
[setup-develop-egg]
recipe = zc.recipe.egg:develop
[caucase-setup]
<= setup-develop-egg
egg = caucase
setup = ${caucase-repository:location}
[erp5.util-setup]
<= setup-develop-egg
# XXX erp5.util does not have `test` extra require, but has a `testnode` extra require with same dependencies
egg = erp5.util[testnode]
setup = ${erp5.util-repository:location}
depends = ${slapos.core-setup:egg}
[slapos.cookbook-setup]
<= setup-develop-egg
# XXX slapos.cookbook does not have `test` extra require, `mock` is only listed in `tests_require` and is listed explicitly
egg = slapos.cookbook
setup = ${slapos.cookbook-repository:location}
depends = ${slapos.core-setup:egg}
[slapos.core-setup]
<= setup-develop-egg
# XXX slapos.cookbook does not have `test` extra require, `mock`, `pyflakes` and `httmock` are only listed in `tests_require` and are listed explicitly
egg = slapos.core
setup = ${slapos.core-repository:location}
[slapos.recipe.build-setup]
<= setup-develop-egg
egg = slapos.recipe.build[test]
setup = ${slapos.recipe.build-repository:location}
[slapos.recipe.cmmi-setup]
<= setup-develop-egg
egg = slapos.recipe.cmmi[test]
setup = ${slapos.recipe.cmmi-repository:location}
depends = ${slapos.recipe.build-setup:egg}
[slapos.recipe.template-setup]
<= setup-develop-egg
# XXX slapos.recipe.template does not have `test` extra require, `zope.testing` is only listed in `tests_require` and is listed explicitly
egg = slapos.recipe.template
setup = ${slapos.recipe.template-repository:location}
[slapos.toolbox-setup]
<= setup-develop-egg
# XXX slapos.toolbox does not have `test` extra require, `mock` and `pycurl` are only listed in `tests_require` and are listed explicitly
egg = slapos.toolbox
setup = ${slapos.toolbox-repository:location}
depends = ${slapos.core-setup:egg}
[eggs] [eggs]
recipe = zc.recipe.egg recipe = zc.recipe.egg
eggs = eggs =
...@@ -35,18 +87,24 @@ eggs = ...@@ -35,18 +87,24 @@ eggs =
${bcrypt:egg} ${bcrypt:egg}
dnspython dnspython
Jinja2 Jinja2
caucase ${caucase-setup:egg}
erp5.util ${erp5.util-setup:egg}
slapos.cookbook ${slapos.cookbook-setup:egg}
collective.recipe.template ${slapos.core-setup:egg}
plone.recipe.command ${slapos.recipe.build-setup:egg}
slapos.recipe.template ${slapos.recipe.cmmi-setup:egg}
slapos.recipe.cmmi ${slapos.recipe.template-setup:egg}
slapos.toolbox ${slapos.toolbox-setup:egg}
mock
zope.testing
httmock
pyflakes
entry-points = entry-points =
runTestSuite=erp5.util.testsuite:runTestSuite runTestSuite=erp5.util.testsuite:runTestSuite
scripts = scripts =
runTestSuite runTestSuite
interpreter=
python_for_test
[git-clone-repository] [git-clone-repository]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
...@@ -58,6 +116,10 @@ branch = master ...@@ -58,6 +116,10 @@ branch = master
<= git-clone-repository <= git-clone-repository
repository = https://lab.nexedi.com/nexedi/caucase.git repository = https://lab.nexedi.com/nexedi/caucase.git
[erp5.util-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/erp5.git
[slapos.cookbook-repository] [slapos.cookbook-repository]
<= git-clone-repository <= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.git repository = https://lab.nexedi.com/nexedi/slapos.git
...@@ -73,28 +135,38 @@ repository = https://lab.nexedi.com/nexedi/slapos.recipe.template.git ...@@ -73,28 +135,38 @@ repository = https://lab.nexedi.com/nexedi/slapos.recipe.template.git
[slapos.recipe.build-repository] [slapos.recipe.build-repository]
<= git-clone-repository <= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.recipe.build.git repository = https://lab.nexedi.com/nexedi/slapos.recipe.build.git
# We use the system git and not slapos provided one, because
# slapos.recipe.build is a dependency of slapos.recipe.cmmi
#git-executable = git
[slapos.recipe.cmmi-repository] [slapos.recipe.cmmi-repository]
<= git-clone-repository <= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.recipe.cmmi.git repository = https://lab.nexedi.com/nexedi/slapos.recipe.cmmi.git
# We use the system git and not slapos provided one, because slapos git needs
# slapos.recipe.cmmi to be installed. This circular dependency cause parts to
# be reinstalled everytime buildout is run because signatures are not stable.
#git-executable = git
[slapos.toolbox-repository] [slapos.toolbox-repository]
<= git-clone-repository <= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.toolbox.git repository = https://lab.nexedi.com/nexedi/slapos.toolbox.git
[erp5-util-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/erp5.git
[template] [template]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg url = ${:_profile_base_location_}/${:filename}
md5sum = 6626794c9dbb2530bb8ba3d331e27542
output = ${buildout:directory}/template.cfg output = ${buildout:directory}/template.cfg
mode = 640 mode = 640
[versions] [versions]
Pygments = 2.1.3 Pygments = 2.1.3
collective.recipe.template = 1.10
plone.recipe.command = 1.1 # clear the version of tested eggs, to make sure we installed the developped ones
slapos.recipe.template = 4.3 caucase =
erp5.util =
slapos.cookbook =
slapos.core =
slapos.recipe.build =
slapos.recipe.cmmi =
slapos.recipe.template =
slapos.toolbox =
...@@ -16,7 +16,7 @@ gunicorn = 19.7.1 ...@@ -16,7 +16,7 @@ gunicorn = 19.7.1
prettytable = 0.7.2 prettytable = 0.7.2
pycurl = 7.43.0 pycurl = 7.43.0
slapos.recipe.template = 4.3 slapos.recipe.template = 4.3
slapos.toolbox = 0.74 slapos.toolbox = 0.76
smmap = 0.9.0 smmap = 0.9.0
# Required by: # Required by:
......
...@@ -95,7 +95,7 @@ futures = 3.1.1 ...@@ -95,7 +95,7 @@ futures = 3.1.1
gitdb2 = 2.0.2 gitdb2 = 2.0.2
gunicorn = 19.7.1 gunicorn = 19.7.1
slapos.recipe.template = 4.3 slapos.recipe.template = 4.3
slapos.toolbox = 0.74 slapos.toolbox = 0.76
smmap2 = 2.0.3 smmap2 = 2.0.3
# Required by: # Required by:
......
...@@ -583,7 +583,7 @@ extra-paths = ...@@ -583,7 +583,7 @@ extra-paths =
# patches for eggs # patches for eggs
patch-binary = ${patch:location}/bin/patch patch-binary = ${patch:location}/bin/patch
Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic.patch#e8029103350dad364d25747514a20327 Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic.patch#1d9a56e9af4371f5b6951ebf217a15d7
Acquisition-patch-options = -p1 Acquisition-patch-options = -p1
Products.DCWorkflow-patches = ${:_profile_base_location_}/../../component/egg-patch/Products.DCWorkflow/workflow_method.patch#975b49e96bae33ac8563454fe5fa9899 Products.DCWorkflow-patches = ${:_profile_base_location_}/../../component/egg-patch/Products.DCWorkflow/workflow_method.patch#975b49e96bae33ac8563454fe5fa9899
Products.DCWorkflow-patch-options = -p1 Products.DCWorkflow-patch-options = -p1
...@@ -625,7 +625,7 @@ scripts += ...@@ -625,7 +625,7 @@ scripts +=
# neoppod, mysqlclient, slapos.recipe.template & [slapos-deps-eggs] # neoppod, mysqlclient, slapos.recipe.template & [slapos-deps-eggs]
# patched eggs # patched eggs
Acquisition = 2.13.9+SlapOSPatched001 Acquisition = 2.13.12+SlapOSPatched001
Products.DCWorkflow = 2.2.4+SlapOSPatched001 Products.DCWorkflow = 2.2.4+SlapOSPatched001
ocropy = 1.0+SlapOSPatched001 ocropy = 1.0+SlapOSPatched001
pysvn = 1.7.10+SlapOSPatched002 pysvn = 1.7.10+SlapOSPatched002
......
[versions] [versions]
AccessControl = 2.13.15 AccessControl = 2.13.16
Acquisition = 2.13.11 Acquisition = 2.13.12
DateTime = 2.12.8 DateTime = 2.12.8
DocumentTemplate = 2.13.4 DocumentTemplate = 2.13.4
ExtensionClass = 2.13.2 ExtensionClass = 2.13.2
...@@ -21,31 +21,35 @@ Products.Sessions = 3.0 ...@@ -21,31 +21,35 @@ Products.Sessions = 3.0
Products.StandardCacheManagers = 2.13.1 Products.StandardCacheManagers = 2.13.1
Products.TemporaryFolder = 3.0 Products.TemporaryFolder = 3.0
Products.ZCTextIndex = 2.13.5 Products.ZCTextIndex = 2.13.5
Products.ZCatalog = 2.13.27 Products.ZCatalog = 2.13.29
Pygments = 2.2.0 Pygments = 2.2.0
Record = 2.13.0 Record = 2.13.0
RestrictedPython = 3.6.0 RestrictedPython = 3.6.0
Sphinx = 1.0.8 Sphinx = 1.0.8
ZConfig = 2.9.3 ZConfig = 2.9.3
ZODB3 = 3.10.7 ZODB3 = 3.10.7
Zope2 = 2.13.26
ZServer = 3.0 ZServer = 3.0
Zope2 = 2.13.27
ZopeUndo = 2.12.0 ZopeUndo = 2.12.0
docutils = 0.12 docutils = 0.12
initgroups = 2.13.0 initgroups = 2.13.0
mechanize = 0.2.5 mechanize = 0.2.5
mr.developer = 1.35 mr.developer = 1.38
pluggy = 0.6.0
py = 1.5.2
pytz = 2017.2
repoze.retry = 1.2 repoze.retry = 1.2
repoze.tm2 = 1.0 repoze.tm2 = 1.0
repoze.who = 2.0 repoze.who = 2.0
tempstorage = 2.12.2 tempstorage = 2.12.2
tox = 2.9.1
transaction = 1.1.1 transaction = 1.1.1
z3c.checkversions = 0.5 z3c.checkversions = 0.5
zExceptions = 2.13.0 zExceptions = 2.13.0
zLOG = 2.11.2 zLOG = 2.11.2
zc.buildout = 2.3.1 zc.buildout = 2.3.1
zc.lockfile = 1.0.2 zc.lockfile = 1.0.2
zc.recipe.egg = 2.0.3 zc.recipe.egg = 2.0.5
zc.recipe.testrunner = 1.2.1 zc.recipe.testrunner = 1.2.1
zdaemon = 2.0.7 zdaemon = 2.0.7
zope.annotation = 3.5.0 zope.annotation = 3.5.0
...@@ -66,7 +70,7 @@ zope.exceptions = 3.6.2 ...@@ -66,7 +70,7 @@ zope.exceptions = 3.6.2
zope.filerepresentation = 3.6.1 zope.filerepresentation = 3.6.1
zope.i18n = 3.7.4 zope.i18n = 3.7.4
zope.i18nmessageid = 3.5.3 zope.i18nmessageid = 3.5.3
zope.interface = 3.6.7 zope.interface = 3.6.8
zope.lifecycleevent = 3.6.2 zope.lifecycleevent = 3.6.2
zope.location = 3.9.1 zope.location = 3.9.1
zope.pagetemplate = 3.5.2 zope.pagetemplate = 3.5.2
...@@ -87,14 +91,3 @@ zope.testbrowser = 3.11.1 ...@@ -87,14 +91,3 @@ zope.testbrowser = 3.11.1
zope.testing = 3.9.7 zope.testing = 3.9.7
zope.traversing = 3.13.2 zope.traversing = 3.13.2
zope.viewlet = 3.7.2 zope.viewlet = 3.7.2
# Required by:
# Jinja2==2.8.1
MarkupSafe = 0.23
# Required by:
# DateTime==2.12.8
# Zope2==2.13.26
# zope.i18n==3.7.4
# zope.testbrowser==3.11.1
pytz = 2016.10
...@@ -84,66 +84,42 @@ NB: You should use double $ (ex: $${monitor-template:rendered}) instead of one $ ...@@ -84,66 +84,42 @@ NB: You should use double $ (ex: $${monitor-template:rendered}) instead of one $
Add a promise Add a promise
------------- -------------
Monitor stack will include slapos promise directory etc/promise to promise folder. All files in that directory will be considered as a promise. To learn how to write a promise in SlapOS, please read this document:
This mean that all slapos promises will be checked frequently by monitor.
https://www.erp5.com/slapos-TechnicalNote.General.SlapOS.Monitoring.Specifications
[monitor-conf-parameters] Writing a promise consists of defining a class called RunPromise which inherits from GenericPromise class and defining methods: anomaly(), sense() and test(). Python promises should be placed into the folder etc/plugin of the computer partition.
... New promises should be placed into the folder etc/plugin, legacy promise are into the folder etc/promise. Legacy promises are bash or other executable promises script which does not use GenericPromise class.
promise-folder = ${directory:promises}
...
Monitor will run each promise every minutes and save the result in a json file. Here is an exemple of promise result:
{"status": "ERROR", "change-time": 1466415901.53, "hosting_subscription": "XXXX", "title": "vnc_promise", "start-date": "2016-06-21 10:47:01", "instance": "XXXX-title", "_links": {"monitor": {"href": "MONITOR_PRIVATE_URL"}}, "message": "PROMISE_OUPT_MESSAGE", "type": "status"}
A promise will be ran during a short time and report the status: ERROR or OK, plus an ouput message which says what was good or bad.
The promise should not run for more that 20 seconds, else it will be interrupted because of time out. However this value can be modified from monitoring web interface, see parameter "promise-timeout" of your hosting subscription.
On slapos, the default timeout value is also 20 seconds, if the value is modified on monitor (ex: to 50 seconds), it will still fails when slapgrid will process instance if the promise execution exceed 20 seconds.
Promises result are published in web public folder, access URL is: MONITOR_BASE_URL/private/PROMISE_NAME.status.json
Everytime monitor will run a promise an history of result will be also updated. The promise history will be updated during one day, after that a new history will be created.
To access promise history file as JSON, use URL MONITOR_BASE_URL/private/PROMISE_NAME.history.json
Add a promise: monitor promise
------------------------------
Monitor promise is also a promise like normal promise script but it will be placed into the folder ${monitor-directory:promises}:
[monitor-promise-xxxxx]
recipe = slapos.recipe.template:jinja2
rendered = ${monitor-directory:promises}/my-custom-monitor-promise
Theses promises will be executed only by monitor (not slapos) every minutes and will report same infor as default promises. This is another way to
add more custom promises to check if server is overloaded, or if network start to be slow, etc...
You will use slapos.cookbook:promise.plugin to generate your promise script into `etc/plugin` directory. Add promise will look like this:
Add custom scripts to monitor [promise-check-site]
----------------------------- recipe = slapos.cookbook:promise.plugin
eggs =
slapos.toolbox
output = ${directory:plugins}/promise-check-mysite-status.py
content =
from slapos.promise.plugin.check_site_state import RunPromise
config-site-url = ${publish:site-url}
config-connection-timeout = 20
config-foo = bar
mode = 600
Custom script will be automatically run by the monitor and result will be reported in monitor private folder. To add your custom script in ${monitor-directory:reports} folder. Here is an example: Then you will have to add `promise-check-site` section to buildout parts, so it will be installed.
[monitor-check-webrunner-internal-instance] In your promise code, you will be able to call `self.getConfig('site-url')`, `self.getConfig('connection-timeout')` and `self.getConfig('foo')`. The
recipe = slapos.recipe.template:jinja2 returned value is `None` if the config parameter is not set.
template = ${monitor-check-webrunner-internal-instance:location}/${monitor-check-webrunner-internal-instance:filename}
rendered = $${monitor-directory:reports}/$${:filename}
filename = monitor-check-webrunner-internal-instance
mode = 0744
The script will be executed every minutes by default. To change, put periodicity in script name: Slapgrid will run each promise every time a partition is processed (every minutes in theory), if the partition is up to date, slapgrid will only run promises anomaly check and save the result in a json file. Here is an exemple of promise result:
- monitor-check-webrunner-internal-instance_every_1_minute
- monitor-check-webrunner-internal-instance_every_25_minute
- monitor-check-webrunner-internal-instance_every_1_hour
- monitor-check-webrunner-internal-instance_every_3_hour
- ...
the script name should end with _every_XX_hour or _every_XX_minute. With this, we can set filename as: {"result": {"date": "2018-03-22T15:35:07", "failed": false, "message": "buildout is OK", "type": "Test Result"}, "path": "PARTITION_DIRECTORY/etc/plugin/buildout-slappart0-status.py", "name": "buildout-slappart0-status.py", "execution-time": 0.1, "title": "buildout-slappart0-status"}
filename = monitor-check-webrunner-internal-instance_every_2_minute The promise execution time should be short, by default promise-timeout in slapgrid is to 20 seconds. If a promise runs in more than the defined promise-timeout, the process is killed and a "promise timed out" message is returned.
JSON in the folder `PARTITION_DIRECTORY/.slapgrid/promise/result`, and promise logs are in `PARTITION_DIRECTORY/.slapgrid/promise/log`.
You can get custom script results files at MONITOR_BASE_URL/private/FILE_NAME.
Monitor will expose promise JSON result in web public folder, access URL is: `MONITOR_BASE_URL/public/promise/PROMISE_TITLE.status.json`. Log files are exposed in monitor private web folder,
access URL is: `MONITOR_BASE_URL/private/log/monitor/promise`
Add custom file or directory to monitor Add custom file or directory to monitor
--------------------------------------- ---------------------------------------
...@@ -158,20 +134,22 @@ Log or others files can be added in monitor public or private directory: ...@@ -158,20 +134,22 @@ Log or others files can be added in monitor public or private directory:
${directory:log} ${directory:log}
... ...
files in public directory are accessible at MONITOR_BASE_URL/public, and for private directory: MONITOR_BASE_URL/private. files in public directory are accessible at `MONITOR_BASE_URL/public`, and for private directory: `MONITOR_BASE_URL/private`.
Monitor RSS and OPML Feed Monitor promise history, RSS and OPML Feed
------------------------- ------------------------------------------
Monitor generate rss containing latest result for all promises, this feed will be upaded every minutes. The Rss fee URL is Monitor read every minutes JSON promises result files to build Rss and full instance state. The Rss feed URL is
MONITOR_BASE_URL/public/feed `MONITOR_BASE_URL/public/feed`
OPML Feed is used to aggregate many feed URL, this is used on monitor to link many single monitor instances. For example, a resilient OPML Feed is used to aggregate many feed URL, this is used on monitor to link many single monitor instances. For example, a resilient
webrunner has 5 instances at least, each instance has a monitor which leads to 5 monitor instances too. One main instance (usally the root instance) webrunner has 3 instances at least, each instance has a monitor which leads to 3 monitor instances too. One main instance (usally the root instance)
will collect rss feeds of all others monitor's in a single OPML file. This file is published and used to configure a monitor backend to the web interface. will collect rss feed of all others monitor's in a single OPML file. This file is published and used to configure a monitor backend to the web interface.
The URL of OPML feed is: MONITOR_BASE_URL/public/feeds The URL of OPML feed is: `MONITOR_BASE_URL/public/feeds`
Everytime monitor will also produce history of for each promise in a single JSON file.
To access promise history file as JSON, use URL `MONITOR_BASE_URL/public/PROMISE_TITLE.history.json`
Monitor Base web directory tree Monitor Base web directory tree
------------------------------- -------------------------------
...@@ -184,16 +162,13 @@ Monitor Base web directory tree ...@@ -184,16 +162,13 @@ Monitor Base web directory tree
(webdav) X Y (webdav) X Y
| |
--------------------------------- ---------------------------------
| | | | | |
public private jio_public jio_private public private
X Y | |
.jio_documents .jio_documents
X Y X Y
MONITOR_BASE_URL/public or private is for normal HTTPS. MONITOR_BASE_URL/public or private is for normal HTTPS.
MONITOR_BASE_URL/share is the webdav URL. public/ and private/ are linked to public and private directories. MONITOR_BASE_URL/share is the webdav URL. public/ and private/ are linked to public and private directories.
webdav also has jio_public/.jio_documents and jio_private/.jio_documents which are linked to public and private directory and they works with jio webdav pluging.
Access to Monitor Access to Monitor
......
...@@ -44,6 +44,7 @@ eggs = ...@@ -44,6 +44,7 @@ eggs =
collective.recipe.template collective.recipe.template
cns.recipe.symlink cns.recipe.symlink
slapos.toolbox slapos.toolbox
slapos.core
# Do no generate any scripts here as all of them are generated by extraeggs # Do no generate any scripts here as all of them are generated by extraeggs
scripts = scripts =
...@@ -72,22 +73,12 @@ md5sum = 1695c9a06a2b11ccfe893d7a224e489d ...@@ -72,22 +73,12 @@ md5sum = 1695c9a06a2b11ccfe893d7a224e489d
[monitor-conf] [monitor-conf]
<= monitor-template-base <= monitor-template-base
filename = monitor.conf.in filename = monitor.conf.in
md5sum = 888e2845d09bfaa59c25f56f5bcf76b1 md5sum = 91c4c9bba1f7df788b9b7a059ed89ac2
[monitor-instance-info]
<= monitor-template-base
filename = instance-info.conf.in
md5sum = 1bdb4e05c6be04f4e5766c64467fbcec
[monitor-httpd-cors] [monitor-httpd-cors]
<= monitor-template-base <= monitor-template-base
filename = httpd-cors.cfg.in filename = httpd-cors.cfg.in
md5sum = 683ea85fc054094248baf5752dd089bf md5sum = 683ea85fc054094248baf5752dd089bf
[monitor-check-free-disk-space]
<= monitor-template-base
filename = check_free_disk.in
md5sum = bab457ac4d139ed31d0b343a7d14d996
# End templates files # End templates files
# XXX keep compatibility (with software/ipython_notebook/software.cfg ) # XXX keep compatibility (with software/ipython_notebook/software.cfg )
...@@ -112,7 +103,6 @@ context = ...@@ -112,7 +103,6 @@ context =
raw monitor_configwrite ${buildout:directory}/bin/monitor.configwrite raw monitor_configwrite ${buildout:directory}/bin/monitor.configwrite
raw monitor_conf_template ${monitor-conf:location}/${monitor-conf:filename} raw monitor_conf_template ${monitor-conf:location}/${monitor-conf:filename}
raw monitor_https_cors ${monitor-httpd-cors:location}/${monitor-httpd-cors:filename} raw monitor_https_cors ${monitor-httpd-cors:location}/${monitor-httpd-cors:filename}
raw monitor_instance_info ${monitor-instance-info:location}/${monitor-instance-info:filename}
raw curl_executable_location ${curl:location}/bin/curl raw curl_executable_location ${curl:location}/bin/curl
raw dash_executable_location ${dash:location}/bin/dash raw dash_executable_location ${dash:location}/bin/dash
raw dcron_executable_location ${dcron:location}/sbin/crond raw dcron_executable_location ${dcron:location}/sbin/crond
...@@ -122,7 +112,7 @@ context = ...@@ -122,7 +112,7 @@ context =
raw python_executable ${buildout:executable} raw python_executable ${buildout:executable}
raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter} raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter}
raw template_wrapper ${monitor-template-wrapper:location}/${monitor-template-wrapper:filename} raw template_wrapper ${monitor-template-wrapper:location}/${monitor-template-wrapper:filename}
raw template_check_disk_space ${monitor-check-free-disk-space:location}/${monitor-check-free-disk-space:filename} raw check_disk_space ${buildout:directory}/bin/check-free-disk
raw bin_directory ${buildout:directory}/bin raw bin_directory ${buildout:directory}/bin
depends = depends =
${monitor-eggs:eggs} ${monitor-eggs:eggs}
...@@ -131,6 +121,6 @@ depends = ...@@ -131,6 +121,6 @@ depends =
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
pycurl = 7.43.0 pycurl = 7.43.0
slapos.toolbox = 0.74 slapos.toolbox = 0.76
pyasn1 = 0.3.7 pyasn1 = 0.3.7
...@@ -15,4 +15,4 @@ ...@@ -15,4 +15,4 @@
# not need these here). # not need these here).
[monitor2-template] [monitor2-template]
filename = instance-monitor.cfg.jinja2.in filename = instance-monitor.cfg.jinja2.in
md5sum = 75fe1b222c269e69226796bf6059a747 md5sum = 86517ddccc86c76deaedf11066470550
...@@ -21,28 +21,23 @@ log = ${:var}/log ...@@ -21,28 +21,23 @@ log = ${:var}/log
scripts = ${:etc}/run scripts = ${:etc}/run
services = ${:etc}/service services = ${:etc}/service
promises = ${:etc}/promise promises = ${:etc}/promise
plugins = ${:etc}/plugin
monitor = ${:srv}/monitor monitor = ${:srv}/monitor
monitor-promise = ${:etc}/monitor-promise
monitor-report = ${:etc}/monitor-report
[monitor-directory] [monitor-directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
bin = ${directory:bin} bin = ${directory:bin}
etc = ${directory:etc} etc = ${directory:etc}
run = ${directory:monitor}/run promises = ${directory:etc}/monitor-promise
#run = ${directory:scripts} reports = ${directory:etc}/monitor-report
promises = ${directory:monitor-promise}
reports = ${directory:monitor-report}
pids = ${directory:run}/monitor pids = ${directory:run}/monitor
cgi-bin = ${directory:monitor}/cgi-bin
webdav = ${directory:monitor}/webdav webdav = ${directory:monitor}/webdav
public = ${directory:monitor}/public public = ${directory:monitor}/public
private = ${directory:monitor}/private private = ${directory:monitor}/private
services = ${directory:services} documents = ${:private}/documents
services-conf = ${directory:etc}/monitor.conf.d
log = ${directory:log}/monitor log = ${directory:log}/monitor
monitor-var = ${directory:var}/monitor promise-result = ${buildout:directory}/.slapgrid/promise/result
monitor-log = ${directory:monitor}/private/monitor-log promise-log = ${buildout:directory}/.slapgrid/promise/log
[ca-directory] [ca-directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
...@@ -58,7 +53,7 @@ recipe = slapos.cookbook:certificate_authority ...@@ -58,7 +53,7 @@ recipe = slapos.cookbook:certificate_authority
openssl-binary = {{ openssl_executable_location }} openssl-binary = {{ openssl_executable_location }}
ca-dir = ${ca-directory:root} ca-dir = ${ca-directory:root}
requests-directory = ${ca-directory:requests} requests-directory = ${ca-directory:requests}
wrapper = ${monitor-directory:services}/certificate_authority wrapper = ${directory:services}/certificate_authority
ca-private = ${ca-directory:private} ca-private = ${ca-directory:private}
ca-certs = ${ca-directory:certs} ca-certs = ${ca-directory:certs}
ca-newcerts = ${ca-directory:newcerts} ca-newcerts = ${ca-directory:newcerts}
...@@ -78,18 +73,12 @@ root-title = ${monitor-instance-parameter:root-instance-title} ...@@ -78,18 +73,12 @@ root-title = ${monitor-instance-parameter:root-instance-title}
public-folder = ${monitor-directory:public} public-folder = ${monitor-directory:public}
private-folder = ${monitor-directory:private} private-folder = ${monitor-directory:private}
webdav-folder = ${monitor-directory:webdav} webdav-folder = ${monitor-directory:webdav}
report-folder = ${monitor-directory:reports}
base-url = ${monitor-instance-parameter:monitor-base-url} base-url = ${monitor-instance-parameter:monitor-base-url}
monitor-hal-json = ${monitor-directory:public}/monitor.hal.json
service-pid-folder = ${monitor-directory:pids} service-pid-folder = ${monitor-directory:pids}
crond-folder = ${logrotate-directory:cron-entries} crond-folder = ${logrotate-directory:cron-entries}
logrotate-folder = ${logrotate-directory:logrotate-entries} log-folder = ${monitor-directory:log}
promise-runner = {{ monitor_runpromise }} document-folder = ${monitor-directory:documents}
promise-folder = ${directory:promises}
monitor-promise-folder = ${monitor-directory:promises}
promises-timeout-file = ${monitor-promise-timeout-file:file}
pid-file = ${monitor-directory:pids}/monitor-bootstrap.pid pid-file = ${monitor-directory:pids}/monitor-bootstrap.pid
randomsleep = {{ bin_directory }}/randomsleep
public-path-list = public-path-list =
private-path-list = ${directory:log} private-path-list = ${directory:log}
...@@ -99,26 +88,28 @@ parameter-file-path = ${monitor-instance-parameter:configuration-file-path} ...@@ -99,26 +88,28 @@ parameter-file-path = ${monitor-instance-parameter:configuration-file-path}
parameter-list = parameter-list =
raw monitor-user ${monitor-instance-parameter:username} raw monitor-user ${monitor-instance-parameter:username}
htpasswd monitor-password ${httpd-monitor-htpasswd:password-file} ${monitor-instance-parameter:username} ${httpd-monitor-htpasswd:htpasswd-path} htpasswd monitor-password ${httpd-monitor-htpasswd:password-file} ${monitor-instance-parameter:username} ${httpd-monitor-htpasswd:htpasswd-path}
file promise-timeout ${monitor-promise-timeout-file:file}
file min-free-disk-MB ${promise-check-free-disk-space:config-file} file min-free-disk-MB ${promise-check-free-disk-space:config-file}
${monitor-instance-parameter:instance-configuration} ${monitor-instance-parameter:instance-configuration}
# htpasswd entry: htpasswd key password-file username htpasswd-file # htpasswd entry: htpasswd key password-file username htpasswd-file
collector-db = ${monitor-instance-parameter:collector-db}
collect-script = {{ monitor_collect }}
statistic-script = {{ monitor_statistic }}
python = {{ python_with_eggs }}
nice-cmd = ${xnice-bin:output}
promise-output-file = ${directory:monitor}/monitor-bootstrap-status promise-output-file = ${directory:monitor}/monitor-bootstrap-status
[monitor-promise-timeout-file] [monitor-promise-conf]
recipe = plone.recipe.command output-folder = ${monitor-directory:public}/promise
file = ${directory:etc}/promise_timeout history-folder = ${monitor-directory:public}
command = promise-folder = ${directory:plugins}
if [ ! -s "${:file}" ]; then legacy-promise-folder = ${directory:promises}
echo "20" > ${:file} pid-path = ${monitor-directory:pids}/runpromise.pid
fi partition-folder = ${buildout:directory}
master-url = ${slap-connection:server-url}
partition-cert = ${slap-connection:cert-file}
partition-key = ${slap-connection:key-file}
partition-id = ${slap-connection:partition-id}
computer-id = ${slap-connection:computer-id}
ipv4 = ${slap-configuration:ipv4-random}
ipv6 = ${slap-configuration:ipv6-random}
software-release = ${slap-connection:software-release-url}
software-type = ${slap-configuration:slap-software-type}
[monitor-base-url-dict] [monitor-base-url-dict]
# place holder to be used to collect erp5 monitor urls # place holder to be used to collect erp5 monitor urls
...@@ -129,39 +120,14 @@ template = {{ monitor_conf_template }} ...@@ -129,39 +120,14 @@ template = {{ monitor_conf_template }}
rendered = ${directory:etc}/${:filename} rendered = ${directory:etc}/${:filename}
filename = monitor.conf filename = monitor.conf
context = section parameter_dict monitor-conf-parameters context = section parameter_dict monitor-conf-parameters
section promise_parameter_dict monitor-promise-conf
section monitor_base_urls monitor-base-url-dict section monitor_base_urls monitor-base-url-dict
[instance-info-parameters]
name = ${monitor-instance-parameter:monitor-title}
root-name = ${monitor-instance-parameter:root-instance-title}
computer-id = ${slap-connection:computer-id}
ipv4 = ${slap-configuration:ipv4-random}
ipv6 = ${slap-configuration:ipv6-random}
software-release = ${slap-connection:software-release-url}
software-type = ${slap-configuration:slap-software-type}
partition-id = ${slap-connection:partition-id}
[monitor-instance-info]
recipe = slapos.recipe.template:jinja2
template = {{ monitor_instance_info }}
rendered = ${directory:etc}/${:filename}
filename = instance-info.conf
context =
section instance_dict instance-info-parameters
[python-symlink]
recipe = plone.recipe.command
target = ${directory:bin}
command = ln -sf {{ python_with_eggs }} ${:target}/python
update-command = ${:command}
[start-monitor] [start-monitor]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
command-line = {{ python_executable }} {{ monitor_bin }} --config_file ${monitor-conf:rendered} command-line = {{ monitor_bin }} -c ${monitor-conf:rendered}
name = bootstrap-monitor name = bootstrap-monitor
wrapper-path = ${directory:scripts}/${:name} wrapper-path = ${directory:scripts}/${:name}
environment =
PATH=${python-symlink:target}:/usr/local/bin:/usr/bin:/bin
[monitor-htpasswd] [monitor-htpasswd]
recipe = slapos.cookbook:generate.password recipe = slapos.cookbook:generate.password
...@@ -183,12 +149,18 @@ update-command = ${:command} ...@@ -183,12 +149,18 @@ update-command = ${:command}
user = ${monitor-instance-parameter:username} user = ${monitor-instance-parameter:username}
password = ${monitor-instance-parameter:password} password = ${monitor-instance-parameter:password}
[monitor-symlink]
recipe = cns.recipe.symlink
symlink =
${monitor-directory:promise-result} = ${monitor-directory:public}/promise
${monitor-directory:promise-result} = ${monitor-directory:log}/promise
[monitor-httpd-conf-parameter] [monitor-httpd-conf-parameter]
listening-ip = ${monitor-instance-parameter:monitor-httpd-ipv6} listening-ip = ${monitor-instance-parameter:monitor-httpd-ipv6}
port = ${monitor-instance-parameter:monitor-httpd-port} port = ${monitor-instance-parameter:monitor-httpd-port}
pid-file = ${directory:run}/monitor-httpd.pid pid-file = ${directory:run}/monitor-httpd.pid
access-log = ${monitor-directory:log}/monitor-httpd-access.log access-log = ${directory:log}/monitor-httpd-access.log
error-log = ${monitor-directory:log}/monitor-httpd-error.log error-log = ${directory:log}/monitor-httpd-error.log
cert-file = ${ca-directory:certs}/httpd.crt cert-file = ${ca-directory:certs}/httpd.crt
key-file = ${ca-directory:certs}/httpd.key key-file = ${ca-directory:certs}/httpd.key
htpasswd-file = ${httpd-monitor-htpasswd:htpasswd-path} htpasswd-file = ${httpd-monitor-htpasswd:htpasswd-path}
...@@ -235,8 +207,8 @@ command = kill -USR1 $(cat ${monitor-httpd-conf-parameter:pid-file}) ...@@ -235,8 +207,8 @@ command = kill -USR1 $(cat ${monitor-httpd-conf-parameter:pid-file})
[logrotate-entry-monitor-httpd] [logrotate-entry-monitor-httpd]
<= logrotate-entry-base <= logrotate-entry-base
name = monitor-apache name = monitor-apache
log = $${basedirectory:log}/monitor/monitor-httpd-error.log log = ${directory:log}/monitor-httpd-*.log
post = test ! -s $${buildout:directory}/var/run/monitor-httpd.pid || $${buildout:directory}/bin/slapos-kill --pidfile $${buildout:directory}/var/run/monitor-httpd.pid -s USR1 post = test ! -s ${monitor-httpd-conf-parameter:pid-file} || {{ bin_directory }}/slapos-kill --pidfile ${monitor-httpd-conf-parameter:pid-file} -s USR1
[xnice-bin] [xnice-bin]
recipe = collective.recipe.template recipe = collective.recipe.template
...@@ -253,28 +225,74 @@ wrapper-path = ${directory:promises}/promise-monitor-httpd-is-process-older-than ...@@ -253,28 +225,74 @@ wrapper-path = ${directory:promises}/promise-monitor-httpd-is-process-older-than
[monitor-globalstate-wrapper] [monitor-globalstate-wrapper]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
command-line = ${xnice-bin:output} {{ monitor_genstatus }} '${monitor-conf:rendered}' '${monitor-instance-info:rendered}' command-line = ${xnice-bin:output} {{ monitor_genstatus }} '${monitor-conf:rendered}'
wrapper-path = ${directory:bin}/monitor-globalstate wrapper-path = ${directory:bin}/monitor-globalstate
[monitor-configurator-wrapper] [monitor-configurator-wrapper]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
# XXX - hard coded path # XXX - hard coded path
command-line = {{ monitor_configwrite }} --config_folder '${monitor-conf-parameters:private-folder}/config/.jio_documents' --output_cfg_file '${monitor-instance-parameter:configuration-file-path}' --htpasswd_bin '{{ apache_location }}/bin/htpasswd' --monitor_https_cors {{ monitor_https_cors }} command-line = ${xnice-bin:output} {{ monitor_configwrite }}
--config_folder '${monitor-conf-parameters:private-folder}/config/.jio_documents'
--output_cfg_file '${monitor-instance-parameter:configuration-file-path}'
--htpasswd_bin '{{ apache_location }}/bin/htpasswd'
--monitor_https_cors {{ monitor_https_cors }}
wrapper-path = ${directory:bin}/monitor-configurator wrapper-path = ${directory:bin}/monitor-configurator
[monitor-collect-wrapper]
recipe = slapos.cookbook:wrapper
command-line = ${xnice-bin:output} {{ monitor_collect }}
--output_folder ${monitor-directory:documents}
--collector_db ${monitor-instance-parameter:collector-db}
--pid_file ${monitor-directory:pids}/monitor-collect.pid
wrapper-path = ${directory:bin}/monitor-collect
[monitor-globalstate-cron-entry] [monitor-globalstate-cron-entry]
recipe = slapos.cookbook:cron.d recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries} cron-entries = ${cron:cron-entries}
name = monitor-globalstate name = monitor-globalstate
frequency = * * * * * frequency = */2 * * * *
command = {{ bin_directory }}/randomsleep 60 && ${monitor-globalstate-wrapper:wrapper-path} command = {{ bin_directory }}/randomsleep 20 && ${monitor-globalstate-wrapper:wrapper-path}
[monitor-configurator-cron-entry] [monitor-configurator-cron-entry]
recipe = slapos.cookbook:cron.d recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries} cron-entries = ${cron:cron-entries}
name = monitor-configurator name = monitor-configurator
frequency = * * * * * frequency = * * * * *
command = {{ bin_directory }}/randomsleep 60 && ${monitor-configurator-wrapper:wrapper-path} command = {{ bin_directory }}/randomsleep 10 && ${monitor-configurator-wrapper:wrapper-path}
[monitor-collect-cron-entry]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = monitor_collect
frequency = * * * * *
command = {{ bin_directory }}/randomsleep 40 && ${monitor-collect-wrapper:wrapper-path}
[logrotate-entry-monitor-data]
recipe = collective.recipe.template
name = monitor.data
log = ${monitor-directory:private}/*.data.json ${monitor-directory:documents}/*.data.json
input = inline:${:log} {
weekly
nocreate
olddir ${monitor-directory:documents}
rotate 104
nocompress
missingok
extension .json
dateext
dateformat -%Y-%m-%d
notifempty
}
output = ${logrotate-directory:logrotate-entries}/${:name}
mode = 600
[logrotate-entry-monitor-promise-history]
<= logrotate-entry-base
name = monitor.service.status
log = ${monitor-directory:public}/*.history.json
rotate-num = 0
frequency = weekly
pre = {{ monitor_statistic }} --history_folder ${monitor-directory:public}
[monitor-httpd-promise] [monitor-httpd-promise]
recipe = slapos.cookbook:check_url_available recipe = slapos.cookbook:check_url_available
...@@ -336,58 +354,35 @@ curl_path = {{ curl_executable_location }} ...@@ -336,58 +354,35 @@ curl_path = {{ curl_executable_location }}
check-secure = 1 check-secure = 1
[monitor-bootstrap-promise] [monitor-bootstrap-promise]
recipe = collective.recipe.template recipe = slapos.cookbook:promise.plugin
eggs =
slapos.toolbox
file = ${monitor-conf-parameters:promise-output-file} file = ${monitor-conf-parameters:promise-output-file}
error-log-file = ${buildout:directory}/.${slap-connection:partition-id}_${start-monitor:name}.log content =
input = inline:#!{{ dash_executable_location }} from slapos.promise.plugin.monitor_bootstrap_status import RunPromise
pidfile=${monitor-conf-parameters:pid-file} output = ${directory:plugins}/monitor-bootstrap-status.py
if [ -s $pidfile ]; then mode = 600
COUNTER=0 config-process-pid-file = ${monitor-conf-parameters:pid-file}
# Wait until max 20 seconds, the limit promise timeout config-process-name = ${start-monitor:name}
while [ $COUNTER -lt 40 ]; do config-status-file = ${:file}
if [ -n "$(ps -p $(cat $pidfile) -o pid=)" ]; then
((COUNTER=COUNTER+1))
sleep 0.5
else
break
fi
done
fi
if [ ! -f "${:file}" ]; then
echo "Monitor bootstrap exited with error."
log_file="${:error-log-file}"
if [ -s "$log_file" ]; then
echo " ---- Latest monitor-boostrap.log ----"
echo ""
tail -n 3 $log_file
fi
exit 2
else
echo "Bootstrap OK";
fi
output = ${directory:promises}/monitor-bootstrap-status
mode = 700
[promise-check-slapgrid] [promise-check-slapgrid]
recipe = collective.recipe.template recipe = slapos.cookbook:promise.plugin
error-log-file = ${buildout:directory}/.slapgrid-${slap-connection:partition-id}-error.log eggs =
input = inline:#!/bin/sh slapos.toolbox
if [ -f "${:error-log-file}" ]; then output = ${directory:plugins}/buildout-${slap-connection:partition-id}-status.py
>&2 cat ${:error-log-file} content =
exit 1 from slapos.promise.plugin.check_partition_deployment_state import RunPromise
fi config-monitor-url = ${monitor-instance-parameter:monitor-base-url}
output = ${monitor-directory:promises}/buildout-${slap-connection:partition-id}-status mode = 600
mode = 700
[promise-check-free-disk-space] [promise-check-free-disk-space]
recipe = slapos.recipe.template:jinja2 recipe = slapos.cookbook:wrapper
template = {{ template_check_disk_space }} command-line = {{ check_disk_space }}
rendered = ${monitor-directory:promises}/check-free-disk-space --collectordb ${monitor-instance-parameter:collector-db}
mode = 0700 --home_path ${buildout:directory}
context = --config ${:config-file}
key config_file :config-file wrapper-path = ${directory:promises}/check-free-disk-space
raw home_path ${buildout:directory}
raw python_bin {{ python_with_eggs }}
config-file = ${directory:etc}/min-free-disk-size config-file = ${directory:etc}/min-free-disk-size
...@@ -400,6 +395,7 @@ base-url = ${monitor-conf-parameters:base-url} ...@@ -400,6 +395,7 @@ base-url = ${monitor-conf-parameters:base-url}
depends = depends =
${monitor-globalstate-cron-entry:name} ${monitor-globalstate-cron-entry:name}
${monitor-configurator-cron-entry:name} ${monitor-configurator-cron-entry:name}
${monitor-collect-cron-entry:name}
${cron-entry-logrotate:name} ${cron-entry-logrotate:name}
${logrotate-entry-cron:name} ${logrotate-entry-cron:name}
${certificate-authority:wrapper} ${certificate-authority:wrapper}
...@@ -408,9 +404,12 @@ depends = ...@@ -408,9 +404,12 @@ depends =
${ca-monitor-httpd:wrapper} ${ca-monitor-httpd:wrapper}
${monitor-httpd-promise:filename} ${monitor-httpd-promise:filename}
${monitor-bootstrap-promise:file} ${monitor-bootstrap-promise:file}
${promise-check-slapgrid:output} ${monitor-symlink:recipe}
${promise-check-slapgrid:recipe}
${promise-monitor-httpd-is-process-older-than-dependency-set:wrapper-path} ${promise-monitor-httpd-is-process-older-than-dependency-set:wrapper-path}
${logrotate-entry-monitor-httpd:name} ${logrotate-entry-monitor-httpd:name}
${logrotate-entry-monitor-data:name}
${logrotate-entry-monitor-promise-history:name}
[monitor-publish] [monitor-publish]
monitor-base-url = ${monitor-publish-parameters:monitor-base-url} monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
......
#!{{ python_bin }}
import os
import sys
def free_space(path, fn):
while True:
try:
disk = os.statvfs(path)
return fn(disk)
except OSError:
pass
if os.sep not in path:
break
path = os.path.split(path)[0]
def user_free_space(path):
return free_space(path, lambda d: d.f_bsize * d.f_bavail)
def check_inode_usage(path):
max_inode_usage = 97.99 # < 98% usage
st = os.statvfs(path)
usage_output = ""
total_inode = st.f_files
free_inode = st.f_ffree
usage = round((float(total_inode - free_inode) / total_inode), 4) * 100
if usage > max_inode_usage:
return "Disk Inodes are widely used: %s%%" % usage
elif os.path.exists('/tmp'):
# check if /tmp is mounted on another disk than path
tmp_st = os.statvfs('/tmp')
if tmp_st.f_blocks != st.f_blocks:
tmp_usage = round((float(tmp_st.f_files - tmp_st.f_ffree) / tmp_st.f_files), 4) * 100
if tmp_usage > max_inode_usage:
return "Disk Inodes are widely used: %s%%" % tmp_usage
return ""
if __name__ == '__main__':
home_path = '{{ home_path }}'
config_file = '{{ config_file }}'
min_free_size = 1024*1024*1024*2 # 2G by default
if os.path.exists(config_file):
with open(config_file) as f:
min_size_str = f.read().strip()
if min_size_str == '0':
# disable check
print "Free disk space check is disabled\n set a number up to 0 to enable!"
exit(0)
if min_size_str.isdigit():
value = int(min_size_str)
if value >= 200:
# Minimum value is 200Mb, it's already low
min_free_size = int(min_size_str)*1024*1024
else:
with open(config_file, 'w') as f:
f.write(str(min_free_size/(1024*1024)))
real_free_space = user_free_space(home_path)
if real_free_space > min_free_size:
inode_usage = check_inode_usage(home_path)
if inode_usage:
print inode_usage
exit(2)
print "Disk usage: OK"
exit(0)
real_space_g = round(real_free_space/(1024.0*1024*1024), 2)
min_space_g = round(min_free_size/(1024.0*1024*1024), 2)
print 'Free disk space low: remaning %s G (threshold: %s G)' % (
real_space_g, min_space_g)
print 'You can modify minimum value on your monitor interface.'
exit(1)
[instance]
name = {{ instance_dict['name'] }}
root-name = {{ instance_dict['root-name'] }}
computer = {{ instance_dict['computer-id'] }}
ipv4 = {{ instance_dict['ipv4'] }}
ipv6 = {{ instance_dict['ipv6'] }}
software-release = {{ instance_dict['software-release'] }}
software-type = {{ instance_dict['software-type'] }}
partition = {{ instance_dict['partition-id'] }}
\ No newline at end of file
...@@ -11,4 +11,9 @@ monitor-url-list = ...@@ -11,4 +11,9 @@ monitor-url-list =
{% for key, value in monitor_base_urls.items() -%} {% for key, value in monitor_base_urls.items() -%}
{{ ' ' ~ value }} {{ ' ' ~ value }}
{% endfor -%} {% endfor -%}
{% endif -%} {% endif %}
[promises]
{% for key, value in promise_parameter_dict.items() -%}
{{ key }} = {{ value.strip().replace("\n", "\n ") }}
{% endfor -%}
\ No newline at end of file
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
# not need these here). # not need these here).
[pbsready] [pbsready]
filename = pbsready.cfg.in filename = pbsready.cfg.in
md5sum = bceb082deb6b3ee9b015c8b04436b52c md5sum = 367f8b8a09c3098844739f9b2856b912
[pbsready-import] [pbsready-import]
filename = pbsready-import.cfg.in filename = pbsready-import.cfg.in
...@@ -27,7 +27,7 @@ md5sum = c6c11db5372150019debb1ce519b907d ...@@ -27,7 +27,7 @@ md5sum = c6c11db5372150019debb1ce519b907d
[template-pull-backup] [template-pull-backup]
filename = instance-pull-backup.cfg.in filename = instance-pull-backup.cfg.in
md5sum = 5f35d9c0d08a268ed0c0f0dbef114f42 md5sum = 85e777bd349a5c881d36e747882aee8a
[template-replicated] [template-replicated]
filename = template-replicated.cfg.in filename = template-replicated.cfg.in
......
...@@ -136,13 +136,13 @@ notifier-url = http://[$${notifier:host}]:$${notifier:port} ...@@ -136,13 +136,13 @@ notifier-url = http://[$${notifier:host}]:$${notifier:port}
slave-instance-list = $${slap-parameter:slave_instance_list} slave-instance-list = $${slap-parameter:slave_instance_list}
ignore-known-hosts-file = $${slap-parameter:ignore-known-hosts-file} ignore-known-hosts-file = $${slap-parameter:ignore-known-hosts-file}
# To get a verbose feed about PBS state # To get a verbose feed about PBS state
instance-root-name = $${instance-info-parameters:root-name} instance-root-name = $${monitor-instance-parameter:root-instance-title}
log-url = $${monitor-publish-parameters:monitor-base-url}/private/notifier/ log-url = $${monitor-publish-parameters:monitor-base-url}/private/notifier/
status-item-directory = $${directory:notifier-status-items} status-item-directory = $${directory:notifier-status-items}
[pbs-resilient-status-feed] [pbs-resilient-status-feed]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
command-line = ${buildout:directory}/bin/generatefeed --output $${:feed-path} --status-item-path $${pbs:status-item-directory} --title "Status feed for $${instance-info-parameters:root-name}-PBS" --link $${pbs:log-url} command-line = ${buildout:directory}/bin/generatefeed --output $${:feed-path} --status-item-path $${pbs:status-item-directory} --title "Status feed for $${monitor-instance-parameter:root-instance-title}-PBS" --link $${pbs:log-url}
feed-path = $${directory:monitor-resilient}/pbs-status-rss feed-path = $${directory:monitor-resilient}/pbs-status-rss
wrapper-path = $${rootdirectory:bin}/resilient-genstatusrss.py wrapper-path = $${rootdirectory:bin}/resilient-genstatusrss.py
......
...@@ -135,7 +135,7 @@ command = ${buildout:bin-directory}/pubsubserver --callbacks $${directory:notifi ...@@ -135,7 +135,7 @@ command = ${buildout:bin-directory}/pubsubserver --callbacks $${directory:notifi
notifier-binary = ${buildout:bin-directory}/pubsubnotifier notifier-binary = ${buildout:bin-directory}/pubsubnotifier
host = $${slap-network-information:global-ipv6} host = $${slap-network-information:global-ipv6}
port = $${notifier-port:port} port = $${notifier-port:port}
instance-root-name = $${instance-info-parameters:root-name} instance-root-name = $${monitor-instance-parameter:root-instance-title}
log-url = $${monitor-publish-parameters:monitor-base-url}/resilient/notifier-status-rss log-url = $${monitor-publish-parameters:monitor-base-url}/resilient/notifier-status-rss
status-item-directory = $${directory:notifier-status-items} status-item-directory = $${directory:notifier-status-items}
context = context =
......
...@@ -131,8 +131,8 @@ pyparsing = 2.2.0 ...@@ -131,8 +131,8 @@ pyparsing = 2.2.0
pytz = 2016.10 pytz = 2016.10
requests = 2.13.0 requests = 2.13.0
six = 1.10.0 six = 1.10.0
slapos.cookbook = 1.0.59 slapos.cookbook = 1.0.62
slapos.core = 1.4.4 slapos.core = 1.4.7
slapos.extension.strip = 0.4 slapos.extension.strip = 0.4
slapos.libnetworkcache = 0.15 slapos.libnetworkcache = 0.15
slapos.rebootstrap = 4.1 slapos.rebootstrap = 4.1
...@@ -144,7 +144,7 @@ xml-marshaller = 0.9.7 ...@@ -144,7 +144,7 @@ xml-marshaller = 0.9.7
paramiko = 2.1.3 paramiko = 2.1.3
# Required by: # Required by:
# slapos.core==1.4.4 # slapos.core==1.4.7
Flask = 0.12 Flask = 0.12
# Required by: # Required by:
...@@ -160,11 +160,11 @@ functools32 = 3.2.3.post2 ...@@ -160,11 +160,11 @@ functools32 = 3.2.3.post2
ipaddress = 1.0.18 ipaddress = 1.0.18
# Required by: # Required by:
# slapos.cookbook==1.0.59 # slapos.cookbook==1.0.62
jsonschema = 2.6.0 jsonschema = 2.6.0
# Required by: # Required by:
# slapos.core==1.4.4 # slapos.core==1.4.7
# XXX 'slapos node format' raises an exception with netifaces 0.10.5. # XXX 'slapos node format' raises an exception with netifaces 0.10.5.
netifaces = 0.10.4 netifaces = 0.10.4
...@@ -177,15 +177,15 @@ packaging = 16.8 ...@@ -177,15 +177,15 @@ packaging = 16.8
pycparser = 2.17 pycparser = 2.17
# Required by: # Required by:
# slapos.core==1.4.4 # slapos.core==1.4.7
supervisor = 3.3.3 supervisor = 3.3.3
# Required by: # Required by:
# slapos.core==1.4.4 # slapos.core==1.4.7
uritemplate = 3.0.0 uritemplate = 3.0.0
# Required by: # Required by:
# slapos.core==1.4.4 # slapos.core==1.4.7
zope.interface = 4.3.3 zope.interface = 4.3.3
[networkcache] [networkcache]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment