diff --git a/CHANGES.rst b/CHANGES.rst index 21d8a60f9dd632ec9c3eb24d56449ea3357b34da..28f9b02a4d359415269e2f1313bf2d7b31bb1128 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,11 @@ Changes ======= +1.0.62 (2018-04-10) +------------------- + +* promise.plugin: new recipe for python promises plugin script generation + 1.0.59 (2018-03-15) ------------------- * librecipe.execute: fix convert process arguments to string formatting. diff --git a/component/apache/buildout.cfg b/component/apache/buildout.cfg index 437b9fc87d02d1fe4d18c7a1cb5ee933ed7235dc..c5f8a10a531a8a9cbbf986194a925ea25623ef05 100644 --- a/component/apache/buildout.cfg +++ b/component/apache/buildout.cfg @@ -34,9 +34,9 @@ md5sum = 2202b18f269ad606d70e1864857ed93c [apache] # inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/ recipe = slapos.recipe.cmmi -version = 2.4.29 +version = 2.4.33 url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2 -md5sum = 0c599404ef6b69eee95bcd9fcd094407 +md5sum = 6ef469d3f16fffeb688bc6e0346823e5 pre-configure = cp -ar ${apr:location}/apr-${apr:version} srclib/apr/ && cp -ar ${apr-util:location}/apr-util-${apr-util:version} srclib/apr-util diff --git a/component/egg-patch/Acquisition/aq_dynamic.patch b/component/egg-patch/Acquisition/aq_dynamic.patch index deb801fcfe0cf5b727954be127aed7d1c5cf11bf..fec1c900e162cc70ff1368d51e4e476df4f48435 100644 --- a/component/egg-patch/Acquisition/aq_dynamic.patch +++ b/component/egg-patch/Acquisition/aq_dynamic.patch @@ -1,7 +1,7 @@ -diff -uNr Acquisition-2.13.8/src/Acquisition/_Acquisition.c Acquisition-2.13.8nxd001/src/Acquisition/_Acquisition.c ---- Acquisition-2.13.8/src/Acquisition/_Acquisition.c 2011-06-11 17:19:14.000000000 +0200 -+++ Acquisition-2.13.8nxd001/src/Acquisition/_Acquisition.c 2013-10-31 16:24:55.665085888 +0100 -@@ -448,6 +448,64 @@ +diff -uNr Acquisition-2.13.12.orig/src/Acquisition/_Acquisition.c Acquisition-2.13.12/src/Acquisition/_Acquisition.c +--- Acquisition-2.13.12.orig/src/Acquisition/_Acquisition.c 2017-12-01 12:01:34.000000000 +0100 ++++ Acquisition-2.13.12/src/Acquisition/_Acquisition.c 2018-04-09 17:10:15.394836944 +0200 +@@ -449,6 +449,64 @@ } static PyObject * @@ -66,7 +66,7 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/_Acquisition.c Acquisition-2.13.8nx Wrapper_acquire(Wrapper *self, PyObject *oname, PyObject *filter, PyObject *extra, PyObject *orig, int explicit, int containment); -@@ -545,8 +603,8 @@ +@@ -589,8 +647,8 @@ Py_XDECREF(r); Py_XDECREF(v); Py_XDECREF(tb); r=NULL; } @@ -77,7 +77,7 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/_Acquisition.c Acquisition-2.13.8nx { if (r==Acquired) { -@@ -670,7 +728,7 @@ +@@ -714,7 +772,7 @@ Py_XDECREF(r); Py_XDECREF(v); Py_XDECREF(tb); r=NULL; @@ -86,7 +86,7 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/_Acquisition.c Acquisition-2.13.8nx if (r == Acquired) { Py_DECREF(r); } -@@ -707,7 +765,7 @@ +@@ -751,7 +809,7 @@ Wrapper_getattro(Wrapper *self, PyObject *oname) { if (self->obj || self->container) @@ -95,18 +95,18 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/_Acquisition.c Acquisition-2.13.8nx /* Maybe we are getting initialized? */ return Py_FindAttr(OBJECT(self),oname); -@@ -724,7 +782,7 @@ - return Py_FindAttr(OBJECT(self),oname); +@@ -776,7 +834,7 @@ + result = Py_FindAttr(OBJECT(self),oname); - if (self->obj || self->container) -- return Wrapper_findattr(self, oname, NULL, NULL, NULL, 1, 0, 0, 0); -+ return Wrapper_findattr(self, oname, NULL, NULL, OBJECT(self), 1, 0, 0, 0); + else if (self->obj || self->container) +- result = Wrapper_findattr(self, oname, NULL, NULL, NULL, 1, 0, 0, 0); ++ result = Wrapper_findattr(self, oname, NULL, NULL, OBJECT(self), 1, 0, 0, 0); /* Maybe we are getting initialized? */ - return Py_FindAttr(OBJECT(self),oname); -diff -uNr Acquisition-2.13.8/src/Acquisition/test_dynamic_acquisition.py Acquisition-2.13.8nxd001/src/Acquisition/test_dynamic_acquisition.py ---- Acquisition-2.13.8/src/Acquisition/test_dynamic_acquisition.py 1970-01-01 01:00:00.000000000 +0100 -+++ Acquisition-2.13.8nxd001/src/Acquisition/test_dynamic_acquisition.py 2013-10-31 16:24:55.665085888 +0100 + else result = Py_FindAttr(OBJECT(self),oname); +diff -uNr Acquisition-2.13.12.orig/src/Acquisition/test_dynamic_acquisition.py Acquisition-2.13.12/src/Acquisition/test_dynamic_acquisition.py +--- Acquisition-2.13.12.orig/src/Acquisition/test_dynamic_acquisition.py 1970-01-01 01:00:00.000000000 +0100 ++++ Acquisition-2.13.12/src/Acquisition/test_dynamic_acquisition.py 2018-04-09 17:07:34.863985305 +0200 @@ -0,0 +1,160 @@ +############################################################################## +# @@ -268,10 +268,10 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/test_dynamic_acquisition.py Acquisi + >>> assert not b.a.aq_inContextOf('somestring') +''' + -diff -uNr Acquisition-2.13.8/src/Acquisition/tests.py Acquisition-2.13.8nxd001/src/Acquisition/tests.py ---- Acquisition-2.13.8/src/Acquisition/tests.py 2011-06-11 17:09:38.000000000 +0200 -+++ Acquisition-2.13.8nxd001/src/Acquisition/tests.py 2013-10-31 16:24:55.669085888 +0100 -@@ -2552,6 +2552,7 @@ +diff -uNr Acquisition-2.13.12.orig/src/Acquisition/tests.py Acquisition-2.13.12/src/Acquisition/tests.py +--- Acquisition-2.13.12.orig/src/Acquisition/tests.py 2017-12-01 12:01:34.000000000 +0100 ++++ Acquisition-2.13.12/src/Acquisition/tests.py 2018-04-09 17:07:34.867985476 +0200 +@@ -2588,6 +2588,7 @@ def test_suite(): return unittest.TestSuite(( DocTestSuite(), @@ -279,10 +279,10 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/tests.py Acquisition-2.13.8nxd001/s DocFileSuite('README.txt', package='Acquisition'), unittest.makeSuite(TestParent), unittest.makeSuite(TestAcquire), -diff -uNr Acquisition-2.13.8/src/Acquisition.egg-info/SOURCES.txt Acquisition-2.13.8nxd001/src/Acquisition.egg-info/SOURCES.txt ---- Acquisition-2.13.8/src/Acquisition.egg-info/SOURCES.txt 2011-06-11 17:21:18.000000000 +0200 -+++ Acquisition-2.13.8nxd001/src/Acquisition.egg-info/SOURCES.txt 2013-10-31 16:24:55.669085888 +0100 -@@ -15,6 +15,7 @@ +diff -uNr Acquisition-2.13.12.orig/src/Acquisition.egg-info/SOURCES.txt Acquisition-2.13.12/src/Acquisition.egg-info/SOURCES.txt +--- Acquisition-2.13.12.orig/src/Acquisition.egg-info/SOURCES.txt 2017-12-01 12:08:29.000000000 +0100 ++++ Acquisition-2.13.12/src/Acquisition.egg-info/SOURCES.txt 2018-04-09 17:07:34.867985476 +0200 +@@ -10,6 +10,7 @@ src/Acquisition/_Acquisition.c src/Acquisition/__init__.py src/Acquisition/interfaces.py diff --git a/component/egg-patch/ZEO4/TCP_NODELAY.patch b/component/egg-patch/ZEO4/TCP_NODELAY.patch new file mode 100644 index 0000000000000000000000000000000000000000..31505347553822ac49e68fa683c72404f5973b8d --- /dev/null +++ b/component/egg-patch/ZEO4/TCP_NODELAY.patch @@ -0,0 +1,39 @@ +From 8b31ccec54584a287cc61501948283d7d6ee7073 Mon Sep 17 00:00:00 2001 +From: Julien Muchembled <jm@nexedi.com> +Date: Mon, 26 Mar 2018 20:39:07 +0200 +Subject: [PATCH] Enable TCP_NODELAY for inet(6) sockets + +See commit 3d886d426243655b9f5a2528636e42b5c7662c19. +--- + src/ZEO/zrpc/client.py | 2 ++ + src/ZEO/zrpc/server.py | 1 + + 2 files changed, 3 insertions(+) + +diff --git a/src/ZEO/zrpc/client.py b/src/ZEO/zrpc/client.py +index 32a7a877..669f5962 100644 +--- a/src/ZEO/zrpc/client.py ++++ b/src/ZEO/zrpc/client.py +@@ -568,6 +568,8 @@ def __init__(self, domain, addr, mgr, client): + self.close() + return + self.sock.setblocking(0) ++ if domain in (socket.AF_INET, socket.AF_INET6): ++ self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + self.state = "opened" + + def connect_procedure(self): +diff --git a/src/ZEO/zrpc/server.py b/src/ZEO/zrpc/server.py +index b83cc004..af91e3e4 100644 +--- a/src/ZEO/zrpc/server.py ++++ b/src/ZEO/zrpc/server.py +@@ -66,6 +66,7 @@ def _open_socket(self): + socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, True) + else: + self.create_socket(socket.AF_INET, socket.SOCK_STREAM) ++ self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + else: + self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.set_reuse_addr() +-- +2.14.1 + diff --git a/component/gcc/buildout.cfg b/component/gcc/buildout.cfg index 2e711fe2e440176412ca20d8830a04a51c30d4c3..4971d7be0827de395b4641a7fdc1dfe6a18cf917 100644 --- a/component/gcc/buildout.cfg +++ b/component/gcc/buildout.cfg @@ -14,8 +14,8 @@ parts = [gcc-common] recipe = slapos.recipe.cmmi -url = http://ftp.gnu.org/gnu/gcc/gcc-5.4.0/gcc-5.4.0.tar.bz2 -md5sum = 4c626ac2a83ef30dfb9260e6f59c2b30 +url = http://ftp.gnu.org/gnu/gcc/gcc-5.5.0/gcc-5.5.0.tar.gz +md5sum = 781bc0195edeb0ceaace8428f63ae63d # make install does not work when several core are used make-targets = install -j1 diff --git a/component/golang/buildout.cfg b/component/golang/buildout.cfg index 204841ea506c35b391ce776ddc43a935edeae156..129fa6ed4a13b7980fcdd4322af93016054ccc1c 100644 --- a/component/golang/buildout.cfg +++ b/component/golang/buildout.cfg @@ -3,6 +3,7 @@ extends = ../findutils/buildout.cfg ../git/buildout.cfg + ../pkgconfig/buildout.cfg parts = gowork @@ -64,7 +65,15 @@ environment-extra = # github.com/pkg/profile \ # golang.org/x/perf/cmd/benchstat # -# it is possible to specify Go build flags used for compilation e.g. this way: +# For Cgo support pkg-config is made pre-available by gowork, and users +# should list paths where to search for pkg-config files, e.g. this way: +# +# [gowork] +# cpkgpath = +# ${sqlite3:location}/lib/pkgconfig +# ${zlib:location}/lib/pkgconfig +# +# It is also possible to specify Go build flags used for compilation e.g. this way: # # [gowork] # buildflags = -race @@ -80,6 +89,9 @@ golang = ${golang19:location} # no special build flags by default buildflags = +# empty pkg-config path by default +cpkgpath = + # everything is done by dependent parts recipe = plone.recipe.command command = : @@ -93,7 +105,7 @@ recipe = slapos.recipe.template url = ${:_profile_base_location_}/goenv.sh.in output = ${gowork:directory}/env.sh depends = ${gowork.mkdir:recipe} -md5sum = a9a265135931b3da53f4392870748264 +md5sum = 7a067a3974c446c3eaa0e82818ba1adb [gowork.mkdir] # NOTE do not use slapos.cookbook:mkdirectory here - if anything in software (not instance) diff --git a/component/golang/goenv.sh.in b/component/golang/goenv.sh.in index 0a409372da6426da63845c2759f0825f1c460318..4c0e98b4b8ed0ed1670ae55a87165a225b8e2e09 100644 --- a/component/golang/goenv.sh.in +++ b/component/golang/goenv.sh.in @@ -3,13 +3,16 @@ # ---- 8< ---- (buildout substitution here) # PATH so that go & friends work out of the box -export PATH=${gowork:golang}/bin:${git:location}/bin:${buildout:bin-directory}:$PATH +export PATH=${gowork:golang}/bin:${git:location}/bin:${pkgconfig:location}/bin:${buildout:bin-directory}:$PATH X=${gowork:directory} + +export PKG_CONFIG_PATH=$(echo -n "${gowork:cpkgpath}" |tr '\n' ':'):$PKG_CONFIG_PATH # ---- 8< ---- export GOPATH=$X:$GOPATH export PATH=$X/bin:$PATH export PS1="(`basename $X`) $PS1" -# strip trailing : from $GOPATH +# strip trailing : from $GOPATH, $PKG_CONFIG_PATH GOPATH=$${GOPATH%:} +PKG_CONFIG_PATH=$${PKG_CONFIG_PATH%:} diff --git a/component/groonga/buildout.cfg b/component/groonga/buildout.cfg index de4639d07273c93d1dd13d80cd8682736951bf25..5f5b99cce34e277693368431788a8ffd03454097 100644 --- a/component/groonga/buildout.cfg +++ b/component/groonga/buildout.cfg @@ -14,8 +14,8 @@ extends = [groonga] recipe = slapos.recipe.cmmi -url = http://packages.groonga.org/source/groonga/groonga-7.0.5.tar.gz -md5sum = d42dd98c5272ed2b9ab50ded54eb818b +url = http://packages.groonga.org/source/groonga/groonga-8.0.0.tar.gz +md5sum = f4641e68a5301e641399f2d112a494dd # temporary patch to respect more tokens in natural language mode. patches = ${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b diff --git a/component/icu/buildout.cfg b/component/icu/buildout.cfg index 12a454af0ae0a579154773faefc5481e1829cc75..1ef78bb01142833a341b60af39ef52d728bdad31 100644 --- a/component/icu/buildout.cfg +++ b/component/icu/buildout.cfg @@ -14,6 +14,8 @@ configure-options = --prefix=${:location} --disable-static --enable-rpath +patches = + ${:_profile_base_location_}/fix-glibc2.26-ftbfs.patch#f1622be16964029fc66a70b8f9e1693c [icu4c-slaposgcc] # need for onlyoffice-core diff --git a/component/icu/fix-glibc2.26-ftbfs.patch b/component/icu/fix-glibc2.26-ftbfs.patch new file mode 100644 index 0000000000000000000000000000000000000000..6a50e56d9496ef2ae75aa083b2d996cdf31e0b78 --- /dev/null +++ b/component/icu/fix-glibc2.26-ftbfs.patch @@ -0,0 +1,15 @@ +--- source/i18n/digitlst.cpp 2018-03-29 10:59:45.706746334 +0900 ++++ source/i18n/digitlst.cpp 2018-03-29 11:01:15.933534983 +0900 +@@ -61,11 +61,7 @@ + #endif + + #if U_USE_STRTOD_L +-# if U_PLATFORM_USES_ONLY_WIN32_API || U_PLATFORM == U_PF_CYGWIN +-# include <locale.h> +-# else +-# include <xlocale.h> +-# endif ++# include <locale.h> + #endif + + // *************************************************************************** diff --git a/component/mariadb/1a910088ec55c61434029d8e1cc3c6192ac508d7.diff b/component/mariadb/1a910088ec55c61434029d8e1cc3c6192ac508d7.diff deleted file mode 100644 index 96be511ef52b1d288a13950b81b1cf70727c8716..0000000000000000000000000000000000000000 --- a/component/mariadb/1a910088ec55c61434029d8e1cc3c6192ac508d7.diff +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/mrn_mysql_compat.h b/mrn_mysql_compat.h -index d2b227ff..337de968 100644 ---- a/mrn_mysql_compat.h -+++ b/mrn_mysql_compat.h -@@ -362,6 +362,7 @@ - - #if defined(MRN_MARIADB_P) && \ - ((MYSQL_VERSION_ID >= 100207) || \ -+ ((MYSQL_VERSION_ID >= 100126) && (MYSQL_VERSION_ID < 100200)) || \ - ((MYSQL_VERSION_ID >= 50557) && (MYSQL_VERSION_ID < 100000))) - # define mrn_create_partition_name(out, \ - out_length, \ diff --git a/component/mariadb/buildout.cfg b/component/mariadb/buildout.cfg index e8efca31aed59435151c895854290c47efc4083c..90ccc806602db0847c3f306f0fa06c1c5d08f6d0 100644 --- a/component/mariadb/buildout.cfg +++ b/component/mariadb/buildout.cfg @@ -28,8 +28,8 @@ parts = [mariadb] recipe = slapos.recipe.cmmi url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve -version = 10.1.31 -md5sum = 14ab0398c019eb531bc29f2c437ccb51 +version = 10.1.32 +md5sum = 389ce891cf00957748ba98b09f433c14 patch-options = -p0 patches = ${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3 @@ -84,8 +84,8 @@ post-install = # mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users. # http://mroonga.github.com/ recipe = slapos.recipe.cmmi -url = http://packages.groonga.org/source/mroonga/mroonga-7.05.tar.gz -md5sum = d289667a60a6aa78d8bc276ab61b2996 +url = http://packages.groonga.org/source/mroonga/mroonga-8.00.tar.gz +md5sum = 8b906705be5fc14785207ca6752b96e9 pre-configure = set -e rm -rf fake_mariadb_source mkdir -p fake_mariadb_source @@ -106,7 +106,6 @@ configure-options = patch-options = -p1 patches = ${:_profile_base_location_}/mroonga_boolean.patch#c818568fe35ca6a4298f18e575d962a0 - ${:_profile_base_location_}/1a910088ec55c61434029d8e1cc3c6192ac508d7.diff#a8b8df4dd870a1a269dfd9dc6941467b pre-build = sed -i -e "s,${mariadb:location}/include,$(pwd)/fake_mariadb_source/include,g" Makefile */Makefile environment = diff --git a/component/mariadb/mariarocks.cfg b/component/mariadb/mariarocks.cfg index 306bc40925d21a3a766bbfccdb27214cedd1a102..49a619eaca9262a77879e05e58347fa9ae1b8525 100644 --- a/component/mariadb/mariarocks.cfg +++ b/component/mariadb/mariarocks.cfg @@ -1,8 +1,8 @@ # Do not extend any file that touch buildout:parts. [mariadb] -version = 10.2.13 -md5sum = 20c61bd4059ba287e54cfb2862bae81d +version = 10.2.14 +md5sum = d98cce6f3c0e2971afa061fc67183b91 stable-patches = configure-options += -DCMAKE_C_COMPILER=${gcc:location}/bin/gcc diff --git a/component/noVNC/buildout.cfg b/component/noVNC/buildout.cfg index 213f04ebb4e3137480fe16889ea4bcb43494e68d..2ff4907b40926cfe8788062b842a3ac7780b6e19 100644 --- a/component/noVNC/buildout.cfg +++ b/component/noVNC/buildout.cfg @@ -5,7 +5,7 @@ parts = [noVNC] recipe = hexagonit.recipe.download ignore-existing = true -# version-0.6.1 release from 04 Jul 2016 -url = https://github.com/kanaka/noVNC/archive/v0.6.1.tar.gz -md5sum = d153c6aa69a9178081768fecbace1932 +# version-1.0.0 release on Feb 22, 2018 +url = https://github.com/novnc/noVNC/archive/v1.0.0.tar.gz +md5sum = d63c2944abd12fecec6ec504e82f27c8 strip-top-level-dir = true diff --git a/component/onlyoffice-core/buildout.cfg b/component/onlyoffice-core/buildout.cfg index 50c7b136fa226c1241b1e37b97ade078fd3eca93..2a031fb8fe5ba011a3ba468c89dec1d0a6fc27db 100644 --- a/component/onlyoffice-core/buildout.cfg +++ b/component/onlyoffice-core/buildout.cfg @@ -1,5 +1,6 @@ [buildout] extends = + ../binutils/buildout.cfg ../gcc/buildout.cfg ../libxml2/buildout.cfg ../zlib/buildout.cfg @@ -15,12 +16,12 @@ recipe = slapos.recipe.cmmi location = ${buildout:parts-directory}/${:_buildout_section_name_} # This url contains the hash provided by the DocumentServer core submodule hash. # https://github.com/ONLYOFFICE/DocumentServer/ -url = https://lab.nexedi.com/bk/onlyoffice_core/repository/archive.tar.bz2?ref=b051e75b179b3599c09937668fbbd2d7e2c50683 -md5sum = b2713373d687dd1c7121c286fa156626 +url = https://lab.nexedi.com/bk/onlyoffice_core/repository/archive.tar.bz2?ref=8a40eb47bd80a40ecde14c223525b21852d2fc9f +md5sum = 9cd1cd731202511e475971eee58ba7b6 configure-command = true make-targets = lib bin environment = - PATH=${gcc:location}/bin:${qt5-qmake:location}/bin:%(PATH)s + PATH=${binutils:location}/bin:${gcc:location}/bin:${qt5-qmake:location}/bin:%(PATH)s CXXFLAGS=-I${libxml2:location}/include -I${zlib:location}/include -I${icu4c-slaposgcc:location}/include -I${boost-lib:location}/include -Wno-comment -Wno-deprecated-declarations -Wno-endif-labels -Wno-parentheses -Wno-reorder -Wno-sign-compare -Wno-switch -Wno-unknown-pragmas -Wno-unused LDFLAGS=-L${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib -L${gcc:location}/lib64 -Wl,-rpath=${gcc:location}/lib64 -L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${icu4c-slaposgcc:location}/lib -Wl,-rpath=${icu4c-slaposgcc:location}/lib -L${boost-lib:location}/lib -Wl,-rpath=${boost-lib:location}/lib -Wl,-rpath=${:location}/lib post-install = diff --git a/component/openssl/buildout.cfg b/component/openssl/buildout.cfg index 0c9c8bbd2ededd8536e115cf77cbdbcad8d6f466..6cbe2b32631786651115fd085198735e249390a2 100644 --- a/component/openssl/buildout.cfg +++ b/component/openssl/buildout.cfg @@ -16,8 +16,8 @@ parts = [openssl] recipe = slapos.recipe.cmmi -url = https://www.openssl.org/source/openssl-1.0.2n.tar.gz -md5sum = 13bdc1b1d1ff39b6fd42a255e74676a4 +url = https://www.openssl.org/source/openssl-1.0.2o.tar.gz +md5sum = 44279b8557c3247cbe324e2322ecd114 location = ${buildout:parts-directory}/${:_buildout_section_name_} # 'prefix' option to override --openssldir/--prefix (which is useful # when combined with INSTALL_PREFIX). Used by slapos.package.git/obs diff --git a/component/qemu-kvm/buildout.cfg b/component/qemu-kvm/buildout.cfg index 6d490e427dcb3582f999412d11a66c1a57983521..791c61eaf8b894d7ca74d75f42fc77a1ce1f7d66 100644 --- a/component/qemu-kvm/buildout.cfg +++ b/component/qemu-kvm/buildout.cfg @@ -64,7 +64,6 @@ md5sum = 096c1c18b44c269808bd815d58c53c8f [debian-amd64-jessie-netinst.iso] <= debian-amd64-netinst-base -release = archive version = 8.10.0 md5sum = 19dcfc381bd3e609c6056216d203f5bc diff --git a/component/sqlite3/buildout.cfg b/component/sqlite3/buildout.cfg index 199b21aa1cead899a08150c83e8abc2e572e2f1e..224aff3f73a56ee96e2b47f9d181a8076678a780 100644 --- a/component/sqlite3/buildout.cfg +++ b/component/sqlite3/buildout.cfg @@ -5,8 +5,8 @@ parts = [sqlite3] recipe = slapos.recipe.cmmi -url = http://sqlite.org/2017/sqlite-autoconf-3190000.tar.gz -md5sum = 2426883b4dd3a9fd6aeb28a16b81a72b +url = https://sqlite.org/2018/sqlite-autoconf-3220000.tar.gz +md5sum = 96b5648d542e8afa6ab7ffb8db8ddc3d configure-options = --disable-static --enable-readline diff --git a/component/vm-img/debian.cfg b/component/vm-img/debian.cfg index f7ba8c8418cda8948358d30a619b7bcfebdeaeb4..d769d8d4bdfcfa353207586844a588ff2be32fb6 100644 --- a/component/vm-img/debian.cfg +++ b/component/vm-img/debian.cfg @@ -22,7 +22,7 @@ dist = ${vm-debian:dists} [vm-debian] recipe = slapos.recipe.build:vm.install-debian environment = vm-install-environment -dists = debian-jessie +dists = debian-stretch size = 1Gi late-command = # rdnssd causes too much trouble with recent QEMU, because the latter acts as @@ -55,18 +55,22 @@ preseed.recommends = false preseed.tasks = [debian-squeeze] -<= debian-jessie +<= debian-stretch x86_64.iso = debian-amd64-squeeze-netinst.iso [debian-wheezy] -<= debian-jessie +<= debian-stretch x86_64.iso = debian-amd64-wheezy-netinst.iso [debian-jessie] +<= debian-stretch +x86_64.iso = debian-amd64-jessie-netinst.iso + +[debian-stretch] x86_64.iso = debian-amd64-netinst.iso x86_64.kernel = install.amd/vmlinuz x86_64.initrd = install.amd/initrd.gz -[debian-stretch] -<= debian-jessie +[debian-buster] +<= debian-stretch x86_64.iso = debian-amd64-testing-netinst.iso diff --git a/setup.py b/setup.py index f8292c78f116f2c93f5336175870b4c770422ead..a87bf58fe83fa309cfd957a3dc5b7b99bb42744f 100755 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ from setuptools import setup, find_packages import glob import os -version = '1.0.60.dev0' +version = '1.0.62' name = 'slapos.cookbook' long_description = open("README.rst").read() + "\n" + \ open("CHANGES.rst").read() + "\n" @@ -99,7 +99,6 @@ setup(name=name, 'dropbear.add_authorized_key = slapos.recipe.dropbear:AddAuthorizedKey', 'dropbear.client = slapos.recipe.dropbear:Client', 'duplicity = slapos.recipe.duplicity:Recipe', - 'egg_test = slapos.recipe.erp5_test:EggTestRecipe', 'equeue = slapos.recipe.equeue:Recipe', 'erp5.promise = slapos.recipe.erp5_promise:Recipe', 'erp5.test = slapos.recipe.erp5_test:Recipe', @@ -151,6 +150,7 @@ setup(name=name, 'postgres.export = slapos.recipe.postgres.backup:ExportRecipe', 'postgres.import = slapos.recipe.postgres.backup:ImportRecipe', 'proactive = slapos.recipe.proactive:Recipe', + 'promise.plugin= slapos.recipe.promise_plugin:Recipe', 'publish = slapos.recipe.publish:Recipe', 'publish.serialised = slapos.recipe.publish:Serialised', 'publish-early = slapos.recipe.publish_early:Recipe', diff --git a/slapos/recipe/erp5_test/__init__.py b/slapos/recipe/erp5_test/__init__.py index 76cf4b0e647a562938acfe993f1eae28df3a5c95..a965ca774dbc6d2914e556a0f0854dd8b827c91c 100644 --- a/slapos/recipe/erp5_test/__init__.py +++ b/slapos/recipe/erp5_test/__init__.py @@ -113,26 +113,3 @@ class CloudoooRecipe(GenericBaseRecipe): return path_list -class EggTestRecipe(GenericBaseRecipe): - """ - Recipe used to create wrapper used to run test suite (python setup.py test) - off a list of Python eggs. - """ - def install(self): - test_list = self.options['test-list'].strip().replace('\n', ',') - - common_dict = {} - if self.options.get('environment'): - environment_part = self.buildout.get(self.options['environment']) - if environment_part: - common_dict['environment'] = dict(environment_part) - - if 'prepend-path' in self.options: - common_dict['prepend_path'] = self.options['prepend-path'] - - return self.createPythonScript( - self.options['run-test-suite'], __name__ + '.test.runTestSuite', - ((self.options['run-test-suite-binary'], - "--source_code_path_list", test_list), - common_dict) - ) diff --git a/slapos/recipe/promise_plugin.py b/slapos/recipe/promise_plugin.py new file mode 100644 index 0000000000000000000000000000000000000000..0df9a392227e009976aa268d7fe0f3639992cb7f --- /dev/null +++ b/slapos/recipe/promise_plugin.py @@ -0,0 +1,135 @@ +############################################################################## +# +# Copyright (c) 2018 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## +import logging, os, sys +import zc.buildout.easy_install +from slapos.recipe.librecipe import GenericBaseRecipe + +script_template = '''# This script is auto generated by slapgrid, do not edit! +import sys +sys.path[0:0] = [ + %(path)s +] + +extra_config_dict = { + %(config)s +} + +# We want to cleanup all imported modules from slapos namespace, because +# they will conflict with slapos.core. +# In fact as slapos.grid is already imported, the promise cannot reimport +# his own slapos.grid from an updated sys.path. Then all module imports which +# are not in slapos.core will fail. +# +# call reload(slapos) only solve a part of problem because not all modules +# will be reloaded, and some new modules won't be added. +# The solution is to delete all cached 'slapos' modules as well as all cached +# 'pkg_resources' modules which is responsible of namespace declaration. +# They will be re-imported again using the updated sys.path +for module in sys.modules.keys(): + if 'slapos' in module or 'pkg_resources' in module: + del sys.modules[module] + +import slapos.grid.promise + +%(content)s +''' + +class Recipe(GenericBaseRecipe): + + _WORKING_SET_CACHE_NAME = "slapos.cookbook_pplugin_ws_cache" + + def __init__(self, buildout, name, options): + buildout_section = buildout['buildout'] + options['eggs-directory'] = buildout_section['eggs-directory'] + options['develop-eggs-directory'] = buildout_section['develop-eggs-directory'] + super(Recipe, self).__init__(buildout, name, options) + + def _get_cache_storage(self): + """Return a mapping where to store generated working sets. + from https://github.com/buildout/buildout/blob/master/zc.recipe.egg_/src/zc/recipe/egg/egg.py#L170 + """ + cache_storage = getattr( + self.buildout, + self._WORKING_SET_CACHE_NAME, + None + ) + if cache_storage is None: + cache_storage = {} + setattr( + self.buildout, + self._WORKING_SET_CACHE_NAME, + cache_storage + ) + return cache_storage + + def install(self): + develop_eggs_dir = self.options['develop-eggs-directory'] + eggs_dir = self.options['eggs-directory'] + egg_list = [ + egg.strip() + for egg in self.options['eggs'].split('\n') + if egg.strip() + ] + + cache_storage = self._get_cache_storage() + cache_key = ( + tuple(egg_list), + eggs_dir, + develop_eggs_dir, + ) + if cache_key not in cache_storage: + working_set = zc.buildout.easy_install.working_set( + egg_list, + [develop_eggs_dir, eggs_dir] + ) + cache_storage[cache_key] = working_set + else: + working_set = cache_storage[cache_key] + + content = self.options['content'].strip() + output = self.options['output'] + mode = self.options.get('mode', '0600') + path_list_string = "" + for dist in working_set: + path_list_string += ' "%s",\n' % dist.location + + content_string = '\n'.join([line.lstrip() for line in content.split('\n')]) + config_string = "" + for key in self.options: + if key.startswith('config-'): + config_string += " '%s': '%s',\n" % (key[7:], self.options[key]) + + option_dict = dict(path=path_list_string.strip(), + content=content_string, + config=config_string.strip()) + with open(output, 'w') as f: + f.write(script_template % option_dict) + + os.chmod(output, int(mode, 8)) + return (output,) + + update = install diff --git a/slapos/test/recipe/test_free_port.py b/slapos/test/recipe/test_free_port.py index 7693d9e02cd3bbe10d4a20e6fd7e4c7b42979971..e2cea348bf548c5032c5c42cd6960279b9ff3703 100644 --- a/slapos/test/recipe/test_free_port.py +++ b/slapos/test/recipe/test_free_port.py @@ -4,8 +4,6 @@ import unittest from mock import patch -from slapos.recipe import free_port - class SocketMock(): def __init__(self, *args, **kw): self.args = args @@ -28,31 +26,16 @@ class FreePortTest(unittest.TestCase): SocketMock.bind = SocketMock.close = SocketMock.nothing_happen def new_recipe(self, **kw): - buildout = { - 'buildout': { - 'bin-directory': '', - 'find-links': '', - 'allow-hosts': '', - 'develop-eggs-directory': '', - 'eggs-directory': '', - 'python': 'testpython', - 'installed': '.installed.cfg', - }, - 'testpython': { - 'executable': sys.executable, - }, - 'slap-connection': { - 'computer-id': '', - 'partition-id': '', - 'server-url': '', - 'software-release-url': '', - } - } + from slapos.recipe import free_port + from slapos.test.utils import makeRecipe options = { 'ip': '127.0.0.1', } options.update(kw) - return free_port.Recipe(buildout=buildout, name='free_port', options=options) + return makeRecipe( + free_port.Recipe, + options=options, + name='free_port') @useMock def test_ifNoBusyPortThenMinPortIsAlwaysReturned(self): diff --git a/slapos/test/recipe/test_generic_cloudooo.py b/slapos/test/recipe/test_generic_cloudooo.py index d171efcb2d9b4493323b5825591758977a0239ae..29ddf64fdf3a8ff030df3a6e452bc3cc49b33072 100644 --- a/slapos/test/recipe/test_generic_cloudooo.py +++ b/slapos/test/recipe/test_generic_cloudooo.py @@ -4,30 +4,15 @@ import unittest from tempfile import mkdtemp from shutil import rmtree -from slapos.recipe import generic_cloudooo class TestGenericCloudooo(unittest.TestCase): def new_recipe(self, options): - buildout = { - 'buildout': { - 'bin-directory': '', - 'find-links': '', - 'allow-hosts': '', - 'develop-eggs-directory': '', - 'eggs-directory': '', - 'python': 'testpython', - }, - 'testpython': { - 'executable': sys.executable, - }, - 'slap-connection': { - 'computer-id': '', - 'partition-id': '', - 'server-url': '', - 'software-release-url': '', - } - } - return generic_cloudooo.Recipe(buildout=buildout, name='generic_cloudooo', options=options) + from slapos.recipe import generic_cloudooo + from slapos.test.utils import makeRecipe + return makeRecipe( + generic_cloudooo.Recipe, + options=options, + name='generic_cloudooo') def setUp(self): self.test_dir = mkdtemp() diff --git a/slapos/test/recipe/test_pbs.py b/slapos/test/recipe/test_pbs.py index ff7c04f28d97240912d21b3202b9f673967aac8c..08327f8272b7726231c2fcead1dbf8adbecc042e 100644 --- a/slapos/test/recipe/test_pbs.py +++ b/slapos/test/recipe/test_pbs.py @@ -5,37 +5,19 @@ import sys import tempfile import unittest -from slapos.recipe import pbs - class PBSTest(unittest.TestCase): def new_recipe(self): - buildout = { - 'buildout': { - 'bin-directory': '', - 'find-links': '', - 'allow-hosts': '', - 'develop-eggs-directory': '', - 'eggs-directory': '', - 'python': 'testpython', - }, - 'testpython': { - 'executable': sys.executable, - }, - 'slap-connection': { - 'computer-id': '', - 'partition-id': '', - 'server-url': '', - 'software-release-url': '', - } - } - + from slapos.recipe import pbs + from slapos.test.utils import makeRecipe options = { - 'rdiffbackup-binary': '' - } - - return pbs.Recipe(buildout=buildout, name='pbs', options=options) + 'rdiffbackup-binary': '' + } + return makeRecipe( + pbs.Recipe, + options=options, + name='pbs') def test_push(self): recipe = self.new_recipe() diff --git a/slapos/test/recipe/test_re6stnet.py b/slapos/test/recipe/test_re6stnet.py index ca2b5e635f0352742236242244db59284e618af7..0c3ac919b4f486f407b12ac033d5c53f0cff3be0 100644 --- a/slapos/test/recipe/test_re6stnet.py +++ b/slapos/test/recipe/test_re6stnet.py @@ -6,8 +6,6 @@ import tempfile import unittest from slapos.slap.slap import NotFoundError, ConnectionError -from slapos.recipe import re6stnet - class Re6stnetTest(unittest.TestCase): @@ -47,31 +45,21 @@ class Re6stnetTest(unittest.TestCase): shutil.rmtree(path) def new_recipe(self): - buildout = { - 'buildout': { - 'bin-directory': '', - 'find-links': '', - 'allow-hosts': '', - 'develop-eggs-directory': '', - 'eggs-directory': '', - 'python': 'testpython', - }, - 'testpython': { - 'executable': sys.executable, - }, - 'slap-connection': { + from slapos.recipe import re6stnet + from slapos.test.utils import makeRecipe + return makeRecipe( + re6stnet.Recipe, + options=self.options, + slap_connection={ 'computer-id': 'comp-test', 'partition-id': 'slappart0', 'server-url': 'http://server.com', 'software-release-url': 'http://software.com', 'key-file': '/path/to/key', 'cert-file': '/path/to/cert' - } - } - - options = self.options + }, + name='re6stnet') - return re6stnet.Recipe(buildout=buildout, name='re6stnet', options=options) def checkWrapper(self, path): self.assertTrue(os.path.exists(path)) diff --git a/slapos/test/utils.py b/slapos/test/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d605bab9d326748881ef3858b81b1ed47c743b41 --- /dev/null +++ b/slapos/test/utils.py @@ -0,0 +1,62 @@ +"""Test helpers +""" +import sys +import os.path +from ConfigParser import ConfigParser + +import logging + +def makeRecipe(recipe_class, options, name='test', slap_connection=None): + """Instanciate a recipe of `recipe_class` with `options` with a buildout + mapping containing a python and an empty `slapos-connection` mapping, unless + provided as `slap_connection`. + + If running tests in a buildout folder, the test recipe will reuse the + `eggs-directory` and `develop-eggs-directory` from this buildout so that the + test recipe does not need to install eggs again when using working set. + To prevent test accidentally writing to the buildout's eggs repositories, we + set `newest` to false and `offline` to true in this case. + """ + buildout = { + 'buildout': { + 'bin-directory': '', + 'find-links': '', + 'allow-hosts': '', + 'develop-eggs-directory': '', + 'eggs-directory': '', + 'python': 'testpython', + }, + 'testpython': { + 'executable': sys.executable, + }, + 'slap-connection': { + 'computer-id': '', + 'partition-id': '', + 'server-url': '', + 'software-release-url': '', + } + } + if slap_connection is not None: + buildout['slap-connection'] = slap_connection + + # are we in buildout folder ? + # the usual layout is + # ${buildout:directory}/parts/slapos-repository/slapos/test/utils.py , so try + # to find a buildout relative to this file. + buildout_cfg = os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', 'buildout.cfg') + if os.path.exists(buildout_cfg): + parser = ConfigParser() + parser.readfp(open(buildout_cfg)) + eggs_directory = parser.get('buildout', 'eggs-directory') + develop_eggs_directory = parser.get('buildout', 'develop-eggs-directory') + logging.getLogger(__name__).info( + 'Using eggs-directory (%s) and develop-eggs-directory (%s) from buildout at %s', + eggs_directory, + develop_eggs_directory, + buildout_cfg) + buildout['buildout']['eggs-directory'] = eggs_directory + buildout['buildout']['develop-eggs-directory'] = develop_eggs_directory + buildout['buildout']['newest'] = False + buildout['buildout']['offline'] = True + return recipe_class(buildout=buildout, name=name, options=options) + diff --git a/software/agent/software.cfg b/software/agent/software.cfg index b2fea6b00cd1fa3d138f51a800464f2495a98cbe..3d174f659b0a88428622abab6927c5c6b9fb0952 100644 --- a/software/agent/software.cfg +++ b/software/agent/software.cfg @@ -51,7 +51,7 @@ pycrypto = 2.6.1 pycurl = 7.43.0 slapos.recipe.download = 1.0 slapos.recipe.template = 4.3 -slapos.toolbox = 0.74 +slapos.toolbox = 0.76 smmap = 0.9.0 # Required by: diff --git a/software/apache-frontend/buildout.hash.cfg b/software/apache-frontend/buildout.hash.cfg index 82fc22ef348a6998c83c3f829ad718f7c9227cd6..81a5090972790756a5d5331b856474bcc4bcfa89 100644 --- a/software/apache-frontend/buildout.hash.cfg +++ b/software/apache-frontend/buildout.hash.cfg @@ -55,7 +55,7 @@ md5sum = d98a01182f38868612948c87d5231428 [template-default-slave-virtualhost] filename = templates/default-virtualhost.conf.in -md5sum = 5344bff68a3f7dead633a35771ad1a19 +md5sum = 7f38084af107034bedefba971abe165c [template-cached-slave-virtualhost] filename = templates/cached-virtualhost.conf.in diff --git a/software/apache-frontend/software.cfg b/software/apache-frontend/software.cfg index 8c84c680bf2903058ce2ee05f3e77cd5796c6698..3bb800148629aae39957117a777d8ec32c780447 100644 --- a/software/apache-frontend/software.cfg +++ b/software/apache-frontend/software.cfg @@ -11,7 +11,7 @@ plone.recipe.command = 1.1 pycrypto = 2.6.1 rdiff-backup = 1.0.5+SlapOSPatched001 slapos.recipe.template = 4.3 -slapos.toolbox = 0.74 +slapos.toolbox = 0.76 smmap = 0.9.0 numpy = 1.11.2 pyasn1 = 0.2.3 diff --git a/software/build-rina/runTestSuite.in b/software/build-rina/runTestSuite.in index e04ba7e0d4387b6ab97860412742321baed6e58b..89ffec9ab21077b2af24a32379663db300d18aa7 100644 --- a/software/build-rina/runTestSuite.in +++ b/software/build-rina/runTestSuite.in @@ -69,7 +69,7 @@ def main(): test_title = args.test_suite_title or args.test_suite if args.master_url: - tool = taskdistribution.TaskDistributionTool(args.master_url) + tool = taskdistribution.TaskDistributor(args.master_url) test_result = tool.createTestResult(args.revision, dist_list, args.test_node_title, diff --git a/software/build-rina/software.cfg b/software/build-rina/software.cfg index 18f04e0f59f81d9a8bed3394a3d75ab3029b094b..4356f950365902243c83e5cf2c8c02787e280d19 100644 --- a/software/build-rina/software.cfg +++ b/software/build-rina/software.cfg @@ -66,6 +66,7 @@ sparse-checkout = /playbook/roles/rina [vm-debian] # building a generic Debian kernel uses a lot of space size = 16Gi +dists = debian-jessie packages += # generic (another SR that build packages automatically would use the same list) apt-utils build-essential devscripts equivs lsb-release diff --git a/software/kvm/common.cfg b/software/kvm/common.cfg index 2a828ef0b2ae30d931c835341144ec67e3d73c4b..1c1f5584292b8c98545be01f82e09927f2a60587 100644 --- a/software/kvm/common.cfg +++ b/software/kvm/common.cfg @@ -10,7 +10,7 @@ extends = ../../component/netcat/buildout.cfg ../../component/pycurl/buildout.cfg ../../stack/slapos.cfg - ../../stack/nodejs.cfg + ../../component/nodejs/buildout.cfg ../../stack/resilient/buildout.cfg # stacks are listed from most generic to most specific, @@ -99,7 +99,7 @@ recipe = hexagonit.recipe.download ignore-existing = true url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2 mode = 644 -md5sum = 68b66fb3e9020642e57f4a4ee266f2b3 +md5sum = c348188a6dcb15430bea0bad51c249fc download-only = true on-update = true @@ -186,7 +186,7 @@ ignore-existing = true url = ${:_profile_base_location_}/template/template-kvm-run.in mode = 644 filename = template-kvm-run.in -md5sum = bd238397af6236b6b24b693012feeece +md5sum = c6f1536a3502102dadbfb9d82496cc36 download-only = true on-update = true diff --git a/software/kvm/instance-kvm.cfg.jinja2 b/software/kvm/instance-kvm.cfg.jinja2 index 4baf7bd4647786ca91efd8a2a595e666aa9538be..5e35f7ded171316cb1cc19f5bdf2aa334fca35e6 100644 --- a/software/kvm/instance-kvm.cfg.jinja2 +++ b/software/kvm/instance-kvm.cfg.jinja2 @@ -381,8 +381,8 @@ interface-url = {{ slapparameter_dict.get('monitor-interface-url', 'https://moni <= monitor-publish recipe = slapos.cookbook:publish ipv6 = ${slap-network-information:global-ipv6} -backend-url = https://[${novnc-instance:ip}]:${novnc-instance:port}/vnc_auto.html?host=[${novnc-instance:ip}]&port=${novnc-instance:port}&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd} -url = ${request-slave-frontend:connection-url}/vnc_auto.html?host=${request-slave-frontend:connection-domainname}&port=${request-slave-frontend:connection-port}&encrypt=1&path=${request-slave-frontend:connection-resource}&password=${kvm-controller-parameter-dict:vnc-passwd} +backend-url = https://[${novnc-instance:ip}]:${novnc-instance:port}/vnc.html?host=[${novnc-instance:ip}]&port=${novnc-instance:port}&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd} +url = ${request-slave-frontend:connection-url}/vnc.html?host=${request-slave-frontend:connection-domainname}&port=${request-slave-frontend:connection-port}&encrypt=1&path=${request-slave-frontend:connection-resource}&password=${kvm-controller-parameter-dict:vnc-passwd} {% set disk_number = len(storage_dict) -%} maximum-extra-disk-amount = {{ disk_number }} {% set iface = 'eth0' -%} diff --git a/software/kvm/software.cfg b/software/kvm/software.cfg index 66cd359684383e1259072bf71b2e71f444f6693c..0191179a36d8b3853b8b1c39b53609cc123f489e 100644 --- a/software/kvm/software.cfg +++ b/software/kvm/software.cfg @@ -5,7 +5,7 @@ extends = common.cfg # XXX - use websockify = 0.5.1 for compatibility with kvm frontend websockify = 0.5.1 -slapos.toolbox = 0.74 +slapos.toolbox = 0.76 erp5.util = 0.4.49 apache-libcloud = 1.1.0 collective.recipe.environment = 0.2.0 diff --git a/software/kvm/template/template-kvm-run.in b/software/kvm/template/template-kvm-run.in index 82e80d2169b22b61fefddba9a38bd7221a71238b..f8424107502032e1d8e07630b069d0b9dbb490aa 100644 --- a/software/kvm/template/template-kvm-run.in +++ b/software/kvm/template/template-kvm-run.in @@ -264,7 +264,7 @@ if enable_device_hotplug != 'true': ram = '%sM,slots=128,maxmem=%sM' % (ram_size, ram_max_size) else: smp = '1,maxcpus=%s' % smp_max_count - ram = '%sM,slots=128,maxmem=%s' % (init_ram_size, ram_max_size) + ram = '%sM,slots=128,maxmem=%sM' % (init_ram_size, ram_max_size) kvm_argument_list = [qemu_path, '-enable-kvm', '-smp', smp, '-name', vm_name, '-m', ram, '-vga', 'std', '-drive', 'file=%s,if=%s,cache=%s,aio=%s' % (disk_path, disk_type, disk_cache, disk_aio), diff --git a/software/monitor/fluentd-agent.conf.jinja2.in b/software/monitor/fluentd-agent.conf.jinja2.in deleted file mode 100644 index b4410e9b9027a85c016f32f17a18c90aefd042ea..0000000000000000000000000000000000000000 --- a/software/monitor/fluentd-agent.conf.jinja2.in +++ /dev/null @@ -1,147 +0,0 @@ -# DESTINATION -<match td.*.*> - type tdlog - apikey YOUR_API_KEY - - auto_create_table - buffer_type file - buffer_path {{ fluentd_log_directory }}/td-agent/buffer/td - - <secondary> - type file - path {{ fluentd_log_directory }}/failed_records - </secondary> -</match> - -{% set wendelin_streamtool_uri = slapparameter_dict.get('wendelin-streamtool-uri', '') -%} -{% set wendelin_password = slapparameter_dict.get('wendelin-password', '') -%} -{% set wendelin_user = slapparameter_dict.get('wendelin-user', '') -%} -{% set wendelin_uid = slapparameter_dict.get('wendelin-uid', 'UNKNOWN') -%} -{% if wendelin_streamtool_uri and wendelin_password and wendelin_user -%} -<source> - type syslog - port 42185 - bind 127.0.0.1 - tag {{ wendelin_uid }}.system -</source> - -<match {{ wendelin_uid }}.system.**> - @type wendelin - @id wendelin_out - - streamtool_uri {{ wendelin_streamtool_uri }}_syslog - user {{ wendelin_user }} - password {{ wendelin_password }} - - buffer_type memory - flush_interval 20s -</match> - -<source> - type tail - path {{ fluentd_log_directory }}/consumption/*/dump_user.csv - pos_file {{ fluentd_log_directory }}/consumption/dump_user.pos - tag slapos.monitor.usage.user.{{ wendelin_uid }} - format /^(?<partition>[^,]*),(?<pid>[^,]*),(?<process>[^,]*),(?<cpu_percent>[^,]*),(?<cpu_time>[^,]*),(?<cpu_num_threads>[^,]*),(?<memory_percent>[^,]*),(?<memory_rss>[^,]*),(?<io_rw_counter>[^,]*),(?<io_cycles_counter>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/ - read_from_head true -</source> - -<match slapos.monitor.usage.user.{{ wendelin_uid }}> - @type wendelin - @id wendelin_out - - streamtool_uri {{ wendelin_streamtool_uri }}_user - user {{ wendelin_user }} - password {{ wendelin_password }} - - buffer_type memory - flush_interval 20s -</match> - -<source> - type tail - path {{ fluentd_log_directory }}/consumption/*/dump_system.csv - pos_file {{ fluentd_log_directory }}/consumption/dump_system.pos - tag slapos.monitor.usage.system.{{ wendelin_uid }} - format /^(?<loadavg>[^,]*),(?<cpu_percent>[^,]*),(?<memory_used>[^,]*),(?<memory_free>[^,]*),(?<net_in_bytes>[^,]*),(?<net_in_errors>[^,]*),(?<net_in_dropped>[^,]*),(?<net_out_bytes>[^,]*),(?<net_out_errors>[^,]*),(?<net_out_dropped>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/ - read_from_head true -</source> - -<match slapos.monitor.usage.system.{{ wendelin_uid }}> - @type wendelin - @id wendelin_out - - streamtool_uri {{ wendelin_streamtool_uri }}_system - user {{ wendelin_user }} - password {{ wendelin_password }} - - buffer_type memory - flush_interval 20s -</match> - - -<source> - type tail - path {{ fluentd_log_directory }}/consumption/*/dump_computer.csv - pos_file {{ fluentd_log_directory }}/consumption/dump_computer.pos - tag slapos.monitor.usage.computer.{{ wendelin_uid }} - format /^(?<cpu_num_core>[^,]*),(?<cpu_frequency>[^,]*),(?<cpu_type>[^,]*),(?<memory_size>[^,]*),(?<memory_type>[^,]*),(?<partition_list>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/ - - read_from_head true -</source> - -<match slapos.monitor.usage.computer.{{ wendelin_uid }}> - @type wendelin - @id wendelin_out - - streamtool_uri {{ wendelin_streamtool_uri }}_computer - user {{ wendelin_user }} - password {{ wendelin_password }} - - buffer_type memory - flush_interval 20s -</match> - -<source> - type tail - path {{ fluentd_log_directory }}/consumption/*/dump_disk.csv - pos_file {{ fluentd_log_directory }}/consumption/dump_disk.pos - tag slapos.monitor.usage.disk.{{ wendelin_uid }} - format /^(?<partition>[^,]*),(?<used>[^,]*),(?<free>[^,]*),(?<mountpoint>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/ - read_from_head true -</source> - -<match slapos.monitor.usage.disk.{{ wendelin_uid }}> - @type wendelin - @id wendelin_out - - streamtool_uri {{ wendelin_streamtool_uri }}_disk - user {{ wendelin_user }} - password {{ wendelin_password }} - - buffer_type memory - flush_interval 20s -</match> - -<source> - type tail - path {{ fluentd_log_directory }}/consumption/*/dump_temperature.csv - pos_file {{ fluentd_log_directory }}/consumption/dump_temperature.pos - tag slapos.monitor.usage.temperature.{{ wendelin_uid }} - format /^(?<sensor_id>[^,]*),(?<temperature>[^,]*),(?<alarm>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/ - read_from_head true -</source> - -<match slapos.monitor.usage.temperature.{{ wendelin_uid }}> - @type wendelin - @id wendelin_out - - streamtool_uri {{ wendelin_streamtool_uri }}_temperature - user {{ wendelin_user }} - password {{ wendelin_password }} - - buffer_type memory - flush_interval 20s -</match> - -{% endif -%} diff --git a/software/monitor/fluentd-distributor-agent.conf.jinja2.in b/software/monitor/fluentd-distributor-agent.conf.jinja2.in deleted file mode 100644 index f40cdadddba7c806133d6cfceb92725637350b1e..0000000000000000000000000000000000000000 --- a/software/monitor/fluentd-distributor-agent.conf.jinja2.in +++ /dev/null @@ -1,116 +0,0 @@ -# DESTINATION -<match td.*.*> - type tdlog - apikey YOUR_API_KEY - - auto_create_table - buffer_type file - buffer_path {{ fluentd_log_directory }}/td-agent/buffer/td - - <secondary> - type file - path {{ fluentd_log_directory }}/failed_records - </secondary> -</match> - -## match tag=debug.** and dump to console -<match debug.**> - type stdout -</match> - -<source> - type tail - path {{ crawl_log_directory }}/*/*ping.log.20* - pos_file {{ crawl_log_directory }}/tail_in_ping.pos - tag slapos.monitor.networktest.ping.ipv4 - format /^(?<time>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/ - read_from_head true -</source> - -<source> - type tail - path {{ crawl_log_directory }}/*/*ping6.log.20* - pos_file {{ crawl_log_directory }}/tail_in_ping6.pos - tag slapos.monitor.networktest.ping.ipv6 - format /^(?<time>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/ - read_from_head true -</source> - -<match slapos.monitor.networktest.ping.*> - type copy -{% for slave_instance in slave_instance_list -%} -{% if slave_instance.get("ping_ip_list") -%} - <store> - type grep - regexp1 name_or_ip ^{{ slave_instance.get("ping_ip_list") }}$ - add_tag_prefix {{ slave_instance.get("slave_reference") }} - </store> -{% endif -%} -{% if slave_instance.get("ping6_ip_list") -%} - <store> - type grep - regexp1 name_or_ip ^{{ slave_instance.get("ping6_ip_list") }}$ - add_tag_prefix {{ slave_instance.get("slave_reference") }} - </store> -{% endif -%} -{% endfor -%} -</match> - -{% for slave_instance in slave_instance_list -%} -<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv6> - type file - path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping6/log - append true - include_time_key true - include_tag_key true - format json -</match> -<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv4> - type file - path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping/log - append true - include_time_key true - include_tag_key true - format json -</match> -{% endfor -%} - -<source> - type tail - path {{ network_user_logs }}/*/ping*/log/*.log - pos_file {{ crawl_log_directory }}/tail_in_ping6.pos - tag slapos.wendelin.networktest - format json - read_from_head true -</source> - -{% set wendelin_streamtool_uri = slapparameter_dict.get('wendelin-streamtool-uri', '') -%} -{% set wendelin_password = slapparameter_dict.get('wendelin-password', '') -%} -{% set wendelin_user = slapparameter_dict.get('wendelin-user', '') -%} -{% if wendelin_streamtool_uri and wendelin_password and wendelin_user -%} -<match slapos.wendelin.networktest> - - @type wendelin - @id wendelin_out - - streamtool_uri {{ wendelin_streamtool_uri }} - user {{ wendelin_user }} - password {{ wendelin_password }} - - buffer_type memory - flush_interval 20s -</match> -{% endif -%} - -## SOURCE -<source> - type forward -</source> - -## live debugging agent -<source> - type debug_agent - bind 127.0.0.1 - port 24230 -</source> - diff --git a/software/monitor/instance-monitor-distributor.cfg.jinja2 b/software/monitor/instance-monitor-distributor.cfg.jinja2 index 77026a60598a9df04d20783ba44372d2d0fc7551..3cf2c3c0178c542aa9fb1f4936f166538dd02e9e 100644 --- a/software/monitor/instance-monitor-distributor.cfg.jinja2 +++ b/software/monitor/instance-monitor-distributor.cfg.jinja2 @@ -72,26 +72,7 @@ wrapper-path = ${monitor-directory:bin}/log-crawler extends = {{ instance_base_monitor }} parts += slave-test-configuration - fluentd-distributor-wrapper {% for part in part_list %} {{ ' %s' % part }} {% endfor %} -[fluentd-distributor-wrapper] -<=fluentd-wrapper -command-line = {{ fluentd_location }}/bin/fluentd -l ${monitor-directory:log}/fluend-distributor.log -c ${fluentd-distributor-conf-configuration:rendered} -wrapper-path = ${monitor-directory:service}/fluentd-distributor - -[fluentd-distributor-conf-configuration] -recipe = slapos.recipe.template:jinja2 -template = {{ fluent_distributor_conf_output }} -rendered = ${monitor-directory:etc}/fluentd-distributor.cfg -mode = 0744 -context = - key slapparameter_dict slap-configuration:configuration - key slave_instance_list slap-configuration:slave-instance-list - key fluentd_log_directory monitor-directory:fluentd-log - key crawl_log_directory monitor-directory:crawl-log - key network_user_logs monitor-directory:network-user-logs - key computer_id slap-configuration:computer - diff --git a/software/monitor/instance-monitor.cfg.jinja2 b/software/monitor/instance-monitor.cfg.jinja2 index abedcc6106f25acdeb2b503d80a364338a4cdad3..30a6f5715b8bb2cec9f99a341ce3d74818d3c8a8 100644 --- a/software/monitor/instance-monitor.cfg.jinja2 +++ b/software/monitor/instance-monitor.cfg.jinja2 @@ -8,7 +8,6 @@ parts = symlink-re6st-logs symlink-collected-logs monitor-collect-csv-wrapper - fluentd-wrapper monitor-base monitor-check-memory-usage monitor-check-cpu-usage @@ -24,7 +23,10 @@ offline = true recipe = slapos.cookbook:cron.d name = network-bench-test frequency = */10 * * * * -command = {{ buildout }}/networkbench ${network-bench-configuration:rendered} ${monitor-directory:monitor-log} +# skip to not fill cron log file. +# command = {{ buildout_bin }}/networkbench ${network-bench-configuration:rendered} ${monitor-directory:monitor-log} +command = true + [symlink-re6st-logs] recipe = cns.recipe.symlink @@ -63,8 +65,7 @@ monitor-log = ${:private}/monitor-log cache = ${:var}/cache mod-ssl = ${:cache}/httpd_mod_ssl system-log = ${:private}/system-log -fluentd-log = ${:log}/fluentd -consumption = ${:fluentd-log}/consumption +consumption = ${:log}/consumption [monitor-httpd-configuration-file] context = @@ -76,60 +77,37 @@ context = section slave_information slap-configuration key monitor_private_hash slap-configuration:private-hash -[fluentd-wrapper] -recipe = slapos.cookbook:wrapper -command-line = {{ fluentd_location }}/bin/fluentd -qq -l ${monitor-directory:log}/fluend.log -c ${fluentd-conf-configuration:rendered} -wrapper-path = ${monitor-directory:service}/fluentd -environment = - GEM_PATH={{ fluentd_location }}/lib/ruby/gems/1.8/ - -[fluentd-conf-configuration] -recipe = slapos.recipe.template:jinja2 -template = {{ fluent_conf_output }} -rendered = ${monitor-directory:etc}/fluentd.cfg -mode = 0744 -context = - key slapparameter_dict slap-configuration:configuration - key slave_instance_list slap-configuration:slave-instance-list - key fluentd_log_directory monitor-directory:fluentd-log - key computer_id slap-connection:computer-id - [monitor-collect-csv-wrapper] recipe = slapos.cookbook:wrapper command-line = - ${monitor-directory:bin}/python {{ monitor_collect_csv_dump }} --output_folder ${monitor-directory:fluentd-log}/consumption/ + ${monitor-directory:bin}/python {{ monitor_collect_csv_dump }} --output_folder ${monitor-directory:consumption} wrapper-path = ${monitor-directory:reports}/monitor-collect-csv-dump [monitor-check-cpu-usage] recipe = slapos.cookbook:wrapper command-line = ${monitor-directory:bin}/python {{ monitor_check_system_health }} cpu ${init-monitor-parameters:cpu-load-file} -wrapper-path = ${monitor-directory:promises}/system-CPU-load-check +wrapper-path = ${directory:promises}/system-CPU-load-check [monitor-check-memory-usage] recipe = slapos.cookbook:wrapper -command-line = ${monitor-directory:bin}/python {{ monitor_check_system_health }} mem ${init-monitor-parameters:mem-free-file} ${directory:monitor} -wrapper-path = ${monitor-directory:promises}/system-MEMORY-usage-check +command-line = {{ buildout_bin}}/check-computer-memory + -db ${monitor-instance-parameter:collector-db} + --threshold ${slap-parameter:memory-percent-threshold} + --unit percent +wrapper-path = ${directory:promises}/check-computer-memory-usage [publish-connection-information] recipe = slapos.cookbook:publish monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password} server_log_url = ${monitor-publish-parameters:monitor-base-url}/${slap-configuration:private-hash}/ -[monitor-instance-parameter] -instance-configuration = - file max-cpu-load-per-core ${init-monitor-parameters:cpu-load-file} - file min-free-mem-percent ${init-monitor-parameters:mem-free-file} - [init-monitor-parameters] recipe = plone.recipe.command cpu-load-file = ${directory:monitor}/cpu-load-tolerance mem-free-file = ${directory:monitor}/mem-free-limit command = if [ ! -s "${:cpu-load-file}" ]; then - echo "1.5" > ${:cpu-load-file} - fi - if [ ! -s "${:mem-free-file}" ]; then - echo "7.0" > ${:mem-free-file} + echo ${slap-parameter:cpu-load-threshold} > ${:cpu-load-file} fi [slap-configuration] @@ -141,3 +119,7 @@ key = ${slap-connection:key-file} cert = ${slap-connection:cert-file} private-hash = ${pwgen:passwd}${pwgen32:passwd} frontend-domain = + +[slap-parameter] +cpu-load-threshold = 2.0 +memory-percent-threshold = 96 diff --git a/software/monitor/instance.cfg b/software/monitor/instance.cfg index f2e8297489bf5f58d246571258d463864be7ca00..6eadb8445926b83d9e7ffa58de1af7dbcff76760 100644 --- a/software/monitor/instance.cfg +++ b/software/monitor/instance.cfg @@ -15,14 +15,12 @@ recipe = slapos.recipe.template:jinja2 template = ${template-monitor:destination} rendered = $${buildout:directory}/template-base-monitor.cfg extensions = jinja2.ext.do -context = key buildout buildout:bin-directory - key develop_eggs_directory buildout:develop-eggs-directory +context = key develop_eggs_directory buildout:develop-eggs-directory key eggs_directory buildout:eggs-directory key slapparameter_dict slap-configuration:configuration + raw buildout_bin ${buildout:bin-directory} raw monitor_template_output ${monitor-template:output} - raw network_benck_cfg_output ${network-bench-cfg:output} - raw fluentd_location ${fluentd:location} - raw fluent_conf_output ${fluentd-agent-conf:output} + raw network_benck_cfg_output ${network-bench-cfg:output} raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output} raw monitor_check_system_health ${monitor-system-health:output} mode = 0644 @@ -33,15 +31,13 @@ template = ${template-monitor-distributor:destination} rendered = $${buildout:directory}/template-monitor-base-distributor.cfg extensions = jinja2.ext.do context = import json_module json - key buildout buildout:bin-directory key develop_eggs_directory buildout:develop-eggs-directory key eggs_directory buildout:eggs-directory key slapparameter_dict slap-configuration:configuration key instance_base_monitor instance-base-monitor:rendered key slave_instance_list slap-configuration:slave-instance-list + raw buildout_bin ${buildout:bin-directory} raw template_json_distributor_test ${json-test-template:destination} - raw fluentd_location ${fluentd:location} - raw fluent_distributor_conf_output ${fluentd-agent-distributor-conf:output} raw wget_bin ${wget:location}/bin/wget mode = 0644 diff --git a/software/monitor/software.cfg b/software/monitor/software.cfg index 91f9d0d15d0c232843c30082a7da5ae4b733cb76..084b65d5bd26955850899ff66501ba662847e04c 100644 --- a/software/monitor/software.cfg +++ b/software/monitor/software.cfg @@ -5,7 +5,6 @@ extends = ../../component/python-cryptography/buildout.cfg ../../component/wget/buildout.cfg ../../stack/monitor/buildout.cfg - ../../component/fluentd/buildout.cfg ../../stack/slapos.cfg parts = @@ -16,30 +15,27 @@ parts = template template-monitor-distributor template-monitor - fluentd - fluentd-agent-conf - fluentd-agent-distributor-conf monitor-collect-csv-dump [template] recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance.cfg output = ${buildout:directory}/template.cfg -md5sum = 3ff5fb2710bf0ea84632c6d6d3894dd9 +md5sum = 641c5916739f78171c616af00fe974a2 mode = 0644 [template-monitor] recipe = slapos.recipe.build:download url = ${:_profile_base_location_}/instance-monitor.cfg.jinja2 destination = ${buildout:directory}/template-base-monitor.cfg -md5sum = 5def53c8faa0dfca313e53ebcc3229fa +md5sum = db36ed784c690be892bf1834444c1fe7 mode = 0644 [template-monitor-distributor] recipe = slapos.recipe.build:download url = ${:_profile_base_location_}/instance-monitor-distributor.cfg.jinja2 destination = ${buildout:directory}/template-monitor-base-distributor.cfg -md5sum = 0ab9a86df18125335ed365bb673bbcf3 +md5sum = 61c0bfdfc0a2b51ba15fe4a49baf6091 mode = 0644 [json-test-template] @@ -56,20 +52,6 @@ md5sum = cfcbf2002b8eff5153e2bf68ed24b720 output = ${buildout:directory}/template-network-bench-cfg.in mode = 0644 -[fluentd-agent-conf] -recipe = slapos.recipe.template -url = ${:_profile_base_location_}/fluentd-agent.conf.jinja2.in -md5sum = 3ea59906937eab7aeef78f46c4994ecd -output = ${buildout:directory}/fluentd-agent.conf.jinja2.in -mode = 0644 - -[fluentd-agent-distributor-conf] -recipe = slapos.recipe.template -url = ${:_profile_base_location_}/fluentd-distributor-agent.conf.jinja2.in -md5sum = 4b81ddcbe2f16d23013caac37151d396 -output = ${buildout:directory}/fluentd-agent-distributor.conf.jinja2.in -mode = 0644 - [monitor-collect-csv-dump] <= monitor-template-script url = ${:_profile_base_location_}/script/${:filename} @@ -103,12 +85,8 @@ scripts = monitor.genstatus monitor.configwrite is-process-older-than-dependency-set - -[fluentd] -gems += - fluent-plugin-wendelin==0.1.alpha1 - fluent-plugin-grep==0.3.4 - fluent-plugin-bin==0.1 + check-free-disk + check-computer-memory [monitor-eggs] eggs += diff --git a/software/neoppod/software-common.cfg b/software/neoppod/software-common.cfg index dad2905c7cf229b51d66f8fc7f8444a600726ed3..ffadd30abd4f936a31ef2c95ccacad6cb10693ad 100644 --- a/software/neoppod/software-common.cfg +++ b/software/neoppod/software-common.cfg @@ -48,6 +48,11 @@ eggs = neoppod[admin, ctl, master, storage-mysqldb] ZODB zope.testing zodbtools +patch-binary = ${patch:location}/bin/patch +ZEO-patch-options = -p1 +ZEO-patches = + ${:_profile_base_location_}/../../component/egg-patch/ZEO4/TCP_NODELAY.patch#b07288522d5c6857738240d948321df6 + [slapos-deps-eggs] recipe = zc.recipe.egg @@ -117,7 +122,7 @@ persistent = 4.2.3 pycrypto = 2.6.1 pycurl = 7.43.0 slapos.recipe.template = 4.3 -slapos.toolbox = 0.74 +slapos.toolbox = 0.76 smmap2 = 2.0.1 transaction = 1.7.0 zodbpickle = 0.6.0 @@ -161,7 +166,7 @@ zodburi = 2.3.0 # Required by: # zodburi==2.0 # ZEO 5 requires transaction >= 2 -ZEO = 4.3.1 +ZEO = 4.3.1+SlapOSPatched001 # Required by: # zodburi==2.0 diff --git a/software/neoppod/software-zodb5.cfg b/software/neoppod/software-zodb5.cfg index 290ce57b9ecfcda2c89b6a6dfeb91c9e6cb59f4c..b625d1fe56f64f175949ca53efa96a9059c008e6 100644 --- a/software/neoppod/software-zodb5.cfg +++ b/software/neoppod/software-zodb5.cfg @@ -1,6 +1,9 @@ [buildout] extends = software.cfg +[neoppod] +ZEO-patches = + [versions] ZODB = 5.3.0 ZEO = 5.1.0 diff --git a/software/neotest/gowork.cfg b/software/neotest/gowork.cfg index 35cb19c741a402c78e53f9155cb319f36051a3a9..0b5a87c52ade277a72d543f305b116ce1c8205b3 100644 --- a/software/neotest/gowork.cfg +++ b/software/neotest/gowork.cfg @@ -3,95 +3,151 @@ # list of go git repositories to fetch [gowork.goinstall] depends_gitfetch = + ${go_github.com_DataDog_czlib:recipe} ${go_github.com_cznic_strutil:recipe} ${go_github.com_golang_glog:recipe} + ${go_github.com_gwenn_gosqlite:recipe} + ${go_github.com_gwenn_yacr:recipe} ${go_github.com_kisielk_og-rek:recipe} ${go_github.com_kylelemons_godebug:recipe} ${go_github.com_pkg_errors:recipe} ${go_github.com_pkg_profile:recipe} + ${go_github.com_soheilhy_cmux:recipe} ${go_github.com_someonegg_gocontainer:recipe} + ${go_github.com_someonegg_gox:recipe} + ${go_golang.org_x_crypto:recipe} ${go_golang.org_x_net:recipe} ${go_golang.org_x_perf:recipe} ${go_golang.org_x_sync:recipe} + ${go_golang.org_x_sys:recipe} + ${go_golang.org_x_text:recipe} ${go_golang.org_x_tools:recipe} ${go_lab.nexedi.com_kirr_go123:recipe} ${go_lab.nexedi.com_kirr_neo:recipe} +[go_github.com_DataDog_czlib] +<= go-git-package +go.importpath = github.com/DataDog/czlib +repository = https://github.com/DataDog/czlib +revision = 4bc9a24e37 + [go_github.com_cznic_strutil] <= go-git-package go.importpath = github.com/cznic/strutil repository = https://github.com/cznic/strutil -revision = 529a34b1c1 +revision = 529a34b1c1 [go_github.com_golang_glog] <= go-git-package go.importpath = github.com/golang/glog repository = https://github.com/golang/glog -revision = 23def4e6c1 +revision = 23def4e6c1 + +[go_github.com_gwenn_gosqlite] +<= go-git-package +go.importpath = github.com/gwenn/gosqlite +repository = https://github.com/gwenn/gosqlite +revision = 29cd841087 + +[go_github.com_gwenn_yacr] +<= go-git-package +go.importpath = github.com/gwenn/yacr +repository = https://github.com/gwenn/yacr +revision = 77093bdc7e [go_github.com_kisielk_og-rek] <= go-git-package go.importpath = github.com/kisielk/og-rek repository = https://github.com/kisielk/og-rek -revision = dd41cde712 +revision = dd41cde712 [go_github.com_kylelemons_godebug] <= go-git-package go.importpath = github.com/kylelemons/godebug repository = https://github.com/kylelemons/godebug -revision = d65d576e93 +revision = d65d576e93 [go_github.com_pkg_errors] <= go-git-package go.importpath = github.com/pkg/errors repository = https://github.com/pkg/errors -revision = v0.8.0-6-g2b3a18b5f0 +revision = v0.8.0-12-g816c908556 [go_github.com_pkg_profile] <= go-git-package go.importpath = github.com/pkg/profile repository = https://github.com/pkg/profile -revision = v1.2.1-0-g5b67d42886 +revision = v1.2.1-0-g5b67d42886 + +[go_github.com_soheilhy_cmux] +<= go-git-package +go.importpath = github.com/soheilhy/cmux +repository = https://github.com/soheilhy/cmux +revision = e09e9389d8 [go_github.com_someonegg_gocontainer] <= go-git-package go.importpath = github.com/someonegg/gocontainer repository = https://github.com/someonegg/gocontainer -revision = fc2c7e84b5 +revision = fc2c7e84b5 + +[go_github.com_someonegg_gox] +<= go-git-package +go.importpath = github.com/someonegg/gox +repository = https://github.com/someonegg/gox +revision = 4915b7fd7c + +[go_golang.org_x_crypto] +<= go-git-package +go.importpath = golang.org/x/crypto +repository = https://go.googlesource.com/crypto +revision = 88942b9c40 [go_golang.org_x_net] <= go-git-package go.importpath = golang.org/x/net repository = https://go.googlesource.com/net -revision = 1087133bc4 +revision = 6078986fec [go_golang.org_x_perf] <= go-git-package go.importpath = golang.org/x/perf repository = https://go.googlesource.com/perf -revision = 4469e6ce8c +revision = 8c788eb673 [go_golang.org_x_sync] <= go-git-package go.importpath = golang.org/x/sync repository = https://go.googlesource.com/sync -revision = 8e0aa688b6 +revision = 1d60e4601c + +[go_golang.org_x_sys] +<= go-git-package +go.importpath = golang.org/x/sys +repository = https://go.googlesource.com/sys +revision = 91ee8cde43 + +[go_golang.org_x_text] +<= go-git-package +go.importpath = golang.org/x/text +repository = https://go.googlesource.com/text +revision = v0.3.0-42-gab48842968 [go_golang.org_x_tools] <= go-git-package go.importpath = golang.org/x/tools repository = https://go.googlesource.com/tools -revision = 9bd2f44268 +revision = 77106db15f [go_lab.nexedi.com_kirr_go123] <= go-git-package go.importpath = lab.nexedi.com/kirr/go123 repository = https://lab.nexedi.com/kirr/go123.git -revision = 2578d58311 +revision = 76f667ba6c [go_lab.nexedi.com_kirr_neo] <= go-git-package go.importpath = lab.nexedi.com/kirr/neo repository = https://lab.nexedi.com/kirr/neo.git -revision = v1.8-1326-g4d0cd89484 +revision = v1.8.1-1634-g4000df14e4 diff --git a/software/neotest/software.cfg b/software/neotest/software.cfg index 9fe07ba0a0bec00578f43a2f467a92c035d71567..1369d1b6f8ff07789db0bccc3c5740f61d55d946 100644 --- a/software/neotest/software.cfg +++ b/software/neotest/software.cfg @@ -8,6 +8,8 @@ extends = ../neoppod/software-common.cfg ../../component/golang/buildout.cfg gowork.cfg + ../../component/sqlite3/buildout.cfg + ../../component/zlib/buildout.cfg ../../component/wendelin.core/buildout.cfg ../../component/ethtool/buildout.cfg ../../component/ioping/buildout.cfg @@ -48,6 +50,10 @@ install = github.com/pkg/profile \ golang.org/x/perf/cmd/benchstat +cpkgpath = + ${sqlite3:location}/lib/pkgconfig + ${zlib:location}/lib/pkgconfig + # dev-install neo from go checkout [neoppod-develop] setup = ${go_lab.nexedi.com_kirr_neo:location} @@ -108,13 +114,13 @@ eggs = # wendelin.core: latest not yet released [wendelin.core-repository] -revision= v0.11-4-g38fbc83ceb +revision= v0.11-15-gf785ac079b # ping eggs versions [versions] pyasn1 = 0.3.7 ZODB3 = 3.11.0 -numpy = 1.13.3 +numpy = 1.14.2 zope.testing = 4.6.2 erp5.util = 0.4.50 @@ -128,3 +134,7 @@ ZConfig = 3.2.0 # ZEO==4.3.1 # ZODB==4.4.5 zc.lockfile = 1.2.1 + +# Required by: +# neoppod==1.8.1 +python-dateutil = 2.7.1 diff --git a/software/nginx-push-stream/software.cfg b/software/nginx-push-stream/software.cfg index d5955d636eeb5f20277703fe458838593d90899c..6f31e3098d6cd00ffb4ca035d0ac4a547e9c3601 100644 --- a/software/nginx-push-stream/software.cfg +++ b/software/nginx-push-stream/software.cfg @@ -46,7 +46,7 @@ mode = 0644 [versions] slapos.recipe.template = 4.3 -slapos.toolbox = 0.74 +slapos.toolbox = 0.76 dnspython = 1.15.0 PyRSS2Gen = 1.1 erp5.util = 0.4.50 diff --git a/software/re6stnet/instance-re6stnet-slave-input-schema.json b/software/re6stnet/instance-re6stnet-slave-input-schema.json new file mode 100644 index 0000000000000000000000000000000000000000..8d1e2625aea1d6de13ab6687b57c3b53349236f9 --- /dev/null +++ b/software/re6stnet/instance-re6stnet-slave-input-schema.json @@ -0,0 +1,5 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "properties": { + } +} diff --git a/software/re6stnet/software.cfg b/software/re6stnet/software.cfg index 28978d09febcfe51e2691c1a075725666fae0b5e..428aeb045f0d90a8f4d996e270050fe820d1f15f 100644 --- a/software/re6stnet/software.cfg +++ b/software/re6stnet/software.cfg @@ -104,6 +104,7 @@ filename = registry-run.in md5sum = 0bf4f2c03e06b55c6c6cc55fa33e65d6 [versions] +re6stnet = 0.494 apache-libcloud = 0.17.0 ecdsa = 0.13 gitdb = 0.6.4 @@ -111,7 +112,7 @@ plone.recipe.command = 1.1 pycrypto = 2.6.1 pycurl = 7.43.0 slapos.recipe.template = 4.3 -slapos.toolbox = 0.74 +slapos.toolbox = 0.76 smmap = 0.9.0 # Required by: diff --git a/software/re6stnet/software.cfg.json b/software/re6stnet/software.cfg.json index c8a3d60efd3c669e57685ceb1be4019bf11d6b2d..96b20aca86b07a22ba1d715444eab46ed793e5c4 100644 --- a/software/re6stnet/software.cfg.json +++ b/software/re6stnet/software.cfg.json @@ -10,12 +10,14 @@ "response": "instance-re6stnet-output-schema.json", "index": 0 }, - "registry": { - "title": "registry", + "default-slave": { + "title": "Re6st Token", "description": "Re6st registry", - "request": "instance-re6stnet-resilient-input-schema.json", + "software-type": "default", + "request": "instance-re6stnet-slave-input-schema.json", "response": "instance-re6stnet-output-schema.json", + "shared": true, "index": 1 } } -} \ No newline at end of file +} diff --git a/software/slapos-testing/README.md b/software/slapos-testing/README.md new file mode 100644 index 0000000000000000000000000000000000000000..07216cff74692c4c740f854c2583b21f78f95219 --- /dev/null +++ b/software/slapos-testing/README.md @@ -0,0 +1,48 @@ +# Slapos egg tests + +This software release is used to run unit test of slapos eggs. + +The approach is to use setuptools' integrated test runner, `python setup.py test`, to run tests. + +The `python` used in this command will be a `zc.recipe.egg` interpreter with +all eggs pre-installed by this software release. + +Nexedi staff can see the results of this test from the test suite +`SLAPOS-EGG-TEST` in test result module. + + +Here's an example session of how a developer could use this software release in +slaprunner to develop a slapos egg, in the example `slapos.core`, to make +changes to the code, run tests and publish changes. + +```bash +# install this software release +SR=https://lab.nexedi.com/nexedi/slapos/raw/master/software/slapos-testing/software.cfg +COMP=slaprunner +INSTANCE_NAME=$COMP + +slapos supply $SR $COMP +slapos node software +slapos request --node=node=$COMP $INSTANCE_NAME $SR +slapos node instance + +# The source code is a git clone working copy on the instance +cd ~/srv/runner/instance/slappart0/parts/slapos.core/ + +# make some changes to the code +vim slapos/tests/client.py + +# run tests, using bundled python intepreter with pre-installed eggs dependencies +~/srv/runner/instance/slappart0/software_release/bin/python_for_test setup.py build + +# when satified, commit changes +git add -p && git commit + +# add developer's fork remote (this is only needed the first time) +git remote add my_remote https://lab.nexedi.com/your_username/slapos.core.git/ + +# push the changes +git push my_remote HEAD:feature_branch_name + +# then submit merge request +``` diff --git a/software/slapos-testing/buildout.hash.cfg b/software/slapos-testing/buildout.hash.cfg new file mode 100644 index 0000000000000000000000000000000000000000..95c5eb6dd28bcdd520d68086308bc59d3567704d --- /dev/null +++ b/software/slapos-testing/buildout.hash.cfg @@ -0,0 +1,20 @@ +# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax. +# The only allowed lines here are (regexes): +# - "^#" comments, copied verbatim +# - "^[" section beginings, copied verbatim +# - lines containing an "=" sign which must fit in the following categorie. +# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file +# But avoid directories, they are not portable. +# Copied verbatim. +# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported +# by the re-generation script. +# Re-generated. +# - other lines are copied verbatim +# Substitution (${...:...}), extension ([buildout] extends = ...) and +# section inheritance (< = ...) are NOT supported (but you should really +# not need these here). + +[template] +filename = instance.cfg +md5sum = 9dece9d12dc94bf5c35d307cc8aa4d6b + diff --git a/software/slapos-testing/instance.cfg b/software/slapos-testing/instance.cfg index 0a067e729e09c42b9548b76308ac913194b03327..5e50aff78e6a14af73dc93c0e12169969c25a8c4 100644 --- a/software/slapos-testing/instance.cfg +++ b/software/slapos-testing/instance.cfg @@ -1,10 +1,7 @@ [buildout] parts = - slapos.core-setup - erp5.util-setup phantomjs-wrapper slapos-test-runner - sh-environment eggs-directory = ${buildout:eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory} @@ -24,33 +21,26 @@ bin = $${buildout:directory}/bin etc = $${buildout:directory}/etc services = $${:etc}/run srv = $${buildout:directory}/srv -source-code = $${:srv}/eggs-source-code [download-source] recipe = slapos.recipe.build:gitclone git-executable = ${git:location}/bin/git -# Local development -[slapos.core] -<= download-source -repository = ${slapos.core-repository:location} - -[slapos.core-setup] -recipe = plone.recipe.command -command = echo "Updating setup...";cd $${slapos.core:location}; export PATH="$${slapos-test-runner:prepend-path}:$PATH"; export CPPFLAGS="$${environment:CPPFLAGS}"; export LDFLAGS="$${environment:LDFLAGS}"; export PYTHONPATH="$${environment:PYTHONPATH}"; export LOCAL_IPV4="$${environment:LOCAL_IPV4}"; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n -update-command = $${:command} - [caucase] <= download-source repository = ${caucase-repository:location} +[erp5.util] +<= download-source +repository = ${erp5.util-repository:location} + [slapos.cookbook] <= download-source repository = ${slapos.cookbook-repository:location} -[slapos.recipe.template] +[slapos.core] <= download-source -repository = ${slapos.recipe.template-repository:location} +repository = ${slapos.core-repository:location} [slapos.recipe.build] <= download-source @@ -60,57 +50,33 @@ repository = ${slapos.recipe.build-repository:location} <= download-source repository = ${slapos.recipe.cmmi-repository:location} -[slapos.toolbox] +[slapos.recipe.template] <= download-source -repository = ${slapos.toolbox-repository:location} +repository = ${slapos.recipe.template-repository:location} -[erp5-util] +[slapos.toolbox] <= download-source -repository = ${erp5-util-repository:location} +repository = ${slapos.toolbox-repository:location} -[erp5.util-setup] -recipe = plone.recipe.command -command = echo "Updating setup...";cd $${erp5-util:location}; export PATH="$${slapos-test-runner:prepend-path}:$PATH"; export CPPFLAGS="$${environment:CPPFLAGS}"; export LDFLAGS="$${environment:LDFLAGS}"; export PYTHONPATH="$${environment:PYTHONPATH}"; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n -update-command = $${:command} [slapos-test-runner] -recipe = slapos.cookbook:egg_test -run-test-suite = $${create-directory:bin}/runTestSuite -run-test-suite-binary = ${buildout:bin-directory}/runTestSuite -# The list of executables should be defined here and a combination -# of tests should dynamically generated. -#python-list = $${} -test-list = - $${slapos.cookbook:location} - $${slapos.core:location} - $${slapos.recipe.template:location} - $${slapos.recipe.build:location} - $${slapos.recipe.cmmi:location} - $${slapos.toolbox:location} - $${erp5-util:location} - $${caucase:location} -prepend-path = ${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:${python2.7:location}/bin -environment = environment - -[environment] -CPPFLAGS = -I${python2.7:location}/include/python2.7 -I${libxml2:location}/include -I${libxslt:location}/include -LDFLAGS = -L${python2.7:location}/lib -L${libxml2:location}/lib -L${libxslt:location}/lib -L${libxslt:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${python2.7:location}/lib -Wl,-rpath=${libxml2:location}/lib -Wl,-rpath=${libxslt:location}/lib -Wl,-rpath=${zlib:location}/lib -LD_LIBRARY_PATH = ${python2.7:location}/lib:${libxml2:location}/lib:${libxslt:location}/lib:${libxslt:location}/lib:${zlib:location}/lib -PYTHONPATH = ${python-setuptools:pythonpath}:${buildout:eggs-directory}:${buildout:develop-eggs-directory} -LOCAL_IPV4 = $${slap-configuration:ipv4-random} - -[sh-environment] -# Section exposes testing default environment as sh file. It is thus easy -# to directly develop and test the egg inside of this instance. -recipe = collective.recipe.template -input = inline: - export PATH="$${slapos-test-runner:prepend-path}:$PATH" - export CPPFLAGS="$${environment:CPPFLAGS}" - export LDFLAGS="$${environment:LDFLAGS}" - export PYTHONPATH="$${environment:PYTHONPATH}" - export PS1="[slapos-testing env Active] $PS1" -output = $${create-directory:bin}/environment.sh -mode = 755 +recipe = slapos.cookbook:wrapper +wrapper-path = $${create-directory:bin}/runTestSuite +command-line = + ${buildout:bin-directory}/runTestSuite + --python_interpreter=${buildout:bin-directory}/${eggs:interpreter} + --source_code_path_list=$${caucase:location},$${erp5.util:location},$${slapos.cookbook:location},$${slapos.core:location},$${slapos.recipe.build:location},$${slapos.recipe.cmmi:location},$${slapos.recipe.template:location},$${slapos.toolbox:location} + +# Notes about environment: +# * slapos.cookbook:wrapper does not seem to allow "extending" PATH. Tests +# needs ping, which is a setuid binary that cannot be installed via slapos +# way of building software without root access, so we keep "standard" +# /usr/bin and /bin in $PATH +# * LOCAL_IPV4 is needed for some slapos.core tests +environment = + PATH=${coreutils:location}/bin:${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:/usr/bin/:/bin/ + LOCAL_IPV4=$${slap-configuration:ipv4-random} + [phantomjs-wrapper] recipe = slapos.cookbook:wrapper diff --git a/software/slapos-testing/software.cfg b/software/slapos-testing/software.cfg index a1d503a392d6653401ff49c9402fa089b90858b0..beb80b33e696d538f3f710c9c77426bc8b12075b 100644 --- a/software/slapos-testing/software.cfg +++ b/software/slapos-testing/software.cfg @@ -6,26 +6,78 @@ extends = ../../component/libxml2/buildout.cfg ../../component/libxslt/buildout.cfg ../../component/bcrypt/buildout.cfg - ../../component/python-2.7/buildout.cfg - ../../component/python-setuptools/buildout.cfg ../../component/zlib/buildout.cfg ../../component/phantomjs/buildout.cfg ../../component/pycurl/buildout.cfg + ../../component/coreutils/buildout.cfg ../../stack/slapos.cfg + ./buildout.hash.cfg parts = - caucase-repository - slapos.cookbook-repository - slapos.core-repository - slapos.recipe.template-repository - slapos.recipe.build-repository - slapos.recipe.cmmi-repository - slapos.toolbox-repository - erp5-util-repository + bootstrap-slapos.recipe.cmmi eggs phantomjs template +[bootstrap-slapos.recipe.cmmi] +# install our develop version of slapos.recipe.cmmi before anything else, +# otherwise it will be installed from pypi by dependencies. +recipe = zc.recipe.egg +eggs = ${slapos.recipe.cmmi-setup:egg} + +[setup-develop-egg] +recipe = zc.recipe.egg:develop + +[caucase-setup] +<= setup-develop-egg +egg = caucase +setup = ${caucase-repository:location} + +[erp5.util-setup] +<= setup-develop-egg +# XXX erp5.util does not have `test` extra require, but has a `testnode` extra require with same dependencies +egg = erp5.util[testnode] +setup = ${erp5.util-repository:location} +depends = ${slapos.core-setup:egg} + +[slapos.cookbook-setup] +<= setup-develop-egg +# XXX slapos.cookbook does not have `test` extra require, `mock` is only listed in `tests_require` and is listed explicitly +egg = slapos.cookbook +setup = ${slapos.cookbook-repository:location} +depends = ${slapos.core-setup:egg} + +[slapos.core-setup] +<= setup-develop-egg +# XXX slapos.cookbook does not have `test` extra require, `mock`, `pyflakes` and `httmock` are only listed in `tests_require` and are listed explicitly +egg = slapos.core +setup = ${slapos.core-repository:location} + +[slapos.recipe.build-setup] +<= setup-develop-egg +egg = slapos.recipe.build[test] +setup = ${slapos.recipe.build-repository:location} + +[slapos.recipe.cmmi-setup] +<= setup-develop-egg +egg = slapos.recipe.cmmi[test] +setup = ${slapos.recipe.cmmi-repository:location} +depends = ${slapos.recipe.build-setup:egg} + +[slapos.recipe.template-setup] +<= setup-develop-egg +# XXX slapos.recipe.template does not have `test` extra require, `zope.testing` is only listed in `tests_require` and is listed explicitly +egg = slapos.recipe.template +setup = ${slapos.recipe.template-repository:location} + +[slapos.toolbox-setup] +<= setup-develop-egg +# XXX slapos.toolbox does not have `test` extra require, `mock` and `pycurl` are only listed in `tests_require` and are listed explicitly +egg = slapos.toolbox +setup = ${slapos.toolbox-repository:location} +depends = ${slapos.core-setup:egg} + + [eggs] recipe = zc.recipe.egg eggs = @@ -35,18 +87,24 @@ eggs = ${bcrypt:egg} dnspython Jinja2 - caucase - erp5.util - slapos.cookbook - collective.recipe.template - plone.recipe.command - slapos.recipe.template - slapos.recipe.cmmi - slapos.toolbox + ${caucase-setup:egg} + ${erp5.util-setup:egg} + ${slapos.cookbook-setup:egg} + ${slapos.core-setup:egg} + ${slapos.recipe.build-setup:egg} + ${slapos.recipe.cmmi-setup:egg} + ${slapos.recipe.template-setup:egg} + ${slapos.toolbox-setup:egg} + mock + zope.testing + httmock + pyflakes entry-points = runTestSuite=erp5.util.testsuite:runTestSuite scripts = runTestSuite +interpreter= + python_for_test [git-clone-repository] recipe = slapos.recipe.build:gitclone @@ -58,6 +116,10 @@ branch = master <= git-clone-repository repository = https://lab.nexedi.com/nexedi/caucase.git +[erp5.util-repository] +<= git-clone-repository +repository = https://lab.nexedi.com/nexedi/erp5.git + [slapos.cookbook-repository] <= git-clone-repository repository = https://lab.nexedi.com/nexedi/slapos.git @@ -73,28 +135,38 @@ repository = https://lab.nexedi.com/nexedi/slapos.recipe.template.git [slapos.recipe.build-repository] <= git-clone-repository repository = https://lab.nexedi.com/nexedi/slapos.recipe.build.git +# We use the system git and not slapos provided one, because +# slapos.recipe.build is a dependency of slapos.recipe.cmmi +#git-executable = git [slapos.recipe.cmmi-repository] <= git-clone-repository repository = https://lab.nexedi.com/nexedi/slapos.recipe.cmmi.git +# We use the system git and not slapos provided one, because slapos git needs +# slapos.recipe.cmmi to be installed. This circular dependency cause parts to +# be reinstalled everytime buildout is run because signatures are not stable. +#git-executable = git [slapos.toolbox-repository] <= git-clone-repository repository = https://lab.nexedi.com/nexedi/slapos.toolbox.git -[erp5-util-repository] -<= git-clone-repository -repository = https://lab.nexedi.com/nexedi/erp5.git [template] recipe = slapos.recipe.template -url = ${:_profile_base_location_}/instance.cfg -md5sum = 6626794c9dbb2530bb8ba3d331e27542 -output = ${buildout:directory}/template.cfg +url = ${:_profile_base_location_}/${:filename} +output = ${buildout:directory}/template.cfg mode = 640 [versions] Pygments = 2.1.3 -collective.recipe.template = 1.10 -plone.recipe.command = 1.1 -slapos.recipe.template = 4.3 + +# clear the version of tested eggs, to make sure we installed the developped ones +caucase = +erp5.util = +slapos.cookbook = +slapos.core = +slapos.recipe.build = +slapos.recipe.cmmi = +slapos.recipe.template = +slapos.toolbox = diff --git a/software/slaprunner/software.cfg b/software/slaprunner/software.cfg index 277ca9e183236713401bccddc5de0b955450f8a7..101d358641fdcbb272a04ef7399796d70890938f 100644 --- a/software/slaprunner/software.cfg +++ b/software/slaprunner/software.cfg @@ -16,7 +16,7 @@ gunicorn = 19.7.1 prettytable = 0.7.2 pycurl = 7.43.0 slapos.recipe.template = 4.3 -slapos.toolbox = 0.74 +slapos.toolbox = 0.76 smmap = 0.9.0 # Required by: diff --git a/stack/caucase/buildout.cfg b/stack/caucase/buildout.cfg index 4717fb70c1714fbbebc202e7c43f53c7760df53c..e894d4ce8e95ca097d78db2b64b4e6a582dbab8a 100644 --- a/stack/caucase/buildout.cfg +++ b/stack/caucase/buildout.cfg @@ -95,7 +95,7 @@ futures = 3.1.1 gitdb2 = 2.0.2 gunicorn = 19.7.1 slapos.recipe.template = 4.3 -slapos.toolbox = 0.74 +slapos.toolbox = 0.76 smmap2 = 2.0.3 # Required by: diff --git a/stack/erp5/buildout.cfg b/stack/erp5/buildout.cfg index 5096b472028664a75f1a12000cf441eb10f54f58..d9b02924b25bbc392b211be6e81c64a0502dc71d 100644 --- a/stack/erp5/buildout.cfg +++ b/stack/erp5/buildout.cfg @@ -583,7 +583,7 @@ extra-paths = # patches for eggs patch-binary = ${patch:location}/bin/patch -Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic.patch#e8029103350dad364d25747514a20327 +Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic.patch#1d9a56e9af4371f5b6951ebf217a15d7 Acquisition-patch-options = -p1 Products.DCWorkflow-patches = ${:_profile_base_location_}/../../component/egg-patch/Products.DCWorkflow/workflow_method.patch#975b49e96bae33ac8563454fe5fa9899 Products.DCWorkflow-patch-options = -p1 @@ -625,7 +625,7 @@ scripts += # neoppod, mysqlclient, slapos.recipe.template & [slapos-deps-eggs] # patched eggs -Acquisition = 2.13.9+SlapOSPatched001 +Acquisition = 2.13.12+SlapOSPatched001 Products.DCWorkflow = 2.2.4+SlapOSPatched001 ocropy = 1.0+SlapOSPatched001 pysvn = 1.7.10+SlapOSPatched002 diff --git a/stack/erp5/zope-versions.cfg b/stack/erp5/zope-versions.cfg index d10d80bd055e1e67da505f94c37cc2ac853fcd41..cc80b1ec4b937efb44df70b06a4d5ac41ba07be7 100644 --- a/stack/erp5/zope-versions.cfg +++ b/stack/erp5/zope-versions.cfg @@ -1,6 +1,6 @@ [versions] -AccessControl = 2.13.15 -Acquisition = 2.13.11 +AccessControl = 2.13.16 +Acquisition = 2.13.12 DateTime = 2.12.8 DocumentTemplate = 2.13.4 ExtensionClass = 2.13.2 @@ -21,31 +21,35 @@ Products.Sessions = 3.0 Products.StandardCacheManagers = 2.13.1 Products.TemporaryFolder = 3.0 Products.ZCTextIndex = 2.13.5 -Products.ZCatalog = 2.13.27 +Products.ZCatalog = 2.13.29 Pygments = 2.2.0 Record = 2.13.0 RestrictedPython = 3.6.0 Sphinx = 1.0.8 ZConfig = 2.9.3 ZODB3 = 3.10.7 -Zope2 = 2.13.26 ZServer = 3.0 +Zope2 = 2.13.27 ZopeUndo = 2.12.0 docutils = 0.12 initgroups = 2.13.0 mechanize = 0.2.5 -mr.developer = 1.35 +mr.developer = 1.38 +pluggy = 0.6.0 +py = 1.5.2 +pytz = 2017.2 repoze.retry = 1.2 repoze.tm2 = 1.0 repoze.who = 2.0 tempstorage = 2.12.2 +tox = 2.9.1 transaction = 1.1.1 z3c.checkversions = 0.5 zExceptions = 2.13.0 zLOG = 2.11.2 zc.buildout = 2.3.1 zc.lockfile = 1.0.2 -zc.recipe.egg = 2.0.3 +zc.recipe.egg = 2.0.5 zc.recipe.testrunner = 1.2.1 zdaemon = 2.0.7 zope.annotation = 3.5.0 @@ -66,7 +70,7 @@ zope.exceptions = 3.6.2 zope.filerepresentation = 3.6.1 zope.i18n = 3.7.4 zope.i18nmessageid = 3.5.3 -zope.interface = 3.6.7 +zope.interface = 3.6.8 zope.lifecycleevent = 3.6.2 zope.location = 3.9.1 zope.pagetemplate = 3.5.2 @@ -87,14 +91,3 @@ zope.testbrowser = 3.11.1 zope.testing = 3.9.7 zope.traversing = 3.13.2 zope.viewlet = 3.7.2 - -# Required by: -# Jinja2==2.8.1 -MarkupSafe = 0.23 - -# Required by: -# DateTime==2.12.8 -# Zope2==2.13.26 -# zope.i18n==3.7.4 -# zope.testbrowser==3.11.1 -pytz = 2016.10 diff --git a/stack/monitor/README.md b/stack/monitor/README.md index a8d5860333adca9c640105f3e552e26375611a1b..cd5142445b7cb0af03bc26be6516ff012dfab57f 100644 --- a/stack/monitor/README.md +++ b/stack/monitor/README.md @@ -70,8 +70,8 @@ You don't need to define all parameters, you can only set what is required to be If you have sub-instances, you should collect the base monitor url from all instances with monitor and send it to monitor-url-list or you can override "monitor-base-url-dict" section and add all the urls as key/value pairs in the root instance. [monitor-base-url-list] - monitor1-url = https://[xxxx:xxx:xxxx:e:11::1fb1]:4200 - monitor2-url = https://[xxxx:xxx:xxxx:e:22::2fb2]:4200 + monitor1-url = https://[xxxx:xxx:xxxx:e:11::1fb1]:4200 + monitor2-url = https://[xxxx:xxx:xxxx:e:22::2fb2]:4200 .. .. @@ -84,66 +84,42 @@ NB: You should use double $ (ex: $${monitor-template:rendered}) instead of one $ Add a promise ------------- -Monitor stack will include slapos promise directory etc/promise to promise folder. All files in that directory will be considered as a promise. -This mean that all slapos promises will be checked frequently by monitor. +To learn how to write a promise in SlapOS, please read this document: + https://www.erp5.com/slapos-TechnicalNote.General.SlapOS.Monitoring.Specifications - [monitor-conf-parameters] - ... - promise-folder = ${directory:promises} - ... - - -Monitor will run each promise every minutes and save the result in a json file. Here is an exemple of promise result: - - {"status": "ERROR", "change-time": 1466415901.53, "hosting_subscription": "XXXX", "title": "vnc_promise", "start-date": "2016-06-21 10:47:01", "instance": "XXXX-title", "_links": {"monitor": {"href": "MONITOR_PRIVATE_URL"}}, "message": "PROMISE_OUPT_MESSAGE", "type": "status"} - -A promise will be ran during a short time and report the status: ERROR or OK, plus an ouput message which says what was good or bad. -The promise should not run for more that 20 seconds, else it will be interrupted because of time out. However this value can be modified from monitoring web interface, see parameter "promise-timeout" of your hosting subscription. -On slapos, the default timeout value is also 20 seconds, if the value is modified on monitor (ex: to 50 seconds), it will still fails when slapgrid will process instance if the promise execution exceed 20 seconds. - -Promises result are published in web public folder, access URL is: MONITOR_BASE_URL/private/PROMISE_NAME.status.json -Everytime monitor will run a promise an history of result will be also updated. The promise history will be updated during one day, after that a new history will be created. -To access promise history file as JSON, use URL MONITOR_BASE_URL/private/PROMISE_NAME.history.json - -Add a promise: monitor promise ------------------------------- - -Monitor promise is also a promise like normal promise script but it will be placed into the folder ${monitor-directory:promises}: - - [monitor-promise-xxxxx] - recipe = slapos.recipe.template:jinja2 - rendered = ${monitor-directory:promises}/my-custom-monitor-promise - -Theses promises will be executed only by monitor (not slapos) every minutes and will report same infor as default promises. This is another way to -add more custom promises to check if server is overloaded, or if network start to be slow, etc... +Writing a promise consists of defining a class called RunPromise which inherits from GenericPromise class and defining methods: anomaly(), sense() and test(). Python promises should be placed into the folder etc/plugin of the computer partition. +New promises should be placed into the folder etc/plugin, legacy promise are into the folder etc/promise. Legacy promises are bash or other executable promises script which does not use GenericPromise class. +You will use slapos.cookbook:promise.plugin to generate your promise script into `etc/plugin` directory. Add promise will look like this: -Add custom scripts to monitor ------------------------------ + [promise-check-site] + recipe = slapos.cookbook:promise.plugin + eggs = + slapos.toolbox + output = ${directory:plugins}/promise-check-mysite-status.py + content = + from slapos.promise.plugin.check_site_state import RunPromise + config-site-url = ${publish:site-url} + config-connection-timeout = 20 + config-foo = bar + mode = 600 -Custom script will be automatically run by the monitor and result will be reported in monitor private folder. To add your custom script in ${monitor-directory:reports} folder. Here is an example: +Then you will have to add `promise-check-site` section to buildout parts, so it will be installed. - [monitor-check-webrunner-internal-instance] - recipe = slapos.recipe.template:jinja2 - template = ${monitor-check-webrunner-internal-instance:location}/${monitor-check-webrunner-internal-instance:filename} - rendered = $${monitor-directory:reports}/$${:filename} - filename = monitor-check-webrunner-internal-instance - mode = 0744 +In your promise code, you will be able to call `self.getConfig('site-url')`, `self.getConfig('connection-timeout')` and `self.getConfig('foo')`. The +returned value is `None` if the config parameter is not set. -The script will be executed every minutes by default. To change, put periodicity in script name: - - monitor-check-webrunner-internal-instance_every_1_minute - - monitor-check-webrunner-internal-instance_every_25_minute - - monitor-check-webrunner-internal-instance_every_1_hour - - monitor-check-webrunner-internal-instance_every_3_hour - - ... +Slapgrid will run each promise every time a partition is processed (every minutes in theory), if the partition is up to date, slapgrid will only run promises anomaly check and save the result in a json file. Here is an exemple of promise result: -the script name should end with _every_XX_hour or _every_XX_minute. With this, we can set filename as: + {"result": {"date": "2018-03-22T15:35:07", "failed": false, "message": "buildout is OK", "type": "Test Result"}, "path": "PARTITION_DIRECTORY/etc/plugin/buildout-slappart0-status.py", "name": "buildout-slappart0-status.py", "execution-time": 0.1, "title": "buildout-slappart0-status"} - filename = monitor-check-webrunner-internal-instance_every_2_minute +The promise execution time should be short, by default promise-timeout in slapgrid is to 20 seconds. If a promise runs in more than the defined promise-timeout, the process is killed and a "promise timed out" message is returned. +JSON in the folder `PARTITION_DIRECTORY/.slapgrid/promise/result`, and promise logs are in `PARTITION_DIRECTORY/.slapgrid/promise/log`. -You can get custom script results files at MONITOR_BASE_URL/private/FILE_NAME. +Monitor will expose promise JSON result in web public folder, access URL is: `MONITOR_BASE_URL/public/promise/PROMISE_TITLE.status.json`. Log files are exposed in monitor private web folder, +access URL is: `MONITOR_BASE_URL/private/log/monitor/promise` Add custom file or directory to monitor --------------------------------------- @@ -158,20 +134,22 @@ Log or others files can be added in monitor public or private directory: ${directory:log} ... -files in public directory are accessible at MONITOR_BASE_URL/public, and for private directory: MONITOR_BASE_URL/private. +files in public directory are accessible at `MONITOR_BASE_URL/public`, and for private directory: `MONITOR_BASE_URL/private`. -Monitor RSS and OPML Feed -------------------------- +Monitor promise history, RSS and OPML Feed +------------------------------------------ -Monitor generate rss containing latest result for all promises, this feed will be upaded every minutes. The Rss fee URL is -MONITOR_BASE_URL/public/feed +Monitor read every minutes JSON promises result files to build Rss and full instance state. The Rss feed URL is +`MONITOR_BASE_URL/public/feed` OPML Feed is used to aggregate many feed URL, this is used on monitor to link many single monitor instances. For example, a resilient -webrunner has 5 instances at least, each instance has a monitor which leads to 5 monitor instances too. One main instance (usally the root instance) -will collect rss feeds of all others monitor's in a single OPML file. This file is published and used to configure a monitor backend to the web interface. -The URL of OPML feed is: MONITOR_BASE_URL/public/feeds +webrunner has 3 instances at least, each instance has a monitor which leads to 3 monitor instances too. One main instance (usally the root instance) +will collect rss feed of all others monitor's in a single OPML file. This file is published and used to configure a monitor backend to the web interface. +The URL of OPML feed is: `MONITOR_BASE_URL/public/feeds` +Everytime monitor will also produce history of for each promise in a single JSON file. +To access promise history file as JSON, use URL `MONITOR_BASE_URL/public/PROMISE_TITLE.history.json` Monitor Base web directory tree ------------------------------- @@ -184,16 +162,13 @@ Monitor Base web directory tree (webdav) X Y | --------------------------------- - | | | | - public private jio_public jio_private - X Y | | - .jio_documents .jio_documents - X Y + | | + public private + X Y MONITOR_BASE_URL/public or private is for normal HTTPS. MONITOR_BASE_URL/share is the webdav URL. public/ and private/ are linked to public and private directories. - webdav also has jio_public/.jio_documents and jio_private/.jio_documents which are linked to public and private directory and they works with jio webdav pluging. Access to Monitor diff --git a/stack/monitor/buildout.cfg b/stack/monitor/buildout.cfg index 0c1a480ec855dcc003003851f5a169e03eb25b05..69455bb78bc4f2ad2b128cb99c667833254da434 100644 --- a/stack/monitor/buildout.cfg +++ b/stack/monitor/buildout.cfg @@ -44,6 +44,7 @@ eggs = collective.recipe.template cns.recipe.symlink slapos.toolbox + slapos.core # Do no generate any scripts here as all of them are generated by extraeggs scripts = @@ -72,22 +73,12 @@ md5sum = 1695c9a06a2b11ccfe893d7a224e489d [monitor-conf] <= monitor-template-base filename = monitor.conf.in -md5sum = 888e2845d09bfaa59c25f56f5bcf76b1 - -[monitor-instance-info] -<= monitor-template-base -filename = instance-info.conf.in -md5sum = 1bdb4e05c6be04f4e5766c64467fbcec +md5sum = 91c4c9bba1f7df788b9b7a059ed89ac2 [monitor-httpd-cors] <= monitor-template-base filename = httpd-cors.cfg.in md5sum = 683ea85fc054094248baf5752dd089bf - -[monitor-check-free-disk-space] -<= monitor-template-base -filename = check_free_disk.in -md5sum = bab457ac4d139ed31d0b343a7d14d996 # End templates files # XXX keep compatibility (with software/ipython_notebook/software.cfg ) @@ -112,7 +103,6 @@ context = raw monitor_configwrite ${buildout:directory}/bin/monitor.configwrite raw monitor_conf_template ${monitor-conf:location}/${monitor-conf:filename} raw monitor_https_cors ${monitor-httpd-cors:location}/${monitor-httpd-cors:filename} - raw monitor_instance_info ${monitor-instance-info:location}/${monitor-instance-info:filename} raw curl_executable_location ${curl:location}/bin/curl raw dash_executable_location ${dash:location}/bin/dash raw dcron_executable_location ${dcron:location}/sbin/crond @@ -122,7 +112,7 @@ context = raw python_executable ${buildout:executable} raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter} raw template_wrapper ${monitor-template-wrapper:location}/${monitor-template-wrapper:filename} - raw template_check_disk_space ${monitor-check-free-disk-space:location}/${monitor-check-free-disk-space:filename} + raw check_disk_space ${buildout:directory}/bin/check-free-disk raw bin_directory ${buildout:directory}/bin depends = ${monitor-eggs:eggs} @@ -131,6 +121,6 @@ depends = PyRSS2Gen = 1.1 cns.recipe.symlink = 0.2.3 pycurl = 7.43.0 -slapos.toolbox = 0.74 +slapos.toolbox = 0.76 pyasn1 = 0.3.7 diff --git a/stack/monitor/buildout.hash.cfg b/stack/monitor/buildout.hash.cfg index 9a88f8214b53cb8110afba52048cf2d5dd1ac85b..ba9e62e63a7ba619b40977bf90fd525cb696edc7 100644 --- a/stack/monitor/buildout.hash.cfg +++ b/stack/monitor/buildout.hash.cfg @@ -15,4 +15,4 @@ # not need these here). [monitor2-template] filename = instance-monitor.cfg.jinja2.in -md5sum = 75fe1b222c269e69226796bf6059a747 +md5sum = 86517ddccc86c76deaedf11066470550 diff --git a/stack/monitor/instance-monitor.cfg.jinja2.in b/stack/monitor/instance-monitor.cfg.jinja2.in index c57bd8309e9fe447fac0621bf13040ad4342928e..0be3fc450ca4a1aaf3b92105a39747cc32868429 100644 --- a/stack/monitor/instance-monitor.cfg.jinja2.in +++ b/stack/monitor/instance-monitor.cfg.jinja2.in @@ -21,28 +21,23 @@ log = ${:var}/log scripts = ${:etc}/run services = ${:etc}/service promises = ${:etc}/promise +plugins = ${:etc}/plugin monitor = ${:srv}/monitor -monitor-promise = ${:etc}/monitor-promise -monitor-report = ${:etc}/monitor-report [monitor-directory] recipe = slapos.cookbook:mkdirectory bin = ${directory:bin} etc = ${directory:etc} -run = ${directory:monitor}/run -#run = ${directory:scripts} -promises = ${directory:monitor-promise} -reports = ${directory:monitor-report} +promises = ${directory:etc}/monitor-promise +reports = ${directory:etc}/monitor-report pids = ${directory:run}/monitor -cgi-bin = ${directory:monitor}/cgi-bin webdav = ${directory:monitor}/webdav public = ${directory:monitor}/public private = ${directory:monitor}/private -services = ${directory:services} -services-conf = ${directory:etc}/monitor.conf.d +documents = ${:private}/documents log = ${directory:log}/monitor -monitor-var = ${directory:var}/monitor -monitor-log = ${directory:monitor}/private/monitor-log +promise-result = ${buildout:directory}/.slapgrid/promise/result +promise-log = ${buildout:directory}/.slapgrid/promise/log [ca-directory] recipe = slapos.cookbook:mkdirectory @@ -58,7 +53,7 @@ recipe = slapos.cookbook:certificate_authority openssl-binary = {{ openssl_executable_location }} ca-dir = ${ca-directory:root} requests-directory = ${ca-directory:requests} -wrapper = ${monitor-directory:services}/certificate_authority +wrapper = ${directory:services}/certificate_authority ca-private = ${ca-directory:private} ca-certs = ${ca-directory:certs} ca-newcerts = ${ca-directory:newcerts} @@ -78,18 +73,12 @@ root-title = ${monitor-instance-parameter:root-instance-title} public-folder = ${monitor-directory:public} private-folder = ${monitor-directory:private} webdav-folder = ${monitor-directory:webdav} -report-folder = ${monitor-directory:reports} base-url = ${monitor-instance-parameter:monitor-base-url} -monitor-hal-json = ${monitor-directory:public}/monitor.hal.json service-pid-folder = ${monitor-directory:pids} crond-folder = ${logrotate-directory:cron-entries} -logrotate-folder = ${logrotate-directory:logrotate-entries} -promise-runner = {{ monitor_runpromise }} -promise-folder = ${directory:promises} -monitor-promise-folder = ${monitor-directory:promises} -promises-timeout-file = ${monitor-promise-timeout-file:file} +log-folder = ${monitor-directory:log} +document-folder = ${monitor-directory:documents} pid-file = ${monitor-directory:pids}/monitor-bootstrap.pid -randomsleep = {{ bin_directory }}/randomsleep public-path-list = private-path-list = ${directory:log} @@ -99,26 +88,28 @@ parameter-file-path = ${monitor-instance-parameter:configuration-file-path} parameter-list = raw monitor-user ${monitor-instance-parameter:username} htpasswd monitor-password ${httpd-monitor-htpasswd:password-file} ${monitor-instance-parameter:username} ${httpd-monitor-htpasswd:htpasswd-path} - file promise-timeout ${monitor-promise-timeout-file:file} file min-free-disk-MB ${promise-check-free-disk-space:config-file} ${monitor-instance-parameter:instance-configuration} # htpasswd entry: htpasswd key password-file username htpasswd-file -collector-db = ${monitor-instance-parameter:collector-db} -collect-script = {{ monitor_collect }} -statistic-script = {{ monitor_statistic }} -python = {{ python_with_eggs }} -nice-cmd = ${xnice-bin:output} - promise-output-file = ${directory:monitor}/monitor-bootstrap-status -[monitor-promise-timeout-file] -recipe = plone.recipe.command -file = ${directory:etc}/promise_timeout -command = - if [ ! -s "${:file}" ]; then - echo "20" > ${:file} - fi +[monitor-promise-conf] +output-folder = ${monitor-directory:public}/promise +history-folder = ${monitor-directory:public} +promise-folder = ${directory:plugins} +legacy-promise-folder = ${directory:promises} +pid-path = ${monitor-directory:pids}/runpromise.pid +partition-folder = ${buildout:directory} +master-url = ${slap-connection:server-url} +partition-cert = ${slap-connection:cert-file} +partition-key = ${slap-connection:key-file} +partition-id = ${slap-connection:partition-id} +computer-id = ${slap-connection:computer-id} +ipv4 = ${slap-configuration:ipv4-random} +ipv6 = ${slap-configuration:ipv6-random} +software-release = ${slap-connection:software-release-url} +software-type = ${slap-configuration:slap-software-type} [monitor-base-url-dict] # place holder to be used to collect erp5 monitor urls @@ -128,40 +119,15 @@ recipe = slapos.recipe.template:jinja2 template = {{ monitor_conf_template }} rendered = ${directory:etc}/${:filename} filename = monitor.conf -context = section parameter_dict monitor-conf-parameters - section monitor_base_urls monitor-base-url-dict - -[instance-info-parameters] -name = ${monitor-instance-parameter:monitor-title} -root-name = ${monitor-instance-parameter:root-instance-title} -computer-id = ${slap-connection:computer-id} -ipv4 = ${slap-configuration:ipv4-random} -ipv6 = ${slap-configuration:ipv6-random} -software-release = ${slap-connection:software-release-url} -software-type = ${slap-configuration:slap-software-type} -partition-id = ${slap-connection:partition-id} - -[monitor-instance-info] -recipe = slapos.recipe.template:jinja2 -template = {{ monitor_instance_info }} -rendered = ${directory:etc}/${:filename} -filename = instance-info.conf -context = - section instance_dict instance-info-parameters - -[python-symlink] -recipe = plone.recipe.command -target = ${directory:bin} -command = ln -sf {{ python_with_eggs }} ${:target}/python -update-command = ${:command} +context = section parameter_dict monitor-conf-parameters + section promise_parameter_dict monitor-promise-conf + section monitor_base_urls monitor-base-url-dict [start-monitor] recipe = slapos.cookbook:wrapper -command-line = {{ python_executable }} {{ monitor_bin }} --config_file ${monitor-conf:rendered} +command-line = {{ monitor_bin }} -c ${monitor-conf:rendered} name = bootstrap-monitor wrapper-path = ${directory:scripts}/${:name} -environment = - PATH=${python-symlink:target}:/usr/local/bin:/usr/bin:/bin [monitor-htpasswd] recipe = slapos.cookbook:generate.password @@ -183,12 +149,18 @@ update-command = ${:command} user = ${monitor-instance-parameter:username} password = ${monitor-instance-parameter:password} +[monitor-symlink] +recipe = cns.recipe.symlink +symlink = + ${monitor-directory:promise-result} = ${monitor-directory:public}/promise + ${monitor-directory:promise-result} = ${monitor-directory:log}/promise + [monitor-httpd-conf-parameter] listening-ip = ${monitor-instance-parameter:monitor-httpd-ipv6} port = ${monitor-instance-parameter:monitor-httpd-port} pid-file = ${directory:run}/monitor-httpd.pid -access-log = ${monitor-directory:log}/monitor-httpd-access.log -error-log = ${monitor-directory:log}/monitor-httpd-error.log +access-log = ${directory:log}/monitor-httpd-access.log +error-log = ${directory:log}/monitor-httpd-error.log cert-file = ${ca-directory:certs}/httpd.crt key-file = ${ca-directory:certs}/httpd.key htpasswd-file = ${httpd-monitor-htpasswd:htpasswd-path} @@ -235,8 +207,8 @@ command = kill -USR1 $(cat ${monitor-httpd-conf-parameter:pid-file}) [logrotate-entry-monitor-httpd] <= logrotate-entry-base name = monitor-apache -log = $${basedirectory:log}/monitor/monitor-httpd-error.log -post = test ! -s $${buildout:directory}/var/run/monitor-httpd.pid || $${buildout:directory}/bin/slapos-kill --pidfile $${buildout:directory}/var/run/monitor-httpd.pid -s USR1 +log = ${directory:log}/monitor-httpd-*.log +post = test ! -s ${monitor-httpd-conf-parameter:pid-file} || {{ bin_directory }}/slapos-kill --pidfile ${monitor-httpd-conf-parameter:pid-file} -s USR1 [xnice-bin] recipe = collective.recipe.template @@ -253,28 +225,74 @@ wrapper-path = ${directory:promises}/promise-monitor-httpd-is-process-older-than [monitor-globalstate-wrapper] recipe = slapos.cookbook:wrapper -command-line = ${xnice-bin:output} {{ monitor_genstatus }} '${monitor-conf:rendered}' '${monitor-instance-info:rendered}' +command-line = ${xnice-bin:output} {{ monitor_genstatus }} '${monitor-conf:rendered}' wrapper-path = ${directory:bin}/monitor-globalstate [monitor-configurator-wrapper] recipe = slapos.cookbook:wrapper # XXX - hard coded path -command-line = {{ monitor_configwrite }} --config_folder '${monitor-conf-parameters:private-folder}/config/.jio_documents' --output_cfg_file '${monitor-instance-parameter:configuration-file-path}' --htpasswd_bin '{{ apache_location }}/bin/htpasswd' --monitor_https_cors {{ monitor_https_cors }} +command-line = ${xnice-bin:output} {{ monitor_configwrite }} + --config_folder '${monitor-conf-parameters:private-folder}/config/.jio_documents' + --output_cfg_file '${monitor-instance-parameter:configuration-file-path}' + --htpasswd_bin '{{ apache_location }}/bin/htpasswd' + --monitor_https_cors {{ monitor_https_cors }} wrapper-path = ${directory:bin}/monitor-configurator +[monitor-collect-wrapper] +recipe = slapos.cookbook:wrapper +command-line = ${xnice-bin:output} {{ monitor_collect }} + --output_folder ${monitor-directory:documents} + --collector_db ${monitor-instance-parameter:collector-db} + --pid_file ${monitor-directory:pids}/monitor-collect.pid +wrapper-path = ${directory:bin}/monitor-collect + [monitor-globalstate-cron-entry] recipe = slapos.cookbook:cron.d cron-entries = ${cron:cron-entries} name = monitor-globalstate -frequency = * * * * * -command = {{ bin_directory }}/randomsleep 60 && ${monitor-globalstate-wrapper:wrapper-path} +frequency = */2 * * * * +command = {{ bin_directory }}/randomsleep 20 && ${monitor-globalstate-wrapper:wrapper-path} [monitor-configurator-cron-entry] recipe = slapos.cookbook:cron.d cron-entries = ${cron:cron-entries} name = monitor-configurator frequency = * * * * * -command = {{ bin_directory }}/randomsleep 60 && ${monitor-configurator-wrapper:wrapper-path} +command = {{ bin_directory }}/randomsleep 10 && ${monitor-configurator-wrapper:wrapper-path} + +[monitor-collect-cron-entry] +recipe = slapos.cookbook:cron.d +cron-entries = ${cron:cron-entries} +name = monitor_collect +frequency = * * * * * +command = {{ bin_directory }}/randomsleep 40 && ${monitor-collect-wrapper:wrapper-path} + +[logrotate-entry-monitor-data] +recipe = collective.recipe.template +name = monitor.data +log = ${monitor-directory:private}/*.data.json ${monitor-directory:documents}/*.data.json +input = inline:${:log} { + weekly + nocreate + olddir ${monitor-directory:documents} + rotate 104 + nocompress + missingok + extension .json + dateext + dateformat -%Y-%m-%d + notifempty + } +output = ${logrotate-directory:logrotate-entries}/${:name} +mode = 600 + +[logrotate-entry-monitor-promise-history] +<= logrotate-entry-base +name = monitor.service.status +log = ${monitor-directory:public}/*.history.json +rotate-num = 0 +frequency = weekly +pre = {{ monitor_statistic }} --history_folder ${monitor-directory:public} [monitor-httpd-promise] recipe = slapos.cookbook:check_url_available @@ -336,58 +354,35 @@ curl_path = {{ curl_executable_location }} check-secure = 1 [monitor-bootstrap-promise] -recipe = collective.recipe.template +recipe = slapos.cookbook:promise.plugin +eggs = + slapos.toolbox file = ${monitor-conf-parameters:promise-output-file} -error-log-file = ${buildout:directory}/.${slap-connection:partition-id}_${start-monitor:name}.log -input = inline:#!{{ dash_executable_location }} - pidfile=${monitor-conf-parameters:pid-file} - if [ -s $pidfile ]; then - COUNTER=0 - # Wait until max 20 seconds, the limit promise timeout - while [ $COUNTER -lt 40 ]; do - if [ -n "$(ps -p $(cat $pidfile) -o pid=)" ]; then - ((COUNTER=COUNTER+1)) - sleep 0.5 - else - break - fi - done - fi - if [ ! -f "${:file}" ]; then - echo "Monitor bootstrap exited with error." - log_file="${:error-log-file}" - if [ -s "$log_file" ]; then - echo " ---- Latest monitor-boostrap.log ----" - echo "" - tail -n 3 $log_file - fi - exit 2 - else - echo "Bootstrap OK"; - fi -output = ${directory:promises}/monitor-bootstrap-status -mode = 700 +content = + from slapos.promise.plugin.monitor_bootstrap_status import RunPromise +output = ${directory:plugins}/monitor-bootstrap-status.py +mode = 600 +config-process-pid-file = ${monitor-conf-parameters:pid-file} +config-process-name = ${start-monitor:name} +config-status-file = ${:file} [promise-check-slapgrid] -recipe = collective.recipe.template -error-log-file = ${buildout:directory}/.slapgrid-${slap-connection:partition-id}-error.log -input = inline:#!/bin/sh - if [ -f "${:error-log-file}" ]; then - >&2 cat ${:error-log-file} - exit 1 - fi -output = ${monitor-directory:promises}/buildout-${slap-connection:partition-id}-status -mode = 700 +recipe = slapos.cookbook:promise.plugin +eggs = + slapos.toolbox +output = ${directory:plugins}/buildout-${slap-connection:partition-id}-status.py +content = + from slapos.promise.plugin.check_partition_deployment_state import RunPromise +config-monitor-url = ${monitor-instance-parameter:monitor-base-url} +mode = 600 [promise-check-free-disk-space] -recipe = slapos.recipe.template:jinja2 -template = {{ template_check_disk_space }} -rendered = ${monitor-directory:promises}/check-free-disk-space -mode = 0700 -context = - key config_file :config-file - raw home_path ${buildout:directory} - raw python_bin {{ python_with_eggs }} +recipe = slapos.cookbook:wrapper +command-line = {{ check_disk_space }} + --collectordb ${monitor-instance-parameter:collector-db} + --home_path ${buildout:directory} + --config ${:config-file} +wrapper-path = ${directory:promises}/check-free-disk-space config-file = ${directory:etc}/min-free-disk-size @@ -400,6 +395,7 @@ base-url = ${monitor-conf-parameters:base-url} depends = ${monitor-globalstate-cron-entry:name} ${monitor-configurator-cron-entry:name} + ${monitor-collect-cron-entry:name} ${cron-entry-logrotate:name} ${logrotate-entry-cron:name} ${certificate-authority:wrapper} @@ -408,9 +404,12 @@ depends = ${ca-monitor-httpd:wrapper} ${monitor-httpd-promise:filename} ${monitor-bootstrap-promise:file} - ${promise-check-slapgrid:output} + ${monitor-symlink:recipe} + ${promise-check-slapgrid:recipe} ${promise-monitor-httpd-is-process-older-than-dependency-set:wrapper-path} ${logrotate-entry-monitor-httpd:name} + ${logrotate-entry-monitor-data:name} + ${logrotate-entry-monitor-promise-history:name} [monitor-publish] monitor-base-url = ${monitor-publish-parameters:monitor-base-url} diff --git a/stack/monitor/templates/check_free_disk.in b/stack/monitor/templates/check_free_disk.in deleted file mode 100644 index 7226809878ff12f241c9cc7058a6556a64e2b4a0..0000000000000000000000000000000000000000 --- a/stack/monitor/templates/check_free_disk.in +++ /dev/null @@ -1,73 +0,0 @@ -#!{{ python_bin }} - -import os -import sys - -def free_space(path, fn): - while True: - try: - disk = os.statvfs(path) - return fn(disk) - except OSError: - pass - if os.sep not in path: - break - path = os.path.split(path)[0] - - -def user_free_space(path): - return free_space(path, lambda d: d.f_bsize * d.f_bavail) - -def check_inode_usage(path): - max_inode_usage = 97.99 # < 98% usage - st = os.statvfs(path) - usage_output = "" - total_inode = st.f_files - free_inode = st.f_ffree - usage = round((float(total_inode - free_inode) / total_inode), 4) * 100 - if usage > max_inode_usage: - return "Disk Inodes are widely used: %s%%" % usage - elif os.path.exists('/tmp'): - # check if /tmp is mounted on another disk than path - tmp_st = os.statvfs('/tmp') - if tmp_st.f_blocks != st.f_blocks: - tmp_usage = round((float(tmp_st.f_files - tmp_st.f_ffree) / tmp_st.f_files), 4) * 100 - if tmp_usage > max_inode_usage: - return "Disk Inodes are widely used: %s%%" % tmp_usage - return "" - - -if __name__ == '__main__': - home_path = '{{ home_path }}' - config_file = '{{ config_file }}' - min_free_size = 1024*1024*1024*2 # 2G by default - if os.path.exists(config_file): - with open(config_file) as f: - min_size_str = f.read().strip() - if min_size_str == '0': - # disable check - print "Free disk space check is disabled\n set a number up to 0 to enable!" - exit(0) - if min_size_str.isdigit(): - value = int(min_size_str) - if value >= 200: - # Minimum value is 200Mb, it's already low - min_free_size = int(min_size_str)*1024*1024 - else: - with open(config_file, 'w') as f: - f.write(str(min_free_size/(1024*1024))) - real_free_space = user_free_space(home_path) - if real_free_space > min_free_size: - inode_usage = check_inode_usage(home_path) - if inode_usage: - print inode_usage - exit(2) - print "Disk usage: OK" - exit(0) - - real_space_g = round(real_free_space/(1024.0*1024*1024), 2) - min_space_g = round(min_free_size/(1024.0*1024*1024), 2) - print 'Free disk space low: remaning %s G (threshold: %s G)' % ( - real_space_g, min_space_g) - print 'You can modify minimum value on your monitor interface.' - exit(1) diff --git a/stack/monitor/templates/instance-info.conf.in b/stack/monitor/templates/instance-info.conf.in deleted file mode 100644 index 241226d2157b7a18a568458b60f9aacfced71d27..0000000000000000000000000000000000000000 --- a/stack/monitor/templates/instance-info.conf.in +++ /dev/null @@ -1,9 +0,0 @@ -[instance] -name = {{ instance_dict['name'] }} -root-name = {{ instance_dict['root-name'] }} -computer = {{ instance_dict['computer-id'] }} -ipv4 = {{ instance_dict['ipv4'] }} -ipv6 = {{ instance_dict['ipv6'] }} -software-release = {{ instance_dict['software-release'] }} -software-type = {{ instance_dict['software-type'] }} -partition = {{ instance_dict['partition-id'] }} \ No newline at end of file diff --git a/stack/monitor/templates/monitor.conf.in b/stack/monitor/templates/monitor.conf.in index 613d6c431613269ce0645fb53288672e187554c4..469d27c76cbad46a257ded79371b11300f150768 100644 --- a/stack/monitor/templates/monitor.conf.in +++ b/stack/monitor/templates/monitor.conf.in @@ -11,4 +11,9 @@ monitor-url-list = {% for key, value in monitor_base_urls.items() -%} {{ ' ' ~ value }} {% endfor -%} -{% endif -%} +{% endif %} + +[promises] +{% for key, value in promise_parameter_dict.items() -%} +{{ key }} = {{ value.strip().replace("\n", "\n ") }} +{% endfor -%} \ No newline at end of file diff --git a/stack/resilient/buildout.hash.cfg b/stack/resilient/buildout.hash.cfg index 0add644165208490e9e9c3c9a1b53b3d6be041a5..5c8bd9d55bfc23e56fd9aedf0d282ef6281bfff6 100644 --- a/stack/resilient/buildout.hash.cfg +++ b/stack/resilient/buildout.hash.cfg @@ -15,7 +15,7 @@ # not need these here). [pbsready] filename = pbsready.cfg.in -md5sum = bceb082deb6b3ee9b015c8b04436b52c +md5sum = 367f8b8a09c3098844739f9b2856b912 [pbsready-import] filename = pbsready-import.cfg.in @@ -27,7 +27,7 @@ md5sum = c6c11db5372150019debb1ce519b907d [template-pull-backup] filename = instance-pull-backup.cfg.in -md5sum = 5f35d9c0d08a268ed0c0f0dbef114f42 +md5sum = 85e777bd349a5c881d36e747882aee8a [template-replicated] filename = template-replicated.cfg.in diff --git a/stack/resilient/instance-pull-backup.cfg.in b/stack/resilient/instance-pull-backup.cfg.in index 015b1b3478434be2e7b609f9009c31e986125e16..942f233f3c03c718bb0b8bb96fb53ee877b5f1da 100644 --- a/stack/resilient/instance-pull-backup.cfg.in +++ b/stack/resilient/instance-pull-backup.cfg.in @@ -136,13 +136,13 @@ notifier-url = http://[$${notifier:host}]:$${notifier:port} slave-instance-list = $${slap-parameter:slave_instance_list} ignore-known-hosts-file = $${slap-parameter:ignore-known-hosts-file} # To get a verbose feed about PBS state -instance-root-name = $${instance-info-parameters:root-name} +instance-root-name = $${monitor-instance-parameter:root-instance-title} log-url = $${monitor-publish-parameters:monitor-base-url}/private/notifier/ status-item-directory = $${directory:notifier-status-items} [pbs-resilient-status-feed] recipe = slapos.cookbook:wrapper -command-line = ${buildout:directory}/bin/generatefeed --output $${:feed-path} --status-item-path $${pbs:status-item-directory} --title "Status feed for $${instance-info-parameters:root-name}-PBS" --link $${pbs:log-url} +command-line = ${buildout:directory}/bin/generatefeed --output $${:feed-path} --status-item-path $${pbs:status-item-directory} --title "Status feed for $${monitor-instance-parameter:root-instance-title}-PBS" --link $${pbs:log-url} feed-path = $${directory:monitor-resilient}/pbs-status-rss wrapper-path = $${rootdirectory:bin}/resilient-genstatusrss.py diff --git a/stack/resilient/pbsready.cfg.in b/stack/resilient/pbsready.cfg.in index d92ec03ba35945c916f597204401c4c6cecc16c4..2eec9f52276db96e65ca0520a3d522a06c926d85 100644 --- a/stack/resilient/pbsready.cfg.in +++ b/stack/resilient/pbsready.cfg.in @@ -135,7 +135,7 @@ command = ${buildout:bin-directory}/pubsubserver --callbacks $${directory:notifi notifier-binary = ${buildout:bin-directory}/pubsubnotifier host = $${slap-network-information:global-ipv6} port = $${notifier-port:port} -instance-root-name = $${instance-info-parameters:root-name} +instance-root-name = $${monitor-instance-parameter:root-instance-title} log-url = $${monitor-publish-parameters:monitor-base-url}/resilient/notifier-status-rss status-item-directory = $${directory:notifier-status-items} context = diff --git a/stack/slapos.cfg b/stack/slapos.cfg index feade19bbfb366695406773153d5e892cdb9c604..a2a3af75d006ed15bc43e394b98f709edec17e93 100644 --- a/stack/slapos.cfg +++ b/stack/slapos.cfg @@ -131,8 +131,8 @@ pyparsing = 2.2.0 pytz = 2016.10 requests = 2.13.0 six = 1.10.0 -slapos.cookbook = 1.0.59 -slapos.core = 1.4.4 +slapos.cookbook = 1.0.62 +slapos.core = 1.4.7 slapos.extension.strip = 0.4 slapos.libnetworkcache = 0.15 slapos.rebootstrap = 4.1 @@ -144,7 +144,7 @@ xml-marshaller = 0.9.7 paramiko = 2.1.3 # Required by: -# slapos.core==1.4.4 +# slapos.core==1.4.7 Flask = 0.12 # Required by: @@ -160,11 +160,11 @@ functools32 = 3.2.3.post2 ipaddress = 1.0.18 # Required by: -# slapos.cookbook==1.0.59 +# slapos.cookbook==1.0.62 jsonschema = 2.6.0 # Required by: -# slapos.core==1.4.4 +# slapos.core==1.4.7 # XXX 'slapos node format' raises an exception with netifaces 0.10.5. netifaces = 0.10.4 @@ -177,15 +177,15 @@ packaging = 16.8 pycparser = 2.17 # Required by: -# slapos.core==1.4.4 +# slapos.core==1.4.7 supervisor = 3.3.3 # Required by: -# slapos.core==1.4.4 +# slapos.core==1.4.7 uritemplate = 3.0.0 # Required by: -# slapos.core==1.4.4 +# slapos.core==1.4.7 zope.interface = 4.3.3 [networkcache]