Commit 7be9ec30 authored by Alain Takoudjou's avatar Alain Takoudjou

Update Release Candidate

parents 29a41407 8b64fd3a
Changes
=======
1.0.62 (2018-04-10)
-------------------
* promise.plugin: new recipe for python promises plugin script generation
1.0.59 (2018-03-15)
-------------------
* librecipe.execute: fix convert process arguments to string formatting.
......
......@@ -34,9 +34,9 @@ md5sum = 2202b18f269ad606d70e1864857ed93c
[apache]
# inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/
recipe = slapos.recipe.cmmi
version = 2.4.29
version = 2.4.33
url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
md5sum = 0c599404ef6b69eee95bcd9fcd094407
md5sum = 6ef469d3f16fffeb688bc6e0346823e5
pre-configure =
cp -ar ${apr:location}/apr-${apr:version} srclib/apr/ &&
cp -ar ${apr-util:location}/apr-util-${apr-util:version} srclib/apr-util
......
diff -uNr Acquisition-2.13.8/src/Acquisition/_Acquisition.c Acquisition-2.13.8nxd001/src/Acquisition/_Acquisition.c
--- Acquisition-2.13.8/src/Acquisition/_Acquisition.c 2011-06-11 17:19:14.000000000 +0200
+++ Acquisition-2.13.8nxd001/src/Acquisition/_Acquisition.c 2013-10-31 16:24:55.665085888 +0100
@@ -448,6 +448,64 @@
diff -uNr Acquisition-2.13.12.orig/src/Acquisition/_Acquisition.c Acquisition-2.13.12/src/Acquisition/_Acquisition.c
--- Acquisition-2.13.12.orig/src/Acquisition/_Acquisition.c 2017-12-01 12:01:34.000000000 +0100
+++ Acquisition-2.13.12/src/Acquisition/_Acquisition.c 2018-04-09 17:10:15.394836944 +0200
@@ -449,6 +449,64 @@
}
static PyObject *
......@@ -66,7 +66,7 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/_Acquisition.c Acquisition-2.13.8nx
Wrapper_acquire(Wrapper *self, PyObject *oname,
PyObject *filter, PyObject *extra, PyObject *orig,
int explicit, int containment);
@@ -545,8 +603,8 @@
@@ -589,8 +647,8 @@
Py_XDECREF(r); Py_XDECREF(v); Py_XDECREF(tb);
r=NULL;
}
......@@ -77,7 +77,7 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/_Acquisition.c Acquisition-2.13.8nx
{
if (r==Acquired)
{
@@ -670,7 +728,7 @@
@@ -714,7 +772,7 @@
Py_XDECREF(r); Py_XDECREF(v); Py_XDECREF(tb);
r=NULL;
......@@ -86,7 +86,7 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/_Acquisition.c Acquisition-2.13.8nx
if (r == Acquired) {
Py_DECREF(r);
}
@@ -707,7 +765,7 @@
@@ -751,7 +809,7 @@
Wrapper_getattro(Wrapper *self, PyObject *oname)
{
if (self->obj || self->container)
......@@ -95,18 +95,18 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/_Acquisition.c Acquisition-2.13.8nx
/* Maybe we are getting initialized? */
return Py_FindAttr(OBJECT(self),oname);
@@ -724,7 +782,7 @@
return Py_FindAttr(OBJECT(self),oname);
@@ -776,7 +834,7 @@
result = Py_FindAttr(OBJECT(self),oname);
if (self->obj || self->container)
- return Wrapper_findattr(self, oname, NULL, NULL, NULL, 1, 0, 0, 0);
+ return Wrapper_findattr(self, oname, NULL, NULL, OBJECT(self), 1, 0, 0, 0);
else if (self->obj || self->container)
- result = Wrapper_findattr(self, oname, NULL, NULL, NULL, 1, 0, 0, 0);
+ result = Wrapper_findattr(self, oname, NULL, NULL, OBJECT(self), 1, 0, 0, 0);
/* Maybe we are getting initialized? */
return Py_FindAttr(OBJECT(self),oname);
diff -uNr Acquisition-2.13.8/src/Acquisition/test_dynamic_acquisition.py Acquisition-2.13.8nxd001/src/Acquisition/test_dynamic_acquisition.py
--- Acquisition-2.13.8/src/Acquisition/test_dynamic_acquisition.py 1970-01-01 01:00:00.000000000 +0100
+++ Acquisition-2.13.8nxd001/src/Acquisition/test_dynamic_acquisition.py 2013-10-31 16:24:55.665085888 +0100
else result = Py_FindAttr(OBJECT(self),oname);
diff -uNr Acquisition-2.13.12.orig/src/Acquisition/test_dynamic_acquisition.py Acquisition-2.13.12/src/Acquisition/test_dynamic_acquisition.py
--- Acquisition-2.13.12.orig/src/Acquisition/test_dynamic_acquisition.py 1970-01-01 01:00:00.000000000 +0100
+++ Acquisition-2.13.12/src/Acquisition/test_dynamic_acquisition.py 2018-04-09 17:07:34.863985305 +0200
@@ -0,0 +1,160 @@
+##############################################################################
+#
......@@ -268,10 +268,10 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/test_dynamic_acquisition.py Acquisi
+ >>> assert not b.a.aq_inContextOf('somestring')
+'''
+
diff -uNr Acquisition-2.13.8/src/Acquisition/tests.py Acquisition-2.13.8nxd001/src/Acquisition/tests.py
--- Acquisition-2.13.8/src/Acquisition/tests.py 2011-06-11 17:09:38.000000000 +0200
+++ Acquisition-2.13.8nxd001/src/Acquisition/tests.py 2013-10-31 16:24:55.669085888 +0100
@@ -2552,6 +2552,7 @@
diff -uNr Acquisition-2.13.12.orig/src/Acquisition/tests.py Acquisition-2.13.12/src/Acquisition/tests.py
--- Acquisition-2.13.12.orig/src/Acquisition/tests.py 2017-12-01 12:01:34.000000000 +0100
+++ Acquisition-2.13.12/src/Acquisition/tests.py 2018-04-09 17:07:34.867985476 +0200
@@ -2588,6 +2588,7 @@
def test_suite():
return unittest.TestSuite((
DocTestSuite(),
......@@ -279,10 +279,10 @@ diff -uNr Acquisition-2.13.8/src/Acquisition/tests.py Acquisition-2.13.8nxd001/s
DocFileSuite('README.txt', package='Acquisition'),
unittest.makeSuite(TestParent),
unittest.makeSuite(TestAcquire),
diff -uNr Acquisition-2.13.8/src/Acquisition.egg-info/SOURCES.txt Acquisition-2.13.8nxd001/src/Acquisition.egg-info/SOURCES.txt
--- Acquisition-2.13.8/src/Acquisition.egg-info/SOURCES.txt 2011-06-11 17:21:18.000000000 +0200
+++ Acquisition-2.13.8nxd001/src/Acquisition.egg-info/SOURCES.txt 2013-10-31 16:24:55.669085888 +0100
@@ -15,6 +15,7 @@
diff -uNr Acquisition-2.13.12.orig/src/Acquisition.egg-info/SOURCES.txt Acquisition-2.13.12/src/Acquisition.egg-info/SOURCES.txt
--- Acquisition-2.13.12.orig/src/Acquisition.egg-info/SOURCES.txt 2017-12-01 12:08:29.000000000 +0100
+++ Acquisition-2.13.12/src/Acquisition.egg-info/SOURCES.txt 2018-04-09 17:07:34.867985476 +0200
@@ -10,6 +10,7 @@
src/Acquisition/_Acquisition.c
src/Acquisition/__init__.py
src/Acquisition/interfaces.py
......
From 8b31ccec54584a287cc61501948283d7d6ee7073 Mon Sep 17 00:00:00 2001
From: Julien Muchembled <jm@nexedi.com>
Date: Mon, 26 Mar 2018 20:39:07 +0200
Subject: [PATCH] Enable TCP_NODELAY for inet(6) sockets
See commit 3d886d426243655b9f5a2528636e42b5c7662c19.
---
src/ZEO/zrpc/client.py | 2 ++
src/ZEO/zrpc/server.py | 1 +
2 files changed, 3 insertions(+)
diff --git a/src/ZEO/zrpc/client.py b/src/ZEO/zrpc/client.py
index 32a7a877..669f5962 100644
--- a/src/ZEO/zrpc/client.py
+++ b/src/ZEO/zrpc/client.py
@@ -568,6 +568,8 @@ def __init__(self, domain, addr, mgr, client):
self.close()
return
self.sock.setblocking(0)
+ if domain in (socket.AF_INET, socket.AF_INET6):
+ self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.state = "opened"
def connect_procedure(self):
diff --git a/src/ZEO/zrpc/server.py b/src/ZEO/zrpc/server.py
index b83cc004..af91e3e4 100644
--- a/src/ZEO/zrpc/server.py
+++ b/src/ZEO/zrpc/server.py
@@ -66,6 +66,7 @@ def _open_socket(self):
socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, True)
else:
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+ self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
else:
self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.set_reuse_addr()
--
2.14.1
......@@ -14,8 +14,8 @@ parts =
[gcc-common]
recipe = slapos.recipe.cmmi
url = http://ftp.gnu.org/gnu/gcc/gcc-5.4.0/gcc-5.4.0.tar.bz2
md5sum = 4c626ac2a83ef30dfb9260e6f59c2b30
url = http://ftp.gnu.org/gnu/gcc/gcc-5.5.0/gcc-5.5.0.tar.gz
md5sum = 781bc0195edeb0ceaace8428f63ae63d
# make install does not work when several core are used
make-targets = install -j1
......
......@@ -3,6 +3,7 @@
extends =
../findutils/buildout.cfg
../git/buildout.cfg
../pkgconfig/buildout.cfg
parts = gowork
......@@ -64,7 +65,15 @@ environment-extra =
# github.com/pkg/profile \
# golang.org/x/perf/cmd/benchstat
#
# it is possible to specify Go build flags used for compilation e.g. this way:
# For Cgo support pkg-config is made pre-available by gowork, and users
# should list paths where to search for pkg-config files, e.g. this way:
#
# [gowork]
# cpkgpath =
# ${sqlite3:location}/lib/pkgconfig
# ${zlib:location}/lib/pkgconfig
#
# It is also possible to specify Go build flags used for compilation e.g. this way:
#
# [gowork]
# buildflags = -race
......@@ -80,6 +89,9 @@ golang = ${golang19:location}
# no special build flags by default
buildflags =
# empty pkg-config path by default
cpkgpath =
# everything is done by dependent parts
recipe = plone.recipe.command
command = :
......@@ -93,7 +105,7 @@ recipe = slapos.recipe.template
url = ${:_profile_base_location_}/goenv.sh.in
output = ${gowork:directory}/env.sh
depends = ${gowork.mkdir:recipe}
md5sum = a9a265135931b3da53f4392870748264
md5sum = 7a067a3974c446c3eaa0e82818ba1adb
[gowork.mkdir]
# NOTE do not use slapos.cookbook:mkdirectory here - if anything in software (not instance)
......
......@@ -3,13 +3,16 @@
# ---- 8< ---- (buildout substitution here)
# PATH so that go & friends work out of the box
export PATH=${gowork:golang}/bin:${git:location}/bin:${buildout:bin-directory}:$PATH
export PATH=${gowork:golang}/bin:${git:location}/bin:${pkgconfig:location}/bin:${buildout:bin-directory}:$PATH
X=${gowork:directory}
export PKG_CONFIG_PATH=$(echo -n "${gowork:cpkgpath}" |tr '\n' ':'):$PKG_CONFIG_PATH
# ---- 8< ----
export GOPATH=$X:$GOPATH
export PATH=$X/bin:$PATH
export PS1="(`basename $X`) $PS1"
# strip trailing : from $GOPATH
# strip trailing : from $GOPATH, $PKG_CONFIG_PATH
GOPATH=$${GOPATH%:}
PKG_CONFIG_PATH=$${PKG_CONFIG_PATH%:}
......@@ -14,8 +14,8 @@ extends =
[groonga]
recipe = slapos.recipe.cmmi
url = http://packages.groonga.org/source/groonga/groonga-7.0.5.tar.gz
md5sum = d42dd98c5272ed2b9ab50ded54eb818b
url = http://packages.groonga.org/source/groonga/groonga-8.0.0.tar.gz
md5sum = f4641e68a5301e641399f2d112a494dd
# temporary patch to respect more tokens in natural language mode.
patches =
${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b
......
......@@ -14,6 +14,8 @@ configure-options =
--prefix=${:location}
--disable-static
--enable-rpath
patches =
${:_profile_base_location_}/fix-glibc2.26-ftbfs.patch#f1622be16964029fc66a70b8f9e1693c
[icu4c-slaposgcc]
# need for onlyoffice-core
......
--- source/i18n/digitlst.cpp 2018-03-29 10:59:45.706746334 +0900
+++ source/i18n/digitlst.cpp 2018-03-29 11:01:15.933534983 +0900
@@ -61,11 +61,7 @@
#endif
#if U_USE_STRTOD_L
-# if U_PLATFORM_USES_ONLY_WIN32_API || U_PLATFORM == U_PF_CYGWIN
-# include <locale.h>
-# else
-# include <xlocale.h>
-# endif
+# include <locale.h>
#endif
// ***************************************************************************
diff --git a/mrn_mysql_compat.h b/mrn_mysql_compat.h
index d2b227ff..337de968 100644
--- a/mrn_mysql_compat.h
+++ b/mrn_mysql_compat.h
@@ -362,6 +362,7 @@
#if defined(MRN_MARIADB_P) && \
((MYSQL_VERSION_ID >= 100207) || \
+ ((MYSQL_VERSION_ID >= 100126) && (MYSQL_VERSION_ID < 100200)) || \
((MYSQL_VERSION_ID >= 50557) && (MYSQL_VERSION_ID < 100000)))
# define mrn_create_partition_name(out, \
out_length, \
......@@ -28,8 +28,8 @@ parts =
[mariadb]
recipe = slapos.recipe.cmmi
url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
version = 10.1.31
md5sum = 14ab0398c019eb531bc29f2c437ccb51
version = 10.1.32
md5sum = 389ce891cf00957748ba98b09f433c14
patch-options = -p0
patches =
${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3
......@@ -84,8 +84,8 @@ post-install =
# mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
# http://mroonga.github.com/
recipe = slapos.recipe.cmmi
url = http://packages.groonga.org/source/mroonga/mroonga-7.05.tar.gz
md5sum = d289667a60a6aa78d8bc276ab61b2996
url = http://packages.groonga.org/source/mroonga/mroonga-8.00.tar.gz
md5sum = 8b906705be5fc14785207ca6752b96e9
pre-configure = set -e
rm -rf fake_mariadb_source
mkdir -p fake_mariadb_source
......@@ -106,7 +106,6 @@ configure-options =
patch-options = -p1
patches =
${:_profile_base_location_}/mroonga_boolean.patch#c818568fe35ca6a4298f18e575d962a0
${:_profile_base_location_}/1a910088ec55c61434029d8e1cc3c6192ac508d7.diff#a8b8df4dd870a1a269dfd9dc6941467b
pre-build =
sed -i -e "s,${mariadb:location}/include,$(pwd)/fake_mariadb_source/include,g" Makefile */Makefile
environment =
......
# Do not extend any file that touch buildout:parts.
[mariadb]
version = 10.2.13
md5sum = 20c61bd4059ba287e54cfb2862bae81d
version = 10.2.14
md5sum = d98cce6f3c0e2971afa061fc67183b91
stable-patches =
configure-options +=
-DCMAKE_C_COMPILER=${gcc:location}/bin/gcc
......
......@@ -5,7 +5,7 @@ parts =
[noVNC]
recipe = hexagonit.recipe.download
ignore-existing = true
# version-0.6.1 release from 04 Jul 2016
url = https://github.com/kanaka/noVNC/archive/v0.6.1.tar.gz
md5sum = d153c6aa69a9178081768fecbace1932
# version-1.0.0 release on Feb 22, 2018
url = https://github.com/novnc/noVNC/archive/v1.0.0.tar.gz
md5sum = d63c2944abd12fecec6ec504e82f27c8
strip-top-level-dir = true
[buildout]
extends =
../binutils/buildout.cfg
../gcc/buildout.cfg
../libxml2/buildout.cfg
../zlib/buildout.cfg
......@@ -15,12 +16,12 @@ recipe = slapos.recipe.cmmi
location = ${buildout:parts-directory}/${:_buildout_section_name_}
# This url contains the hash provided by the DocumentServer core submodule hash.
# https://github.com/ONLYOFFICE/DocumentServer/
url = https://lab.nexedi.com/bk/onlyoffice_core/repository/archive.tar.bz2?ref=b051e75b179b3599c09937668fbbd2d7e2c50683
md5sum = b2713373d687dd1c7121c286fa156626
url = https://lab.nexedi.com/bk/onlyoffice_core/repository/archive.tar.bz2?ref=8a40eb47bd80a40ecde14c223525b21852d2fc9f
md5sum = 9cd1cd731202511e475971eee58ba7b6
configure-command = true
make-targets = lib bin
environment =
PATH=${gcc:location}/bin:${qt5-qmake:location}/bin:%(PATH)s
PATH=${binutils:location}/bin:${gcc:location}/bin:${qt5-qmake:location}/bin:%(PATH)s
CXXFLAGS=-I${libxml2:location}/include -I${zlib:location}/include -I${icu4c-slaposgcc:location}/include -I${boost-lib:location}/include -Wno-comment -Wno-deprecated-declarations -Wno-endif-labels -Wno-parentheses -Wno-reorder -Wno-sign-compare -Wno-switch -Wno-unknown-pragmas -Wno-unused
LDFLAGS=-L${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib -L${gcc:location}/lib64 -Wl,-rpath=${gcc:location}/lib64 -L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${icu4c-slaposgcc:location}/lib -Wl,-rpath=${icu4c-slaposgcc:location}/lib -L${boost-lib:location}/lib -Wl,-rpath=${boost-lib:location}/lib -Wl,-rpath=${:location}/lib
post-install =
......
......@@ -16,8 +16,8 @@ parts =
[openssl]
recipe = slapos.recipe.cmmi
url = https://www.openssl.org/source/openssl-1.0.2n.tar.gz
md5sum = 13bdc1b1d1ff39b6fd42a255e74676a4
url = https://www.openssl.org/source/openssl-1.0.2o.tar.gz
md5sum = 44279b8557c3247cbe324e2322ecd114
location = ${buildout:parts-directory}/${:_buildout_section_name_}
# 'prefix' option to override --openssldir/--prefix (which is useful
# when combined with INSTALL_PREFIX). Used by slapos.package.git/obs
......
......@@ -64,7 +64,6 @@ md5sum = 096c1c18b44c269808bd815d58c53c8f
[debian-amd64-jessie-netinst.iso]
<= debian-amd64-netinst-base
release = archive
version = 8.10.0
md5sum = 19dcfc381bd3e609c6056216d203f5bc
......
......@@ -5,8 +5,8 @@ parts =
[sqlite3]
recipe = slapos.recipe.cmmi
url = http://sqlite.org/2017/sqlite-autoconf-3190000.tar.gz
md5sum = 2426883b4dd3a9fd6aeb28a16b81a72b
url = https://sqlite.org/2018/sqlite-autoconf-3220000.tar.gz
md5sum = 96b5648d542e8afa6ab7ffb8db8ddc3d
configure-options =
--disable-static
--enable-readline
......
......@@ -22,7 +22,7 @@ dist = ${vm-debian:dists}
[vm-debian]
recipe = slapos.recipe.build:vm.install-debian
environment = vm-install-environment
dists = debian-jessie
dists = debian-stretch
size = 1Gi
late-command =
# rdnssd causes too much trouble with recent QEMU, because the latter acts as
......@@ -55,18 +55,22 @@ preseed.recommends = false
preseed.tasks =
[debian-squeeze]
<= debian-jessie
<= debian-stretch
x86_64.iso = debian-amd64-squeeze-netinst.iso
[debian-wheezy]
<= debian-jessie
<= debian-stretch
x86_64.iso = debian-amd64-wheezy-netinst.iso
[debian-jessie]
<= debian-stretch
x86_64.iso = debian-amd64-jessie-netinst.iso
[debian-stretch]
x86_64.iso = debian-amd64-netinst.iso
x86_64.kernel = install.amd/vmlinuz
x86_64.initrd = install.amd/initrd.gz
[debian-stretch]
<= debian-jessie
[debian-buster]
<= debian-stretch
x86_64.iso = debian-amd64-testing-netinst.iso
......@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob
import os
version = '1.0.60.dev0'
version = '1.0.62'
name = 'slapos.cookbook'
long_description = open("README.rst").read() + "\n" + \
open("CHANGES.rst").read() + "\n"
......@@ -99,7 +99,6 @@ setup(name=name,
'dropbear.add_authorized_key = slapos.recipe.dropbear:AddAuthorizedKey',
'dropbear.client = slapos.recipe.dropbear:Client',
'duplicity = slapos.recipe.duplicity:Recipe',
'egg_test = slapos.recipe.erp5_test:EggTestRecipe',
'equeue = slapos.recipe.equeue:Recipe',
'erp5.promise = slapos.recipe.erp5_promise:Recipe',
'erp5.test = slapos.recipe.erp5_test:Recipe',
......@@ -151,6 +150,7 @@ setup(name=name,
'postgres.export = slapos.recipe.postgres.backup:ExportRecipe',
'postgres.import = slapos.recipe.postgres.backup:ImportRecipe',
'proactive = slapos.recipe.proactive:Recipe',
'promise.plugin= slapos.recipe.promise_plugin:Recipe',
'publish = slapos.recipe.publish:Recipe',
'publish.serialised = slapos.recipe.publish:Serialised',
'publish-early = slapos.recipe.publish_early:Recipe',
......
......@@ -113,26 +113,3 @@ class CloudoooRecipe(GenericBaseRecipe):
return path_list
class EggTestRecipe(GenericBaseRecipe):
"""
Recipe used to create wrapper used to run test suite (python setup.py test)
off a list of Python eggs.
"""
def install(self):
test_list = self.options['test-list'].strip().replace('\n', ',')
common_dict = {}
if self.options.get('environment'):
environment_part = self.buildout.get(self.options['environment'])
if environment_part:
common_dict['environment'] = dict(environment_part)
if 'prepend-path' in self.options:
common_dict['prepend_path'] = self.options['prepend-path']
return self.createPythonScript(
self.options['run-test-suite'], __name__ + '.test.runTestSuite',
((self.options['run-test-suite-binary'],
"--source_code_path_list", test_list),
common_dict)
)
##############################################################################
#
# Copyright (c) 2018 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import logging, os, sys
import zc.buildout.easy_install
from slapos.recipe.librecipe import GenericBaseRecipe
script_template = '''# This script is auto generated by slapgrid, do not edit!
import sys
sys.path[0:0] = [
%(path)s
]
extra_config_dict = {
%(config)s
}
# We want to cleanup all imported modules from slapos namespace, because
# they will conflict with slapos.core.
# In fact as slapos.grid is already imported, the promise cannot reimport
# his own slapos.grid from an updated sys.path. Then all module imports which
# are not in slapos.core will fail.
#
# call reload(slapos) only solve a part of problem because not all modules
# will be reloaded, and some new modules won't be added.
# The solution is to delete all cached 'slapos' modules as well as all cached
# 'pkg_resources' modules which is responsible of namespace declaration.
# They will be re-imported again using the updated sys.path
for module in sys.modules.keys():
if 'slapos' in module or 'pkg_resources' in module:
del sys.modules[module]
import slapos.grid.promise
%(content)s
'''
class Recipe(GenericBaseRecipe):
_WORKING_SET_CACHE_NAME = "slapos.cookbook_pplugin_ws_cache"
def __init__(self, buildout, name, options):
buildout_section = buildout['buildout']
options['eggs-directory'] = buildout_section['eggs-directory']
options['develop-eggs-directory'] = buildout_section['develop-eggs-directory']
super(Recipe, self).__init__(buildout, name, options)
def _get_cache_storage(self):
"""Return a mapping where to store generated working sets.
from https://github.com/buildout/buildout/blob/master/zc.recipe.egg_/src/zc/recipe/egg/egg.py#L170
"""
cache_storage = getattr(
self.buildout,
self._WORKING_SET_CACHE_NAME,
None
)
if cache_storage is None:
cache_storage = {}
setattr(
self.buildout,
self._WORKING_SET_CACHE_NAME,
cache_storage
)
return cache_storage
def install(self):
develop_eggs_dir = self.options['develop-eggs-directory']
eggs_dir = self.options['eggs-directory']
egg_list = [
egg.strip()
for egg in self.options['eggs'].split('\n')
if egg.strip()
]
cache_storage = self._get_cache_storage()
cache_key = (
tuple(egg_list),
eggs_dir,
develop_eggs_dir,
)
if cache_key not in cache_storage:
working_set = zc.buildout.easy_install.working_set(
egg_list,
[develop_eggs_dir, eggs_dir]
)
cache_storage[cache_key] = working_set
else:
working_set = cache_storage[cache_key]
content = self.options['content'].strip()
output = self.options['output']
mode = self.options.get('mode', '0600')
path_list_string = ""
for dist in working_set:
path_list_string += ' "%s",\n' % dist.location
content_string = '\n'.join([line.lstrip() for line in content.split('\n')])
config_string = ""
for key in self.options:
if key.startswith('config-'):
config_string += " '%s': '%s',\n" % (key[7:], self.options[key])
option_dict = dict(path=path_list_string.strip(),
content=content_string,
config=config_string.strip())
with open(output, 'w') as f:
f.write(script_template % option_dict)
os.chmod(output, int(mode, 8))
return (output,)
update = install
......@@ -4,8 +4,6 @@ import unittest
from mock import patch
from slapos.recipe import free_port
class SocketMock():
def __init__(self, *args, **kw):
self.args = args
......@@ -28,31 +26,16 @@ class FreePortTest(unittest.TestCase):
SocketMock.bind = SocketMock.close = SocketMock.nothing_happen
def new_recipe(self, **kw):
buildout = {
'buildout': {
'bin-directory': '',
'find-links': '',
'allow-hosts': '',
'develop-eggs-directory': '',
'eggs-directory': '',
'python': 'testpython',
'installed': '.installed.cfg',
},
'testpython': {
'executable': sys.executable,
},
'slap-connection': {
'computer-id': '',
'partition-id': '',
'server-url': '',
'software-release-url': '',
}
}
from slapos.recipe import free_port
from slapos.test.utils import makeRecipe
options = {
'ip': '127.0.0.1',
}
options.update(kw)
return free_port.Recipe(buildout=buildout, name='free_port', options=options)
return makeRecipe(
free_port.Recipe,
options=options,
name='free_port')
@useMock
def test_ifNoBusyPortThenMinPortIsAlwaysReturned(self):
......
......@@ -4,30 +4,15 @@ import unittest
from tempfile import mkdtemp
from shutil import rmtree
from slapos.recipe import generic_cloudooo
class TestGenericCloudooo(unittest.TestCase):
def new_recipe(self, options):
buildout = {
'buildout': {
'bin-directory': '',
'find-links': '',
'allow-hosts': '',
'develop-eggs-directory': '',
'eggs-directory': '',
'python': 'testpython',
},
'testpython': {
'executable': sys.executable,
},
'slap-connection': {
'computer-id': '',
'partition-id': '',
'server-url': '',
'software-release-url': '',
}
}
return generic_cloudooo.Recipe(buildout=buildout, name='generic_cloudooo', options=options)
from slapos.recipe import generic_cloudooo
from slapos.test.utils import makeRecipe
return makeRecipe(
generic_cloudooo.Recipe,
options=options,
name='generic_cloudooo')
def setUp(self):
self.test_dir = mkdtemp()
......
......@@ -5,37 +5,19 @@ import sys
import tempfile
import unittest
from slapos.recipe import pbs
class PBSTest(unittest.TestCase):
def new_recipe(self):
buildout = {
'buildout': {
'bin-directory': '',
'find-links': '',
'allow-hosts': '',
'develop-eggs-directory': '',
'eggs-directory': '',
'python': 'testpython',
},
'testpython': {
'executable': sys.executable,
},
'slap-connection': {
'computer-id': '',
'partition-id': '',
'server-url': '',
'software-release-url': '',
}
}
from slapos.recipe import pbs
from slapos.test.utils import makeRecipe
options = {
'rdiffbackup-binary': ''
}
return pbs.Recipe(buildout=buildout, name='pbs', options=options)
return makeRecipe(
pbs.Recipe,
options=options,
name='pbs')
def test_push(self):
recipe = self.new_recipe()
......
......@@ -6,8 +6,6 @@ import tempfile
import unittest
from slapos.slap.slap import NotFoundError, ConnectionError
from slapos.recipe import re6stnet
class Re6stnetTest(unittest.TestCase):
......@@ -47,31 +45,21 @@ class Re6stnetTest(unittest.TestCase):
shutil.rmtree(path)
def new_recipe(self):
buildout = {
'buildout': {
'bin-directory': '',
'find-links': '',
'allow-hosts': '',
'develop-eggs-directory': '',
'eggs-directory': '',
'python': 'testpython',
},
'testpython': {
'executable': sys.executable,
},
'slap-connection': {
from slapos.recipe import re6stnet
from slapos.test.utils import makeRecipe
return makeRecipe(
re6stnet.Recipe,
options=self.options,
slap_connection={
'computer-id': 'comp-test',
'partition-id': 'slappart0',
'server-url': 'http://server.com',
'software-release-url': 'http://software.com',
'key-file': '/path/to/key',
'cert-file': '/path/to/cert'
}
}
options = self.options
},
name='re6stnet')
return re6stnet.Recipe(buildout=buildout, name='re6stnet', options=options)
def checkWrapper(self, path):
self.assertTrue(os.path.exists(path))
......
"""Test helpers
"""
import sys
import os.path
from ConfigParser import ConfigParser
import logging
def makeRecipe(recipe_class, options, name='test', slap_connection=None):
"""Instanciate a recipe of `recipe_class` with `options` with a buildout
mapping containing a python and an empty `slapos-connection` mapping, unless
provided as `slap_connection`.
If running tests in a buildout folder, the test recipe will reuse the
`eggs-directory` and `develop-eggs-directory` from this buildout so that the
test recipe does not need to install eggs again when using working set.
To prevent test accidentally writing to the buildout's eggs repositories, we
set `newest` to false and `offline` to true in this case.
"""
buildout = {
'buildout': {
'bin-directory': '',
'find-links': '',
'allow-hosts': '',
'develop-eggs-directory': '',
'eggs-directory': '',
'python': 'testpython',
},
'testpython': {
'executable': sys.executable,
},
'slap-connection': {
'computer-id': '',
'partition-id': '',
'server-url': '',
'software-release-url': '',
}
}
if slap_connection is not None:
buildout['slap-connection'] = slap_connection
# are we in buildout folder ?
# the usual layout is
# ${buildout:directory}/parts/slapos-repository/slapos/test/utils.py , so try
# to find a buildout relative to this file.
buildout_cfg = os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', 'buildout.cfg')
if os.path.exists(buildout_cfg):
parser = ConfigParser()
parser.readfp(open(buildout_cfg))
eggs_directory = parser.get('buildout', 'eggs-directory')
develop_eggs_directory = parser.get('buildout', 'develop-eggs-directory')
logging.getLogger(__name__).info(
'Using eggs-directory (%s) and develop-eggs-directory (%s) from buildout at %s',
eggs_directory,
develop_eggs_directory,
buildout_cfg)
buildout['buildout']['eggs-directory'] = eggs_directory
buildout['buildout']['develop-eggs-directory'] = develop_eggs_directory
buildout['buildout']['newest'] = False
buildout['buildout']['offline'] = True
return recipe_class(buildout=buildout, name=name, options=options)
......@@ -51,7 +51,7 @@ pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.download = 1.0
slapos.recipe.template = 4.3
slapos.toolbox = 0.74
slapos.toolbox = 0.76
smmap = 0.9.0
# Required by:
......
......@@ -55,7 +55,7 @@ md5sum = d98a01182f38868612948c87d5231428
[template-default-slave-virtualhost]
filename = templates/default-virtualhost.conf.in
md5sum = 5344bff68a3f7dead633a35771ad1a19
md5sum = 7f38084af107034bedefba971abe165c
[template-cached-slave-virtualhost]
filename = templates/cached-virtualhost.conf.in
......
......@@ -11,7 +11,7 @@ plone.recipe.command = 1.1
pycrypto = 2.6.1
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.3
slapos.toolbox = 0.74
slapos.toolbox = 0.76
smmap = 0.9.0
numpy = 1.11.2
pyasn1 = 0.2.3
......
......@@ -69,7 +69,7 @@ def main():
test_title = args.test_suite_title or args.test_suite
if args.master_url:
tool = taskdistribution.TaskDistributionTool(args.master_url)
tool = taskdistribution.TaskDistributor(args.master_url)
test_result = tool.createTestResult(args.revision,
dist_list,
args.test_node_title,
......
......@@ -66,6 +66,7 @@ sparse-checkout = /playbook/roles/rina
[vm-debian]
# building a generic Debian kernel uses a lot of space
size = 16Gi
dists = debian-jessie
packages +=
# generic (another SR that build packages automatically would use the same list)
apt-utils build-essential devscripts equivs lsb-release
......
......@@ -10,7 +10,7 @@ extends =
../../component/netcat/buildout.cfg
../../component/pycurl/buildout.cfg
../../stack/slapos.cfg
../../stack/nodejs.cfg
../../component/nodejs/buildout.cfg
../../stack/resilient/buildout.cfg
# stacks are listed from most generic to most specific,
......@@ -99,7 +99,7 @@ recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
mode = 644
md5sum = 68b66fb3e9020642e57f4a4ee266f2b3
md5sum = c348188a6dcb15430bea0bad51c249fc
download-only = true
on-update = true
......@@ -186,7 +186,7 @@ ignore-existing = true
url = ${:_profile_base_location_}/template/template-kvm-run.in
mode = 644
filename = template-kvm-run.in
md5sum = bd238397af6236b6b24b693012feeece
md5sum = c6f1536a3502102dadbfb9d82496cc36
download-only = true
on-update = true
......
......@@ -381,8 +381,8 @@ interface-url = {{ slapparameter_dict.get('monitor-interface-url', 'https://moni
<= monitor-publish
recipe = slapos.cookbook:publish
ipv6 = ${slap-network-information:global-ipv6}
backend-url = https://[${novnc-instance:ip}]:${novnc-instance:port}/vnc_auto.html?host=[${novnc-instance:ip}]&port=${novnc-instance:port}&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}
url = ${request-slave-frontend:connection-url}/vnc_auto.html?host=${request-slave-frontend:connection-domainname}&port=${request-slave-frontend:connection-port}&encrypt=1&path=${request-slave-frontend:connection-resource}&password=${kvm-controller-parameter-dict:vnc-passwd}
backend-url = https://[${novnc-instance:ip}]:${novnc-instance:port}/vnc.html?host=[${novnc-instance:ip}]&port=${novnc-instance:port}&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}
url = ${request-slave-frontend:connection-url}/vnc.html?host=${request-slave-frontend:connection-domainname}&port=${request-slave-frontend:connection-port}&encrypt=1&path=${request-slave-frontend:connection-resource}&password=${kvm-controller-parameter-dict:vnc-passwd}
{% set disk_number = len(storage_dict) -%}
maximum-extra-disk-amount = {{ disk_number }}
{% set iface = 'eth0' -%}
......
......@@ -5,7 +5,7 @@ extends = common.cfg
# XXX - use websockify = 0.5.1 for compatibility with kvm frontend
websockify = 0.5.1
slapos.toolbox = 0.74
slapos.toolbox = 0.76
erp5.util = 0.4.49
apache-libcloud = 1.1.0
collective.recipe.environment = 0.2.0
......
......@@ -264,7 +264,7 @@ if enable_device_hotplug != 'true':
ram = '%sM,slots=128,maxmem=%sM' % (ram_size, ram_max_size)
else:
smp = '1,maxcpus=%s' % smp_max_count
ram = '%sM,slots=128,maxmem=%s' % (init_ram_size, ram_max_size)
ram = '%sM,slots=128,maxmem=%sM' % (init_ram_size, ram_max_size)
kvm_argument_list = [qemu_path,
'-enable-kvm', '-smp', smp, '-name', vm_name, '-m', ram, '-vga', 'std',
'-drive', 'file=%s,if=%s,cache=%s,aio=%s' % (disk_path, disk_type, disk_cache, disk_aio),
......
# DESTINATION
<match td.*.*>
type tdlog
apikey YOUR_API_KEY
auto_create_table
buffer_type file
buffer_path {{ fluentd_log_directory }}/td-agent/buffer/td
<secondary>
type file
path {{ fluentd_log_directory }}/failed_records
</secondary>
</match>
{% set wendelin_streamtool_uri = slapparameter_dict.get('wendelin-streamtool-uri', '') -%}
{% set wendelin_password = slapparameter_dict.get('wendelin-password', '') -%}
{% set wendelin_user = slapparameter_dict.get('wendelin-user', '') -%}
{% set wendelin_uid = slapparameter_dict.get('wendelin-uid', 'UNKNOWN') -%}
{% if wendelin_streamtool_uri and wendelin_password and wendelin_user -%}
<source>
type syslog
port 42185
bind 127.0.0.1
tag {{ wendelin_uid }}.system
</source>
<match {{ wendelin_uid }}.system.**>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}_syslog
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
<source>
type tail
path {{ fluentd_log_directory }}/consumption/*/dump_user.csv
pos_file {{ fluentd_log_directory }}/consumption/dump_user.pos
tag slapos.monitor.usage.user.{{ wendelin_uid }}
format /^(?<partition>[^,]*),(?<pid>[^,]*),(?<process>[^,]*),(?<cpu_percent>[^,]*),(?<cpu_time>[^,]*),(?<cpu_num_threads>[^,]*),(?<memory_percent>[^,]*),(?<memory_rss>[^,]*),(?<io_rw_counter>[^,]*),(?<io_cycles_counter>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
read_from_head true
</source>
<match slapos.monitor.usage.user.{{ wendelin_uid }}>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}_user
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
<source>
type tail
path {{ fluentd_log_directory }}/consumption/*/dump_system.csv
pos_file {{ fluentd_log_directory }}/consumption/dump_system.pos
tag slapos.monitor.usage.system.{{ wendelin_uid }}
format /^(?<loadavg>[^,]*),(?<cpu_percent>[^,]*),(?<memory_used>[^,]*),(?<memory_free>[^,]*),(?<net_in_bytes>[^,]*),(?<net_in_errors>[^,]*),(?<net_in_dropped>[^,]*),(?<net_out_bytes>[^,]*),(?<net_out_errors>[^,]*),(?<net_out_dropped>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
read_from_head true
</source>
<match slapos.monitor.usage.system.{{ wendelin_uid }}>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}_system
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
<source>
type tail
path {{ fluentd_log_directory }}/consumption/*/dump_computer.csv
pos_file {{ fluentd_log_directory }}/consumption/dump_computer.pos
tag slapos.monitor.usage.computer.{{ wendelin_uid }}
format /^(?<cpu_num_core>[^,]*),(?<cpu_frequency>[^,]*),(?<cpu_type>[^,]*),(?<memory_size>[^,]*),(?<memory_type>[^,]*),(?<partition_list>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
read_from_head true
</source>
<match slapos.monitor.usage.computer.{{ wendelin_uid }}>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}_computer
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
<source>
type tail
path {{ fluentd_log_directory }}/consumption/*/dump_disk.csv
pos_file {{ fluentd_log_directory }}/consumption/dump_disk.pos
tag slapos.monitor.usage.disk.{{ wendelin_uid }}
format /^(?<partition>[^,]*),(?<used>[^,]*),(?<free>[^,]*),(?<mountpoint>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
read_from_head true
</source>
<match slapos.monitor.usage.disk.{{ wendelin_uid }}>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}_disk
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
<source>
type tail
path {{ fluentd_log_directory }}/consumption/*/dump_temperature.csv
pos_file {{ fluentd_log_directory }}/consumption/dump_temperature.pos
tag slapos.monitor.usage.temperature.{{ wendelin_uid }}
format /^(?<sensor_id>[^,]*),(?<temperature>[^,]*),(?<alarm>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
read_from_head true
</source>
<match slapos.monitor.usage.temperature.{{ wendelin_uid }}>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}_temperature
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
{% endif -%}
# DESTINATION
<match td.*.*>
type tdlog
apikey YOUR_API_KEY
auto_create_table
buffer_type file
buffer_path {{ fluentd_log_directory }}/td-agent/buffer/td
<secondary>
type file
path {{ fluentd_log_directory }}/failed_records
</secondary>
</match>
## match tag=debug.** and dump to console
<match debug.**>
type stdout
</match>
<source>
type tail
path {{ crawl_log_directory }}/*/*ping.log.20*
pos_file {{ crawl_log_directory }}/tail_in_ping.pos
tag slapos.monitor.networktest.ping.ipv4
format /^(?<time>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/
read_from_head true
</source>
<source>
type tail
path {{ crawl_log_directory }}/*/*ping6.log.20*
pos_file {{ crawl_log_directory }}/tail_in_ping6.pos
tag slapos.monitor.networktest.ping.ipv6
format /^(?<time>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/
read_from_head true
</source>
<match slapos.monitor.networktest.ping.*>
type copy
{% for slave_instance in slave_instance_list -%}
{% if slave_instance.get("ping_ip_list") -%}
<store>
type grep
regexp1 name_or_ip ^{{ slave_instance.get("ping_ip_list") }}$
add_tag_prefix {{ slave_instance.get("slave_reference") }}
</store>
{% endif -%}
{% if slave_instance.get("ping6_ip_list") -%}
<store>
type grep
regexp1 name_or_ip ^{{ slave_instance.get("ping6_ip_list") }}$
add_tag_prefix {{ slave_instance.get("slave_reference") }}
</store>
{% endif -%}
{% endfor -%}
</match>
{% for slave_instance in slave_instance_list -%}
<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv6>
type file
path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping6/log
append true
include_time_key true
include_tag_key true
format json
</match>
<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv4>
type file
path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping/log
append true
include_time_key true
include_tag_key true
format json
</match>
{% endfor -%}
<source>
type tail
path {{ network_user_logs }}/*/ping*/log/*.log
pos_file {{ crawl_log_directory }}/tail_in_ping6.pos
tag slapos.wendelin.networktest
format json
read_from_head true
</source>
{% set wendelin_streamtool_uri = slapparameter_dict.get('wendelin-streamtool-uri', '') -%}
{% set wendelin_password = slapparameter_dict.get('wendelin-password', '') -%}
{% set wendelin_user = slapparameter_dict.get('wendelin-user', '') -%}
{% if wendelin_streamtool_uri and wendelin_password and wendelin_user -%}
<match slapos.wendelin.networktest>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
{% endif -%}
## SOURCE
<source>
type forward
</source>
## live debugging agent
<source>
type debug_agent
bind 127.0.0.1
port 24230
</source>
......@@ -72,26 +72,7 @@ wrapper-path = ${monitor-directory:bin}/log-crawler
extends = {{ instance_base_monitor }}
parts +=
slave-test-configuration
fluentd-distributor-wrapper
{% for part in part_list %}
{{ ' %s' % part }}
{% endfor %}
[fluentd-distributor-wrapper]
<=fluentd-wrapper
command-line = {{ fluentd_location }}/bin/fluentd -l ${monitor-directory:log}/fluend-distributor.log -c ${fluentd-distributor-conf-configuration:rendered}
wrapper-path = ${monitor-directory:service}/fluentd-distributor
[fluentd-distributor-conf-configuration]
recipe = slapos.recipe.template:jinja2
template = {{ fluent_distributor_conf_output }}
rendered = ${monitor-directory:etc}/fluentd-distributor.cfg
mode = 0744
context =
key slapparameter_dict slap-configuration:configuration
key slave_instance_list slap-configuration:slave-instance-list
key fluentd_log_directory monitor-directory:fluentd-log
key crawl_log_directory monitor-directory:crawl-log
key network_user_logs monitor-directory:network-user-logs
key computer_id slap-configuration:computer
......@@ -8,7 +8,6 @@ parts =
symlink-re6st-logs
symlink-collected-logs
monitor-collect-csv-wrapper
fluentd-wrapper
monitor-base
monitor-check-memory-usage
monitor-check-cpu-usage
......@@ -24,7 +23,10 @@ offline = true
recipe = slapos.cookbook:cron.d
name = network-bench-test
frequency = */10 * * * *
command = {{ buildout }}/networkbench ${network-bench-configuration:rendered} ${monitor-directory:monitor-log}
# skip to not fill cron log file.
# command = {{ buildout_bin }}/networkbench ${network-bench-configuration:rendered} ${monitor-directory:monitor-log}
command = true
[symlink-re6st-logs]
recipe = cns.recipe.symlink
......@@ -63,8 +65,7 @@ monitor-log = ${:private}/monitor-log
cache = ${:var}/cache
mod-ssl = ${:cache}/httpd_mod_ssl
system-log = ${:private}/system-log
fluentd-log = ${:log}/fluentd
consumption = ${:fluentd-log}/consumption
consumption = ${:log}/consumption
[monitor-httpd-configuration-file]
context =
......@@ -76,60 +77,37 @@ context =
section slave_information slap-configuration
key monitor_private_hash slap-configuration:private-hash
[fluentd-wrapper]
recipe = slapos.cookbook:wrapper
command-line = {{ fluentd_location }}/bin/fluentd -qq -l ${monitor-directory:log}/fluend.log -c ${fluentd-conf-configuration:rendered}
wrapper-path = ${monitor-directory:service}/fluentd
environment =
GEM_PATH={{ fluentd_location }}/lib/ruby/gems/1.8/
[fluentd-conf-configuration]
recipe = slapos.recipe.template:jinja2
template = {{ fluent_conf_output }}
rendered = ${monitor-directory:etc}/fluentd.cfg
mode = 0744
context =
key slapparameter_dict slap-configuration:configuration
key slave_instance_list slap-configuration:slave-instance-list
key fluentd_log_directory monitor-directory:fluentd-log
key computer_id slap-connection:computer-id
[monitor-collect-csv-wrapper]
recipe = slapos.cookbook:wrapper
command-line =
${monitor-directory:bin}/python {{ monitor_collect_csv_dump }} --output_folder ${monitor-directory:fluentd-log}/consumption/
${monitor-directory:bin}/python {{ monitor_collect_csv_dump }} --output_folder ${monitor-directory:consumption}
wrapper-path = ${monitor-directory:reports}/monitor-collect-csv-dump
[monitor-check-cpu-usage]
recipe = slapos.cookbook:wrapper
command-line = ${monitor-directory:bin}/python {{ monitor_check_system_health }} cpu ${init-monitor-parameters:cpu-load-file}
wrapper-path = ${monitor-directory:promises}/system-CPU-load-check
wrapper-path = ${directory:promises}/system-CPU-load-check
[monitor-check-memory-usage]
recipe = slapos.cookbook:wrapper
command-line = ${monitor-directory:bin}/python {{ monitor_check_system_health }} mem ${init-monitor-parameters:mem-free-file} ${directory:monitor}
wrapper-path = ${monitor-directory:promises}/system-MEMORY-usage-check
command-line = {{ buildout_bin}}/check-computer-memory
-db ${monitor-instance-parameter:collector-db}
--threshold ${slap-parameter:memory-percent-threshold}
--unit percent
wrapper-path = ${directory:promises}/check-computer-memory-usage
[publish-connection-information]
recipe = slapos.cookbook:publish
monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
server_log_url = ${monitor-publish-parameters:monitor-base-url}/${slap-configuration:private-hash}/
[monitor-instance-parameter]
instance-configuration =
file max-cpu-load-per-core ${init-monitor-parameters:cpu-load-file}
file min-free-mem-percent ${init-monitor-parameters:mem-free-file}
[init-monitor-parameters]
recipe = plone.recipe.command
cpu-load-file = ${directory:monitor}/cpu-load-tolerance
mem-free-file = ${directory:monitor}/mem-free-limit
command =
if [ ! -s "${:cpu-load-file}" ]; then
echo "1.5" > ${:cpu-load-file}
fi
if [ ! -s "${:mem-free-file}" ]; then
echo "7.0" > ${:mem-free-file}
echo ${slap-parameter:cpu-load-threshold} > ${:cpu-load-file}
fi
[slap-configuration]
......@@ -141,3 +119,7 @@ key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
private-hash = ${pwgen:passwd}${pwgen32:passwd}
frontend-domain =
[slap-parameter]
cpu-load-threshold = 2.0
memory-percent-threshold = 96
......@@ -15,14 +15,12 @@ recipe = slapos.recipe.template:jinja2
template = ${template-monitor:destination}
rendered = $${buildout:directory}/template-base-monitor.cfg
extensions = jinja2.ext.do
context = key buildout buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
context = key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
raw buildout_bin ${buildout:bin-directory}
raw monitor_template_output ${monitor-template:output}
raw network_benck_cfg_output ${network-bench-cfg:output}
raw fluentd_location ${fluentd:location}
raw fluent_conf_output ${fluentd-agent-conf:output}
raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
raw monitor_check_system_health ${monitor-system-health:output}
mode = 0644
......@@ -33,15 +31,13 @@ template = ${template-monitor-distributor:destination}
rendered = $${buildout:directory}/template-monitor-base-distributor.cfg
extensions = jinja2.ext.do
context = import json_module json
key buildout buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
key instance_base_monitor instance-base-monitor:rendered
key slave_instance_list slap-configuration:slave-instance-list
raw buildout_bin ${buildout:bin-directory}
raw template_json_distributor_test ${json-test-template:destination}
raw fluentd_location ${fluentd:location}
raw fluent_distributor_conf_output ${fluentd-agent-distributor-conf:output}
raw wget_bin ${wget:location}/bin/wget
mode = 0644
......
......@@ -5,7 +5,6 @@ extends =
../../component/python-cryptography/buildout.cfg
../../component/wget/buildout.cfg
../../stack/monitor/buildout.cfg
../../component/fluentd/buildout.cfg
../../stack/slapos.cfg
parts =
......@@ -16,30 +15,27 @@ parts =
template
template-monitor-distributor
template-monitor
fluentd
fluentd-agent-conf
fluentd-agent-distributor-conf
monitor-collect-csv-dump
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg
md5sum = 3ff5fb2710bf0ea84632c6d6d3894dd9
md5sum = 641c5916739f78171c616af00fe974a2
mode = 0644
[template-monitor]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-monitor.cfg.jinja2
destination = ${buildout:directory}/template-base-monitor.cfg
md5sum = 5def53c8faa0dfca313e53ebcc3229fa
md5sum = db36ed784c690be892bf1834444c1fe7
mode = 0644
[template-monitor-distributor]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-monitor-distributor.cfg.jinja2
destination = ${buildout:directory}/template-monitor-base-distributor.cfg
md5sum = 0ab9a86df18125335ed365bb673bbcf3
md5sum = 61c0bfdfc0a2b51ba15fe4a49baf6091
mode = 0644
[json-test-template]
......@@ -56,20 +52,6 @@ md5sum = cfcbf2002b8eff5153e2bf68ed24b720
output = ${buildout:directory}/template-network-bench-cfg.in
mode = 0644
[fluentd-agent-conf]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/fluentd-agent.conf.jinja2.in
md5sum = 3ea59906937eab7aeef78f46c4994ecd
output = ${buildout:directory}/fluentd-agent.conf.jinja2.in
mode = 0644
[fluentd-agent-distributor-conf]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/fluentd-distributor-agent.conf.jinja2.in
md5sum = 4b81ddcbe2f16d23013caac37151d396
output = ${buildout:directory}/fluentd-agent-distributor.conf.jinja2.in
mode = 0644
[monitor-collect-csv-dump]
<= monitor-template-script
url = ${:_profile_base_location_}/script/${:filename}
......@@ -103,12 +85,8 @@ scripts =
monitor.genstatus
monitor.configwrite
is-process-older-than-dependency-set
[fluentd]
gems +=
fluent-plugin-wendelin==0.1.alpha1
fluent-plugin-grep==0.3.4
fluent-plugin-bin==0.1
check-free-disk
check-computer-memory
[monitor-eggs]
eggs +=
......
......@@ -48,6 +48,11 @@ eggs = neoppod[admin, ctl, master, storage-mysqldb]
ZODB
zope.testing
zodbtools
patch-binary = ${patch:location}/bin/patch
ZEO-patch-options = -p1
ZEO-patches =
${:_profile_base_location_}/../../component/egg-patch/ZEO4/TCP_NODELAY.patch#b07288522d5c6857738240d948321df6
[slapos-deps-eggs]
recipe = zc.recipe.egg
......@@ -117,7 +122,7 @@ persistent = 4.2.3
pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.template = 4.3
slapos.toolbox = 0.74
slapos.toolbox = 0.76
smmap2 = 2.0.1
transaction = 1.7.0
zodbpickle = 0.6.0
......@@ -161,7 +166,7 @@ zodburi = 2.3.0
# Required by:
# zodburi==2.0
# ZEO 5 requires transaction >= 2
ZEO = 4.3.1
ZEO = 4.3.1+SlapOSPatched001
# Required by:
# zodburi==2.0
......
[buildout]
extends = software.cfg
[neoppod]
ZEO-patches =
[versions]
ZODB = 5.3.0
ZEO = 5.1.0
......
......@@ -3,21 +3,35 @@
# list of go git repositories to fetch
[gowork.goinstall]
depends_gitfetch =
${go_github.com_DataDog_czlib:recipe}
${go_github.com_cznic_strutil:recipe}
${go_github.com_golang_glog:recipe}
${go_github.com_gwenn_gosqlite:recipe}
${go_github.com_gwenn_yacr:recipe}
${go_github.com_kisielk_og-rek:recipe}
${go_github.com_kylelemons_godebug:recipe}
${go_github.com_pkg_errors:recipe}
${go_github.com_pkg_profile:recipe}
${go_github.com_soheilhy_cmux:recipe}
${go_github.com_someonegg_gocontainer:recipe}
${go_github.com_someonegg_gox:recipe}
${go_golang.org_x_crypto:recipe}
${go_golang.org_x_net:recipe}
${go_golang.org_x_perf:recipe}
${go_golang.org_x_sync:recipe}
${go_golang.org_x_sys:recipe}
${go_golang.org_x_text:recipe}
${go_golang.org_x_tools:recipe}
${go_lab.nexedi.com_kirr_go123:recipe}
${go_lab.nexedi.com_kirr_neo:recipe}
[go_github.com_DataDog_czlib]
<= go-git-package
go.importpath = github.com/DataDog/czlib
repository = https://github.com/DataDog/czlib
revision = 4bc9a24e37
[go_github.com_cznic_strutil]
<= go-git-package
go.importpath = github.com/cznic/strutil
......@@ -30,6 +44,18 @@ go.importpath = github.com/golang/glog
repository = https://github.com/golang/glog
revision = 23def4e6c1
[go_github.com_gwenn_gosqlite]
<= go-git-package
go.importpath = github.com/gwenn/gosqlite
repository = https://github.com/gwenn/gosqlite
revision = 29cd841087
[go_github.com_gwenn_yacr]
<= go-git-package
go.importpath = github.com/gwenn/yacr
repository = https://github.com/gwenn/yacr
revision = 77093bdc7e
[go_github.com_kisielk_og-rek]
<= go-git-package
go.importpath = github.com/kisielk/og-rek
......@@ -46,7 +72,7 @@ revision = d65d576e93
<= go-git-package
go.importpath = github.com/pkg/errors
repository = https://github.com/pkg/errors
revision = v0.8.0-6-g2b3a18b5f0
revision = v0.8.0-12-g816c908556
[go_github.com_pkg_profile]
<= go-git-package
......@@ -54,44 +80,74 @@ go.importpath = github.com/pkg/profile
repository = https://github.com/pkg/profile
revision = v1.2.1-0-g5b67d42886
[go_github.com_soheilhy_cmux]
<= go-git-package
go.importpath = github.com/soheilhy/cmux
repository = https://github.com/soheilhy/cmux
revision = e09e9389d8
[go_github.com_someonegg_gocontainer]
<= go-git-package
go.importpath = github.com/someonegg/gocontainer
repository = https://github.com/someonegg/gocontainer
revision = fc2c7e84b5
[go_github.com_someonegg_gox]
<= go-git-package
go.importpath = github.com/someonegg/gox
repository = https://github.com/someonegg/gox
revision = 4915b7fd7c
[go_golang.org_x_crypto]
<= go-git-package
go.importpath = golang.org/x/crypto
repository = https://go.googlesource.com/crypto
revision = 88942b9c40
[go_golang.org_x_net]
<= go-git-package
go.importpath = golang.org/x/net
repository = https://go.googlesource.com/net
revision = 1087133bc4
revision = 6078986fec
[go_golang.org_x_perf]
<= go-git-package
go.importpath = golang.org/x/perf
repository = https://go.googlesource.com/perf
revision = 4469e6ce8c
revision = 8c788eb673
[go_golang.org_x_sync]
<= go-git-package
go.importpath = golang.org/x/sync
repository = https://go.googlesource.com/sync
revision = 8e0aa688b6
revision = 1d60e4601c
[go_golang.org_x_sys]
<= go-git-package
go.importpath = golang.org/x/sys
repository = https://go.googlesource.com/sys
revision = 91ee8cde43
[go_golang.org_x_text]
<= go-git-package
go.importpath = golang.org/x/text
repository = https://go.googlesource.com/text
revision = v0.3.0-42-gab48842968
[go_golang.org_x_tools]
<= go-git-package
go.importpath = golang.org/x/tools
repository = https://go.googlesource.com/tools
revision = 9bd2f44268
revision = 77106db15f
[go_lab.nexedi.com_kirr_go123]
<= go-git-package
go.importpath = lab.nexedi.com/kirr/go123
repository = https://lab.nexedi.com/kirr/go123.git
revision = 2578d58311
revision = 76f667ba6c
[go_lab.nexedi.com_kirr_neo]
<= go-git-package
go.importpath = lab.nexedi.com/kirr/neo
repository = https://lab.nexedi.com/kirr/neo.git
revision = v1.8-1326-g4d0cd89484
revision = v1.8.1-1634-g4000df14e4
......@@ -8,6 +8,8 @@ extends =
../neoppod/software-common.cfg
../../component/golang/buildout.cfg
gowork.cfg
../../component/sqlite3/buildout.cfg
../../component/zlib/buildout.cfg
../../component/wendelin.core/buildout.cfg
../../component/ethtool/buildout.cfg
../../component/ioping/buildout.cfg
......@@ -48,6 +50,10 @@ install =
github.com/pkg/profile \
golang.org/x/perf/cmd/benchstat
cpkgpath =
${sqlite3:location}/lib/pkgconfig
${zlib:location}/lib/pkgconfig
# dev-install neo from go checkout
[neoppod-develop]
setup = ${go_lab.nexedi.com_kirr_neo:location}
......@@ -108,13 +114,13 @@ eggs =
# wendelin.core: latest not yet released
[wendelin.core-repository]
revision= v0.11-4-g38fbc83ceb
revision= v0.11-15-gf785ac079b
# ping eggs versions
[versions]
pyasn1 = 0.3.7
ZODB3 = 3.11.0
numpy = 1.13.3
numpy = 1.14.2
zope.testing = 4.6.2
erp5.util = 0.4.50
......@@ -128,3 +134,7 @@ ZConfig = 3.2.0
# ZEO==4.3.1
# ZODB==4.4.5
zc.lockfile = 1.2.1
# Required by:
# neoppod==1.8.1
python-dateutil = 2.7.1
......@@ -46,7 +46,7 @@ mode = 0644
[versions]
slapos.recipe.template = 4.3
slapos.toolbox = 0.74
slapos.toolbox = 0.76
dnspython = 1.15.0
PyRSS2Gen = 1.1
erp5.util = 0.4.50
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"properties": {
}
}
......@@ -104,6 +104,7 @@ filename = registry-run.in
md5sum = 0bf4f2c03e06b55c6c6cc55fa33e65d6
[versions]
re6stnet = 0.494
apache-libcloud = 0.17.0
ecdsa = 0.13
gitdb = 0.6.4
......@@ -111,7 +112,7 @@ plone.recipe.command = 1.1
pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.template = 4.3
slapos.toolbox = 0.74
slapos.toolbox = 0.76
smmap = 0.9.0
# Required by:
......
......@@ -10,11 +10,13 @@
"response": "instance-re6stnet-output-schema.json",
"index": 0
},
"registry": {
"title": "registry",
"default-slave": {
"title": "Re6st Token",
"description": "Re6st registry",
"request": "instance-re6stnet-resilient-input-schema.json",
"software-type": "default",
"request": "instance-re6stnet-slave-input-schema.json",
"response": "instance-re6stnet-output-schema.json",
"shared": true,
"index": 1
}
}
......
# Slapos egg tests
This software release is used to run unit test of slapos eggs.
The approach is to use setuptools' integrated test runner, `python setup.py test`, to run tests.
The `python` used in this command will be a `zc.recipe.egg` interpreter with
all eggs pre-installed by this software release.
Nexedi staff can see the results of this test from the test suite
`SLAPOS-EGG-TEST` in test result module.
Here's an example session of how a developer could use this software release in
slaprunner to develop a slapos egg, in the example `slapos.core`, to make
changes to the code, run tests and publish changes.
```bash
# install this software release
SR=https://lab.nexedi.com/nexedi/slapos/raw/master/software/slapos-testing/software.cfg
COMP=slaprunner
INSTANCE_NAME=$COMP
slapos supply $SR $COMP
slapos node software
slapos request --node=node=$COMP $INSTANCE_NAME $SR
slapos node instance
# The source code is a git clone working copy on the instance
cd ~/srv/runner/instance/slappart0/parts/slapos.core/
# make some changes to the code
vim slapos/tests/client.py
# run tests, using bundled python intepreter with pre-installed eggs dependencies
~/srv/runner/instance/slappart0/software_release/bin/python_for_test setup.py build
# when satified, commit changes
git add -p && git commit
# add developer's fork remote (this is only needed the first time)
git remote add my_remote https://lab.nexedi.com/your_username/slapos.core.git/
# push the changes
git push my_remote HEAD:feature_branch_name
# then submit merge request
```
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# But avoid directories, they are not portable.
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[template]
filename = instance.cfg
md5sum = 9dece9d12dc94bf5c35d307cc8aa4d6b
[buildout]
parts =
slapos.core-setup
erp5.util-setup
phantomjs-wrapper
slapos-test-runner
sh-environment
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
......@@ -24,33 +21,26 @@ bin = $${buildout:directory}/bin
etc = $${buildout:directory}/etc
services = $${:etc}/run
srv = $${buildout:directory}/srv
source-code = $${:srv}/eggs-source-code
[download-source]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
# Local development
[slapos.core]
<= download-source
repository = ${slapos.core-repository:location}
[slapos.core-setup]
recipe = plone.recipe.command
command = echo "Updating setup...";cd $${slapos.core:location}; export PATH="$${slapos-test-runner:prepend-path}:$PATH"; export CPPFLAGS="$${environment:CPPFLAGS}"; export LDFLAGS="$${environment:LDFLAGS}"; export PYTHONPATH="$${environment:PYTHONPATH}"; export LOCAL_IPV4="$${environment:LOCAL_IPV4}"; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n
update-command = $${:command}
[caucase]
<= download-source
repository = ${caucase-repository:location}
[erp5.util]
<= download-source
repository = ${erp5.util-repository:location}
[slapos.cookbook]
<= download-source
repository = ${slapos.cookbook-repository:location}
[slapos.recipe.template]
[slapos.core]
<= download-source
repository = ${slapos.recipe.template-repository:location}
repository = ${slapos.core-repository:location}
[slapos.recipe.build]
<= download-source
......@@ -60,57 +50,33 @@ repository = ${slapos.recipe.build-repository:location}
<= download-source
repository = ${slapos.recipe.cmmi-repository:location}
[slapos.toolbox]
[slapos.recipe.template]
<= download-source
repository = ${slapos.toolbox-repository:location}
repository = ${slapos.recipe.template-repository:location}
[erp5-util]
[slapos.toolbox]
<= download-source
repository = ${erp5-util-repository:location}
repository = ${slapos.toolbox-repository:location}
[erp5.util-setup]
recipe = plone.recipe.command
command = echo "Updating setup...";cd $${erp5-util:location}; export PATH="$${slapos-test-runner:prepend-path}:$PATH"; export CPPFLAGS="$${environment:CPPFLAGS}"; export LDFLAGS="$${environment:LDFLAGS}"; export PYTHONPATH="$${environment:PYTHONPATH}"; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n
update-command = $${:command}
[slapos-test-runner]
recipe = slapos.cookbook:egg_test
run-test-suite = $${create-directory:bin}/runTestSuite
run-test-suite-binary = ${buildout:bin-directory}/runTestSuite
# The list of executables should be defined here and a combination
# of tests should dynamically generated.
#python-list = $${}
test-list =
$${slapos.cookbook:location}
$${slapos.core:location}
$${slapos.recipe.template:location}
$${slapos.recipe.build:location}
$${slapos.recipe.cmmi:location}
$${slapos.toolbox:location}
$${erp5-util:location}
$${caucase:location}
prepend-path = ${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:${python2.7:location}/bin
environment = environment
[environment]
CPPFLAGS = -I${python2.7:location}/include/python2.7 -I${libxml2:location}/include -I${libxslt:location}/include
LDFLAGS = -L${python2.7:location}/lib -L${libxml2:location}/lib -L${libxslt:location}/lib -L${libxslt:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${python2.7:location}/lib -Wl,-rpath=${libxml2:location}/lib -Wl,-rpath=${libxslt:location}/lib -Wl,-rpath=${zlib:location}/lib
LD_LIBRARY_PATH = ${python2.7:location}/lib:${libxml2:location}/lib:${libxslt:location}/lib:${libxslt:location}/lib:${zlib:location}/lib
PYTHONPATH = ${python-setuptools:pythonpath}:${buildout:eggs-directory}:${buildout:develop-eggs-directory}
LOCAL_IPV4 = $${slap-configuration:ipv4-random}
[sh-environment]
# Section exposes testing default environment as sh file. It is thus easy
# to directly develop and test the egg inside of this instance.
recipe = collective.recipe.template
input = inline:
export PATH="$${slapos-test-runner:prepend-path}:$PATH"
export CPPFLAGS="$${environment:CPPFLAGS}"
export LDFLAGS="$${environment:LDFLAGS}"
export PYTHONPATH="$${environment:PYTHONPATH}"
export PS1="[slapos-testing env Active] $PS1"
output = $${create-directory:bin}/environment.sh
mode = 755
recipe = slapos.cookbook:wrapper
wrapper-path = $${create-directory:bin}/runTestSuite
command-line =
${buildout:bin-directory}/runTestSuite
--python_interpreter=${buildout:bin-directory}/${eggs:interpreter}
--source_code_path_list=$${caucase:location},$${erp5.util:location},$${slapos.cookbook:location},$${slapos.core:location},$${slapos.recipe.build:location},$${slapos.recipe.cmmi:location},$${slapos.recipe.template:location},$${slapos.toolbox:location}
# Notes about environment:
# * slapos.cookbook:wrapper does not seem to allow "extending" PATH. Tests
# needs ping, which is a setuid binary that cannot be installed via slapos
# way of building software without root access, so we keep "standard"
# /usr/bin and /bin in $PATH
# * LOCAL_IPV4 is needed for some slapos.core tests
environment =
PATH=${coreutils:location}/bin:${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:/usr/bin/:/bin/
LOCAL_IPV4=$${slap-configuration:ipv4-random}
[phantomjs-wrapper]
recipe = slapos.cookbook:wrapper
......
......@@ -6,26 +6,78 @@ extends =
../../component/libxml2/buildout.cfg
../../component/libxslt/buildout.cfg
../../component/bcrypt/buildout.cfg
../../component/python-2.7/buildout.cfg
../../component/python-setuptools/buildout.cfg
../../component/zlib/buildout.cfg
../../component/phantomjs/buildout.cfg
../../component/pycurl/buildout.cfg
../../component/coreutils/buildout.cfg
../../stack/slapos.cfg
./buildout.hash.cfg
parts =
caucase-repository
slapos.cookbook-repository
slapos.core-repository
slapos.recipe.template-repository
slapos.recipe.build-repository
slapos.recipe.cmmi-repository
slapos.toolbox-repository
erp5-util-repository
bootstrap-slapos.recipe.cmmi
eggs
phantomjs
template
[bootstrap-slapos.recipe.cmmi]
# install our develop version of slapos.recipe.cmmi before anything else,
# otherwise it will be installed from pypi by dependencies.
recipe = zc.recipe.egg
eggs = ${slapos.recipe.cmmi-setup:egg}
[setup-develop-egg]
recipe = zc.recipe.egg:develop
[caucase-setup]
<= setup-develop-egg
egg = caucase
setup = ${caucase-repository:location}
[erp5.util-setup]
<= setup-develop-egg
# XXX erp5.util does not have `test` extra require, but has a `testnode` extra require with same dependencies
egg = erp5.util[testnode]
setup = ${erp5.util-repository:location}
depends = ${slapos.core-setup:egg}
[slapos.cookbook-setup]
<= setup-develop-egg
# XXX slapos.cookbook does not have `test` extra require, `mock` is only listed in `tests_require` and is listed explicitly
egg = slapos.cookbook
setup = ${slapos.cookbook-repository:location}
depends = ${slapos.core-setup:egg}
[slapos.core-setup]
<= setup-develop-egg
# XXX slapos.cookbook does not have `test` extra require, `mock`, `pyflakes` and `httmock` are only listed in `tests_require` and are listed explicitly
egg = slapos.core
setup = ${slapos.core-repository:location}
[slapos.recipe.build-setup]
<= setup-develop-egg
egg = slapos.recipe.build[test]
setup = ${slapos.recipe.build-repository:location}
[slapos.recipe.cmmi-setup]
<= setup-develop-egg
egg = slapos.recipe.cmmi[test]
setup = ${slapos.recipe.cmmi-repository:location}
depends = ${slapos.recipe.build-setup:egg}
[slapos.recipe.template-setup]
<= setup-develop-egg
# XXX slapos.recipe.template does not have `test` extra require, `zope.testing` is only listed in `tests_require` and is listed explicitly
egg = slapos.recipe.template
setup = ${slapos.recipe.template-repository:location}
[slapos.toolbox-setup]
<= setup-develop-egg
# XXX slapos.toolbox does not have `test` extra require, `mock` and `pycurl` are only listed in `tests_require` and are listed explicitly
egg = slapos.toolbox
setup = ${slapos.toolbox-repository:location}
depends = ${slapos.core-setup:egg}
[eggs]
recipe = zc.recipe.egg
eggs =
......@@ -35,18 +87,24 @@ eggs =
${bcrypt:egg}
dnspython
Jinja2
caucase
erp5.util
slapos.cookbook
collective.recipe.template
plone.recipe.command
slapos.recipe.template
slapos.recipe.cmmi
slapos.toolbox
${caucase-setup:egg}
${erp5.util-setup:egg}
${slapos.cookbook-setup:egg}
${slapos.core-setup:egg}
${slapos.recipe.build-setup:egg}
${slapos.recipe.cmmi-setup:egg}
${slapos.recipe.template-setup:egg}
${slapos.toolbox-setup:egg}
mock
zope.testing
httmock
pyflakes
entry-points =
runTestSuite=erp5.util.testsuite:runTestSuite
scripts =
runTestSuite
interpreter=
python_for_test
[git-clone-repository]
recipe = slapos.recipe.build:gitclone
......@@ -58,6 +116,10 @@ branch = master
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/caucase.git
[erp5.util-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/erp5.git
[slapos.cookbook-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.git
......@@ -73,28 +135,38 @@ repository = https://lab.nexedi.com/nexedi/slapos.recipe.template.git
[slapos.recipe.build-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.recipe.build.git
# We use the system git and not slapos provided one, because
# slapos.recipe.build is a dependency of slapos.recipe.cmmi
#git-executable = git
[slapos.recipe.cmmi-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.recipe.cmmi.git
# We use the system git and not slapos provided one, because slapos git needs
# slapos.recipe.cmmi to be installed. This circular dependency cause parts to
# be reinstalled everytime buildout is run because signatures are not stable.
#git-executable = git
[slapos.toolbox-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.toolbox.git
[erp5-util-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/erp5.git
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
md5sum = 6626794c9dbb2530bb8ba3d331e27542
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template.cfg
mode = 640
[versions]
Pygments = 2.1.3
collective.recipe.template = 1.10
plone.recipe.command = 1.1
slapos.recipe.template = 4.3
# clear the version of tested eggs, to make sure we installed the developped ones
caucase =
erp5.util =
slapos.cookbook =
slapos.core =
slapos.recipe.build =
slapos.recipe.cmmi =
slapos.recipe.template =
slapos.toolbox =
......@@ -16,7 +16,7 @@ gunicorn = 19.7.1
prettytable = 0.7.2
pycurl = 7.43.0
slapos.recipe.template = 4.3
slapos.toolbox = 0.74
slapos.toolbox = 0.76
smmap = 0.9.0
# Required by:
......
......@@ -95,7 +95,7 @@ futures = 3.1.1
gitdb2 = 2.0.2
gunicorn = 19.7.1
slapos.recipe.template = 4.3
slapos.toolbox = 0.74
slapos.toolbox = 0.76
smmap2 = 2.0.3
# Required by:
......
......@@ -583,7 +583,7 @@ extra-paths =
# patches for eggs
patch-binary = ${patch:location}/bin/patch
Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic.patch#e8029103350dad364d25747514a20327
Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic.patch#1d9a56e9af4371f5b6951ebf217a15d7
Acquisition-patch-options = -p1
Products.DCWorkflow-patches = ${:_profile_base_location_}/../../component/egg-patch/Products.DCWorkflow/workflow_method.patch#975b49e96bae33ac8563454fe5fa9899
Products.DCWorkflow-patch-options = -p1
......@@ -625,7 +625,7 @@ scripts +=
# neoppod, mysqlclient, slapos.recipe.template & [slapos-deps-eggs]
# patched eggs
Acquisition = 2.13.9+SlapOSPatched001
Acquisition = 2.13.12+SlapOSPatched001
Products.DCWorkflow = 2.2.4+SlapOSPatched001
ocropy = 1.0+SlapOSPatched001
pysvn = 1.7.10+SlapOSPatched002
......
[versions]
AccessControl = 2.13.15
Acquisition = 2.13.11
AccessControl = 2.13.16
Acquisition = 2.13.12
DateTime = 2.12.8
DocumentTemplate = 2.13.4
ExtensionClass = 2.13.2
......@@ -21,31 +21,35 @@ Products.Sessions = 3.0
Products.StandardCacheManagers = 2.13.1
Products.TemporaryFolder = 3.0
Products.ZCTextIndex = 2.13.5
Products.ZCatalog = 2.13.27
Products.ZCatalog = 2.13.29
Pygments = 2.2.0
Record = 2.13.0
RestrictedPython = 3.6.0
Sphinx = 1.0.8
ZConfig = 2.9.3
ZODB3 = 3.10.7
Zope2 = 2.13.26
ZServer = 3.0
Zope2 = 2.13.27
ZopeUndo = 2.12.0
docutils = 0.12
initgroups = 2.13.0
mechanize = 0.2.5
mr.developer = 1.35
mr.developer = 1.38
pluggy = 0.6.0
py = 1.5.2
pytz = 2017.2
repoze.retry = 1.2
repoze.tm2 = 1.0
repoze.who = 2.0
tempstorage = 2.12.2
tox = 2.9.1
transaction = 1.1.1
z3c.checkversions = 0.5
zExceptions = 2.13.0
zLOG = 2.11.2
zc.buildout = 2.3.1
zc.lockfile = 1.0.2
zc.recipe.egg = 2.0.3
zc.recipe.egg = 2.0.5
zc.recipe.testrunner = 1.2.1
zdaemon = 2.0.7
zope.annotation = 3.5.0
......@@ -66,7 +70,7 @@ zope.exceptions = 3.6.2
zope.filerepresentation = 3.6.1
zope.i18n = 3.7.4
zope.i18nmessageid = 3.5.3
zope.interface = 3.6.7
zope.interface = 3.6.8
zope.lifecycleevent = 3.6.2
zope.location = 3.9.1
zope.pagetemplate = 3.5.2
......@@ -87,14 +91,3 @@ zope.testbrowser = 3.11.1
zope.testing = 3.9.7
zope.traversing = 3.13.2
zope.viewlet = 3.7.2
# Required by:
# Jinja2==2.8.1
MarkupSafe = 0.23
# Required by:
# DateTime==2.12.8
# Zope2==2.13.26
# zope.i18n==3.7.4
# zope.testbrowser==3.11.1
pytz = 2016.10
......@@ -84,66 +84,42 @@ NB: You should use double $ (ex: $${monitor-template:rendered}) instead of one $
Add a promise
-------------
Monitor stack will include slapos promise directory etc/promise to promise folder. All files in that directory will be considered as a promise.
This mean that all slapos promises will be checked frequently by monitor.
To learn how to write a promise in SlapOS, please read this document:
https://www.erp5.com/slapos-TechnicalNote.General.SlapOS.Monitoring.Specifications
[monitor-conf-parameters]
...
promise-folder = ${directory:promises}
...
Monitor will run each promise every minutes and save the result in a json file. Here is an exemple of promise result:
{"status": "ERROR", "change-time": 1466415901.53, "hosting_subscription": "XXXX", "title": "vnc_promise", "start-date": "2016-06-21 10:47:01", "instance": "XXXX-title", "_links": {"monitor": {"href": "MONITOR_PRIVATE_URL"}}, "message": "PROMISE_OUPT_MESSAGE", "type": "status"}
A promise will be ran during a short time and report the status: ERROR or OK, plus an ouput message which says what was good or bad.
The promise should not run for more that 20 seconds, else it will be interrupted because of time out. However this value can be modified from monitoring web interface, see parameter "promise-timeout" of your hosting subscription.
On slapos, the default timeout value is also 20 seconds, if the value is modified on monitor (ex: to 50 seconds), it will still fails when slapgrid will process instance if the promise execution exceed 20 seconds.
Promises result are published in web public folder, access URL is: MONITOR_BASE_URL/private/PROMISE_NAME.status.json
Everytime monitor will run a promise an history of result will be also updated. The promise history will be updated during one day, after that a new history will be created.
To access promise history file as JSON, use URL MONITOR_BASE_URL/private/PROMISE_NAME.history.json
Add a promise: monitor promise
------------------------------
Monitor promise is also a promise like normal promise script but it will be placed into the folder ${monitor-directory:promises}:
[monitor-promise-xxxxx]
recipe = slapos.recipe.template:jinja2
rendered = ${monitor-directory:promises}/my-custom-monitor-promise
Theses promises will be executed only by monitor (not slapos) every minutes and will report same infor as default promises. This is another way to
add more custom promises to check if server is overloaded, or if network start to be slow, etc...
Writing a promise consists of defining a class called RunPromise which inherits from GenericPromise class and defining methods: anomaly(), sense() and test(). Python promises should be placed into the folder etc/plugin of the computer partition.
New promises should be placed into the folder etc/plugin, legacy promise are into the folder etc/promise. Legacy promises are bash or other executable promises script which does not use GenericPromise class.
You will use slapos.cookbook:promise.plugin to generate your promise script into `etc/plugin` directory. Add promise will look like this:
Add custom scripts to monitor
-----------------------------
[promise-check-site]
recipe = slapos.cookbook:promise.plugin
eggs =
slapos.toolbox
output = ${directory:plugins}/promise-check-mysite-status.py
content =
from slapos.promise.plugin.check_site_state import RunPromise
config-site-url = ${publish:site-url}
config-connection-timeout = 20
config-foo = bar
mode = 600
Custom script will be automatically run by the monitor and result will be reported in monitor private folder. To add your custom script in ${monitor-directory:reports} folder. Here is an example:
Then you will have to add `promise-check-site` section to buildout parts, so it will be installed.
[monitor-check-webrunner-internal-instance]
recipe = slapos.recipe.template:jinja2
template = ${monitor-check-webrunner-internal-instance:location}/${monitor-check-webrunner-internal-instance:filename}
rendered = $${monitor-directory:reports}/$${:filename}
filename = monitor-check-webrunner-internal-instance
mode = 0744
In your promise code, you will be able to call `self.getConfig('site-url')`, `self.getConfig('connection-timeout')` and `self.getConfig('foo')`. The
returned value is `None` if the config parameter is not set.
The script will be executed every minutes by default. To change, put periodicity in script name:
- monitor-check-webrunner-internal-instance_every_1_minute
- monitor-check-webrunner-internal-instance_every_25_minute
- monitor-check-webrunner-internal-instance_every_1_hour
- monitor-check-webrunner-internal-instance_every_3_hour
- ...
Slapgrid will run each promise every time a partition is processed (every minutes in theory), if the partition is up to date, slapgrid will only run promises anomaly check and save the result in a json file. Here is an exemple of promise result:
the script name should end with _every_XX_hour or _every_XX_minute. With this, we can set filename as:
{"result": {"date": "2018-03-22T15:35:07", "failed": false, "message": "buildout is OK", "type": "Test Result"}, "path": "PARTITION_DIRECTORY/etc/plugin/buildout-slappart0-status.py", "name": "buildout-slappart0-status.py", "execution-time": 0.1, "title": "buildout-slappart0-status"}
filename = monitor-check-webrunner-internal-instance_every_2_minute
The promise execution time should be short, by default promise-timeout in slapgrid is to 20 seconds. If a promise runs in more than the defined promise-timeout, the process is killed and a "promise timed out" message is returned.
JSON in the folder `PARTITION_DIRECTORY/.slapgrid/promise/result`, and promise logs are in `PARTITION_DIRECTORY/.slapgrid/promise/log`.
You can get custom script results files at MONITOR_BASE_URL/private/FILE_NAME.
Monitor will expose promise JSON result in web public folder, access URL is: `MONITOR_BASE_URL/public/promise/PROMISE_TITLE.status.json`. Log files are exposed in monitor private web folder,
access URL is: `MONITOR_BASE_URL/private/log/monitor/promise`
Add custom file or directory to monitor
---------------------------------------
......@@ -158,20 +134,22 @@ Log or others files can be added in monitor public or private directory:
${directory:log}
...
files in public directory are accessible at MONITOR_BASE_URL/public, and for private directory: MONITOR_BASE_URL/private.
files in public directory are accessible at `MONITOR_BASE_URL/public`, and for private directory: `MONITOR_BASE_URL/private`.
Monitor RSS and OPML Feed
-------------------------
Monitor promise history, RSS and OPML Feed
------------------------------------------
Monitor generate rss containing latest result for all promises, this feed will be upaded every minutes. The Rss fee URL is
MONITOR_BASE_URL/public/feed
Monitor read every minutes JSON promises result files to build Rss and full instance state. The Rss feed URL is
`MONITOR_BASE_URL/public/feed`
OPML Feed is used to aggregate many feed URL, this is used on monitor to link many single monitor instances. For example, a resilient
webrunner has 5 instances at least, each instance has a monitor which leads to 5 monitor instances too. One main instance (usally the root instance)
will collect rss feeds of all others monitor's in a single OPML file. This file is published and used to configure a monitor backend to the web interface.
The URL of OPML feed is: MONITOR_BASE_URL/public/feeds
webrunner has 3 instances at least, each instance has a monitor which leads to 3 monitor instances too. One main instance (usally the root instance)
will collect rss feed of all others monitor's in a single OPML file. This file is published and used to configure a monitor backend to the web interface.
The URL of OPML feed is: `MONITOR_BASE_URL/public/feeds`
Everytime monitor will also produce history of for each promise in a single JSON file.
To access promise history file as JSON, use URL `MONITOR_BASE_URL/public/PROMISE_TITLE.history.json`
Monitor Base web directory tree
-------------------------------
......@@ -184,16 +162,13 @@ Monitor Base web directory tree
(webdav) X Y
|
---------------------------------
| | | |
public private jio_public jio_private
X Y | |
.jio_documents .jio_documents
| |
public private
X Y
MONITOR_BASE_URL/public or private is for normal HTTPS.
MONITOR_BASE_URL/share is the webdav URL. public/ and private/ are linked to public and private directories.
webdav also has jio_public/.jio_documents and jio_private/.jio_documents which are linked to public and private directory and they works with jio webdav pluging.
Access to Monitor
......
......@@ -44,6 +44,7 @@ eggs =
collective.recipe.template
cns.recipe.symlink
slapos.toolbox
slapos.core
# Do no generate any scripts here as all of them are generated by extraeggs
scripts =
......@@ -72,22 +73,12 @@ md5sum = 1695c9a06a2b11ccfe893d7a224e489d
[monitor-conf]
<= monitor-template-base
filename = monitor.conf.in
md5sum = 888e2845d09bfaa59c25f56f5bcf76b1
[monitor-instance-info]
<= monitor-template-base
filename = instance-info.conf.in
md5sum = 1bdb4e05c6be04f4e5766c64467fbcec
md5sum = 91c4c9bba1f7df788b9b7a059ed89ac2
[monitor-httpd-cors]
<= monitor-template-base
filename = httpd-cors.cfg.in
md5sum = 683ea85fc054094248baf5752dd089bf
[monitor-check-free-disk-space]
<= monitor-template-base
filename = check_free_disk.in
md5sum = bab457ac4d139ed31d0b343a7d14d996
# End templates files
# XXX keep compatibility (with software/ipython_notebook/software.cfg )
......@@ -112,7 +103,6 @@ context =
raw monitor_configwrite ${buildout:directory}/bin/monitor.configwrite
raw monitor_conf_template ${monitor-conf:location}/${monitor-conf:filename}
raw monitor_https_cors ${monitor-httpd-cors:location}/${monitor-httpd-cors:filename}
raw monitor_instance_info ${monitor-instance-info:location}/${monitor-instance-info:filename}
raw curl_executable_location ${curl:location}/bin/curl
raw dash_executable_location ${dash:location}/bin/dash
raw dcron_executable_location ${dcron:location}/sbin/crond
......@@ -122,7 +112,7 @@ context =
raw python_executable ${buildout:executable}
raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter}
raw template_wrapper ${monitor-template-wrapper:location}/${monitor-template-wrapper:filename}
raw template_check_disk_space ${monitor-check-free-disk-space:location}/${monitor-check-free-disk-space:filename}
raw check_disk_space ${buildout:directory}/bin/check-free-disk
raw bin_directory ${buildout:directory}/bin
depends =
${monitor-eggs:eggs}
......@@ -131,6 +121,6 @@ depends =
PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
pycurl = 7.43.0
slapos.toolbox = 0.74
slapos.toolbox = 0.76
pyasn1 = 0.3.7
......@@ -15,4 +15,4 @@
# not need these here).
[monitor2-template]
filename = instance-monitor.cfg.jinja2.in
md5sum = 75fe1b222c269e69226796bf6059a747
md5sum = 86517ddccc86c76deaedf11066470550
......@@ -21,28 +21,23 @@ log = ${:var}/log
scripts = ${:etc}/run
services = ${:etc}/service
promises = ${:etc}/promise
plugins = ${:etc}/plugin
monitor = ${:srv}/monitor
monitor-promise = ${:etc}/monitor-promise
monitor-report = ${:etc}/monitor-report
[monitor-directory]
recipe = slapos.cookbook:mkdirectory
bin = ${directory:bin}
etc = ${directory:etc}
run = ${directory:monitor}/run
#run = ${directory:scripts}
promises = ${directory:monitor-promise}
reports = ${directory:monitor-report}
promises = ${directory:etc}/monitor-promise
reports = ${directory:etc}/monitor-report
pids = ${directory:run}/monitor
cgi-bin = ${directory:monitor}/cgi-bin
webdav = ${directory:monitor}/webdav
public = ${directory:monitor}/public
private = ${directory:monitor}/private
services = ${directory:services}
services-conf = ${directory:etc}/monitor.conf.d
documents = ${:private}/documents
log = ${directory:log}/monitor
monitor-var = ${directory:var}/monitor
monitor-log = ${directory:monitor}/private/monitor-log
promise-result = ${buildout:directory}/.slapgrid/promise/result
promise-log = ${buildout:directory}/.slapgrid/promise/log
[ca-directory]
recipe = slapos.cookbook:mkdirectory
......@@ -58,7 +53,7 @@ recipe = slapos.cookbook:certificate_authority
openssl-binary = {{ openssl_executable_location }}
ca-dir = ${ca-directory:root}
requests-directory = ${ca-directory:requests}
wrapper = ${monitor-directory:services}/certificate_authority
wrapper = ${directory:services}/certificate_authority
ca-private = ${ca-directory:private}
ca-certs = ${ca-directory:certs}
ca-newcerts = ${ca-directory:newcerts}
......@@ -78,18 +73,12 @@ root-title = ${monitor-instance-parameter:root-instance-title}
public-folder = ${monitor-directory:public}
private-folder = ${monitor-directory:private}
webdav-folder = ${monitor-directory:webdav}
report-folder = ${monitor-directory:reports}
base-url = ${monitor-instance-parameter:monitor-base-url}
monitor-hal-json = ${monitor-directory:public}/monitor.hal.json
service-pid-folder = ${monitor-directory:pids}
crond-folder = ${logrotate-directory:cron-entries}
logrotate-folder = ${logrotate-directory:logrotate-entries}
promise-runner = {{ monitor_runpromise }}
promise-folder = ${directory:promises}
monitor-promise-folder = ${monitor-directory:promises}
promises-timeout-file = ${monitor-promise-timeout-file:file}
log-folder = ${monitor-directory:log}
document-folder = ${monitor-directory:documents}
pid-file = ${monitor-directory:pids}/monitor-bootstrap.pid
randomsleep = {{ bin_directory }}/randomsleep
public-path-list =
private-path-list = ${directory:log}
......@@ -99,26 +88,28 @@ parameter-file-path = ${monitor-instance-parameter:configuration-file-path}
parameter-list =
raw monitor-user ${monitor-instance-parameter:username}
htpasswd monitor-password ${httpd-monitor-htpasswd:password-file} ${monitor-instance-parameter:username} ${httpd-monitor-htpasswd:htpasswd-path}
file promise-timeout ${monitor-promise-timeout-file:file}
file min-free-disk-MB ${promise-check-free-disk-space:config-file}
${monitor-instance-parameter:instance-configuration}
# htpasswd entry: htpasswd key password-file username htpasswd-file
collector-db = ${monitor-instance-parameter:collector-db}
collect-script = {{ monitor_collect }}
statistic-script = {{ monitor_statistic }}
python = {{ python_with_eggs }}
nice-cmd = ${xnice-bin:output}
promise-output-file = ${directory:monitor}/monitor-bootstrap-status
[monitor-promise-timeout-file]
recipe = plone.recipe.command
file = ${directory:etc}/promise_timeout
command =
if [ ! -s "${:file}" ]; then
echo "20" > ${:file}
fi
[monitor-promise-conf]
output-folder = ${monitor-directory:public}/promise
history-folder = ${monitor-directory:public}
promise-folder = ${directory:plugins}
legacy-promise-folder = ${directory:promises}
pid-path = ${monitor-directory:pids}/runpromise.pid
partition-folder = ${buildout:directory}
master-url = ${slap-connection:server-url}
partition-cert = ${slap-connection:cert-file}
partition-key = ${slap-connection:key-file}
partition-id = ${slap-connection:partition-id}
computer-id = ${slap-connection:computer-id}
ipv4 = ${slap-configuration:ipv4-random}
ipv6 = ${slap-configuration:ipv6-random}
software-release = ${slap-connection:software-release-url}
software-type = ${slap-configuration:slap-software-type}
[monitor-base-url-dict]
# place holder to be used to collect erp5 monitor urls
......@@ -129,39 +120,14 @@ template = {{ monitor_conf_template }}
rendered = ${directory:etc}/${:filename}
filename = monitor.conf
context = section parameter_dict monitor-conf-parameters
section promise_parameter_dict monitor-promise-conf
section monitor_base_urls monitor-base-url-dict
[instance-info-parameters]
name = ${monitor-instance-parameter:monitor-title}
root-name = ${monitor-instance-parameter:root-instance-title}
computer-id = ${slap-connection:computer-id}
ipv4 = ${slap-configuration:ipv4-random}
ipv6 = ${slap-configuration:ipv6-random}
software-release = ${slap-connection:software-release-url}
software-type = ${slap-configuration:slap-software-type}
partition-id = ${slap-connection:partition-id}
[monitor-instance-info]
recipe = slapos.recipe.template:jinja2
template = {{ monitor_instance_info }}
rendered = ${directory:etc}/${:filename}
filename = instance-info.conf
context =
section instance_dict instance-info-parameters
[python-symlink]
recipe = plone.recipe.command
target = ${directory:bin}
command = ln -sf {{ python_with_eggs }} ${:target}/python
update-command = ${:command}
[start-monitor]
recipe = slapos.cookbook:wrapper
command-line = {{ python_executable }} {{ monitor_bin }} --config_file ${monitor-conf:rendered}
command-line = {{ monitor_bin }} -c ${monitor-conf:rendered}
name = bootstrap-monitor
wrapper-path = ${directory:scripts}/${:name}
environment =
PATH=${python-symlink:target}:/usr/local/bin:/usr/bin:/bin
[monitor-htpasswd]
recipe = slapos.cookbook:generate.password
......@@ -183,12 +149,18 @@ update-command = ${:command}
user = ${monitor-instance-parameter:username}
password = ${monitor-instance-parameter:password}
[monitor-symlink]
recipe = cns.recipe.symlink
symlink =
${monitor-directory:promise-result} = ${monitor-directory:public}/promise
${monitor-directory:promise-result} = ${monitor-directory:log}/promise
[monitor-httpd-conf-parameter]
listening-ip = ${monitor-instance-parameter:monitor-httpd-ipv6}
port = ${monitor-instance-parameter:monitor-httpd-port}
pid-file = ${directory:run}/monitor-httpd.pid
access-log = ${monitor-directory:log}/monitor-httpd-access.log
error-log = ${monitor-directory:log}/monitor-httpd-error.log
access-log = ${directory:log}/monitor-httpd-access.log
error-log = ${directory:log}/monitor-httpd-error.log
cert-file = ${ca-directory:certs}/httpd.crt
key-file = ${ca-directory:certs}/httpd.key
htpasswd-file = ${httpd-monitor-htpasswd:htpasswd-path}
......@@ -235,8 +207,8 @@ command = kill -USR1 $(cat ${monitor-httpd-conf-parameter:pid-file})
[logrotate-entry-monitor-httpd]
<= logrotate-entry-base
name = monitor-apache
log = $${basedirectory:log}/monitor/monitor-httpd-error.log
post = test ! -s $${buildout:directory}/var/run/monitor-httpd.pid || $${buildout:directory}/bin/slapos-kill --pidfile $${buildout:directory}/var/run/monitor-httpd.pid -s USR1
log = ${directory:log}/monitor-httpd-*.log
post = test ! -s ${monitor-httpd-conf-parameter:pid-file} || {{ bin_directory }}/slapos-kill --pidfile ${monitor-httpd-conf-parameter:pid-file} -s USR1
[xnice-bin]
recipe = collective.recipe.template
......@@ -253,28 +225,74 @@ wrapper-path = ${directory:promises}/promise-monitor-httpd-is-process-older-than
[monitor-globalstate-wrapper]
recipe = slapos.cookbook:wrapper
command-line = ${xnice-bin:output} {{ monitor_genstatus }} '${monitor-conf:rendered}' '${monitor-instance-info:rendered}'
command-line = ${xnice-bin:output} {{ monitor_genstatus }} '${monitor-conf:rendered}'
wrapper-path = ${directory:bin}/monitor-globalstate
[monitor-configurator-wrapper]
recipe = slapos.cookbook:wrapper
# XXX - hard coded path
command-line = {{ monitor_configwrite }} --config_folder '${monitor-conf-parameters:private-folder}/config/.jio_documents' --output_cfg_file '${monitor-instance-parameter:configuration-file-path}' --htpasswd_bin '{{ apache_location }}/bin/htpasswd' --monitor_https_cors {{ monitor_https_cors }}
command-line = ${xnice-bin:output} {{ monitor_configwrite }}
--config_folder '${monitor-conf-parameters:private-folder}/config/.jio_documents'
--output_cfg_file '${monitor-instance-parameter:configuration-file-path}'
--htpasswd_bin '{{ apache_location }}/bin/htpasswd'
--monitor_https_cors {{ monitor_https_cors }}
wrapper-path = ${directory:bin}/monitor-configurator
[monitor-collect-wrapper]
recipe = slapos.cookbook:wrapper
command-line = ${xnice-bin:output} {{ monitor_collect }}
--output_folder ${monitor-directory:documents}
--collector_db ${monitor-instance-parameter:collector-db}
--pid_file ${monitor-directory:pids}/monitor-collect.pid
wrapper-path = ${directory:bin}/monitor-collect
[monitor-globalstate-cron-entry]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = monitor-globalstate
frequency = * * * * *
command = {{ bin_directory }}/randomsleep 60 && ${monitor-globalstate-wrapper:wrapper-path}
frequency = */2 * * * *
command = {{ bin_directory }}/randomsleep 20 && ${monitor-globalstate-wrapper:wrapper-path}
[monitor-configurator-cron-entry]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = monitor-configurator
frequency = * * * * *
command = {{ bin_directory }}/randomsleep 60 && ${monitor-configurator-wrapper:wrapper-path}
command = {{ bin_directory }}/randomsleep 10 && ${monitor-configurator-wrapper:wrapper-path}
[monitor-collect-cron-entry]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = monitor_collect
frequency = * * * * *
command = {{ bin_directory }}/randomsleep 40 && ${monitor-collect-wrapper:wrapper-path}
[logrotate-entry-monitor-data]
recipe = collective.recipe.template
name = monitor.data
log = ${monitor-directory:private}/*.data.json ${monitor-directory:documents}/*.data.json
input = inline:${:log} {
weekly
nocreate
olddir ${monitor-directory:documents}
rotate 104
nocompress
missingok
extension .json
dateext
dateformat -%Y-%m-%d
notifempty
}
output = ${logrotate-directory:logrotate-entries}/${:name}
mode = 600
[logrotate-entry-monitor-promise-history]
<= logrotate-entry-base
name = monitor.service.status
log = ${monitor-directory:public}/*.history.json
rotate-num = 0
frequency = weekly
pre = {{ monitor_statistic }} --history_folder ${monitor-directory:public}
[monitor-httpd-promise]
recipe = slapos.cookbook:check_url_available
......@@ -336,58 +354,35 @@ curl_path = {{ curl_executable_location }}
check-secure = 1
[monitor-bootstrap-promise]
recipe = collective.recipe.template
recipe = slapos.cookbook:promise.plugin
eggs =
slapos.toolbox
file = ${monitor-conf-parameters:promise-output-file}
error-log-file = ${buildout:directory}/.${slap-connection:partition-id}_${start-monitor:name}.log
input = inline:#!{{ dash_executable_location }}
pidfile=${monitor-conf-parameters:pid-file}
if [ -s $pidfile ]; then
COUNTER=0
# Wait until max 20 seconds, the limit promise timeout
while [ $COUNTER -lt 40 ]; do
if [ -n "$(ps -p $(cat $pidfile) -o pid=)" ]; then
((COUNTER=COUNTER+1))
sleep 0.5
else
break
fi
done
fi
if [ ! -f "${:file}" ]; then
echo "Monitor bootstrap exited with error."
log_file="${:error-log-file}"
if [ -s "$log_file" ]; then
echo " ---- Latest monitor-boostrap.log ----"
echo ""
tail -n 3 $log_file
fi
exit 2
else
echo "Bootstrap OK";
fi
output = ${directory:promises}/monitor-bootstrap-status
mode = 700
content =
from slapos.promise.plugin.monitor_bootstrap_status import RunPromise
output = ${directory:plugins}/monitor-bootstrap-status.py
mode = 600
config-process-pid-file = ${monitor-conf-parameters:pid-file}
config-process-name = ${start-monitor:name}
config-status-file = ${:file}
[promise-check-slapgrid]
recipe = collective.recipe.template
error-log-file = ${buildout:directory}/.slapgrid-${slap-connection:partition-id}-error.log
input = inline:#!/bin/sh
if [ -f "${:error-log-file}" ]; then
>&2 cat ${:error-log-file}
exit 1
fi
output = ${monitor-directory:promises}/buildout-${slap-connection:partition-id}-status
mode = 700
recipe = slapos.cookbook:promise.plugin
eggs =
slapos.toolbox
output = ${directory:plugins}/buildout-${slap-connection:partition-id}-status.py
content =
from slapos.promise.plugin.check_partition_deployment_state import RunPromise
config-monitor-url = ${monitor-instance-parameter:monitor-base-url}
mode = 600
[promise-check-free-disk-space]
recipe = slapos.recipe.template:jinja2
template = {{ template_check_disk_space }}
rendered = ${monitor-directory:promises}/check-free-disk-space
mode = 0700
context =
key config_file :config-file
raw home_path ${buildout:directory}
raw python_bin {{ python_with_eggs }}
recipe = slapos.cookbook:wrapper
command-line = {{ check_disk_space }}
--collectordb ${monitor-instance-parameter:collector-db}
--home_path ${buildout:directory}
--config ${:config-file}
wrapper-path = ${directory:promises}/check-free-disk-space
config-file = ${directory:etc}/min-free-disk-size
......@@ -400,6 +395,7 @@ base-url = ${monitor-conf-parameters:base-url}
depends =
${monitor-globalstate-cron-entry:name}
${monitor-configurator-cron-entry:name}
${monitor-collect-cron-entry:name}
${cron-entry-logrotate:name}
${logrotate-entry-cron:name}
${certificate-authority:wrapper}
......@@ -408,9 +404,12 @@ depends =
${ca-monitor-httpd:wrapper}
${monitor-httpd-promise:filename}
${monitor-bootstrap-promise:file}
${promise-check-slapgrid:output}
${monitor-symlink:recipe}
${promise-check-slapgrid:recipe}
${promise-monitor-httpd-is-process-older-than-dependency-set:wrapper-path}
${logrotate-entry-monitor-httpd:name}
${logrotate-entry-monitor-data:name}
${logrotate-entry-monitor-promise-history:name}
[monitor-publish]
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
......
#!{{ python_bin }}
import os
import sys
def free_space(path, fn):
while True:
try:
disk = os.statvfs(path)
return fn(disk)
except OSError:
pass
if os.sep not in path:
break
path = os.path.split(path)[0]
def user_free_space(path):
return free_space(path, lambda d: d.f_bsize * d.f_bavail)
def check_inode_usage(path):
max_inode_usage = 97.99 # < 98% usage
st = os.statvfs(path)
usage_output = ""
total_inode = st.f_files
free_inode = st.f_ffree
usage = round((float(total_inode - free_inode) / total_inode), 4) * 100
if usage > max_inode_usage:
return "Disk Inodes are widely used: %s%%" % usage
elif os.path.exists('/tmp'):
# check if /tmp is mounted on another disk than path
tmp_st = os.statvfs('/tmp')
if tmp_st.f_blocks != st.f_blocks:
tmp_usage = round((float(tmp_st.f_files - tmp_st.f_ffree) / tmp_st.f_files), 4) * 100
if tmp_usage > max_inode_usage:
return "Disk Inodes are widely used: %s%%" % tmp_usage
return ""
if __name__ == '__main__':
home_path = '{{ home_path }}'
config_file = '{{ config_file }}'
min_free_size = 1024*1024*1024*2 # 2G by default
if os.path.exists(config_file):
with open(config_file) as f:
min_size_str = f.read().strip()
if min_size_str == '0':
# disable check
print "Free disk space check is disabled\n set a number up to 0 to enable!"
exit(0)
if min_size_str.isdigit():
value = int(min_size_str)
if value >= 200:
# Minimum value is 200Mb, it's already low
min_free_size = int(min_size_str)*1024*1024
else:
with open(config_file, 'w') as f:
f.write(str(min_free_size/(1024*1024)))
real_free_space = user_free_space(home_path)
if real_free_space > min_free_size:
inode_usage = check_inode_usage(home_path)
if inode_usage:
print inode_usage
exit(2)
print "Disk usage: OK"
exit(0)
real_space_g = round(real_free_space/(1024.0*1024*1024), 2)
min_space_g = round(min_free_size/(1024.0*1024*1024), 2)
print 'Free disk space low: remaning %s G (threshold: %s G)' % (
real_space_g, min_space_g)
print 'You can modify minimum value on your monitor interface.'
exit(1)
[instance]
name = {{ instance_dict['name'] }}
root-name = {{ instance_dict['root-name'] }}
computer = {{ instance_dict['computer-id'] }}
ipv4 = {{ instance_dict['ipv4'] }}
ipv6 = {{ instance_dict['ipv6'] }}
software-release = {{ instance_dict['software-release'] }}
software-type = {{ instance_dict['software-type'] }}
partition = {{ instance_dict['partition-id'] }}
\ No newline at end of file
......@@ -11,4 +11,9 @@ monitor-url-list =
{% for key, value in monitor_base_urls.items() -%}
{{ ' ' ~ value }}
{% endfor -%}
{% endif -%}
{% endif %}
[promises]
{% for key, value in promise_parameter_dict.items() -%}
{{ key }} = {{ value.strip().replace("\n", "\n ") }}
{% endfor -%}
\ No newline at end of file
......@@ -15,7 +15,7 @@
# not need these here).
[pbsready]
filename = pbsready.cfg.in
md5sum = bceb082deb6b3ee9b015c8b04436b52c
md5sum = 367f8b8a09c3098844739f9b2856b912
[pbsready-import]
filename = pbsready-import.cfg.in
......@@ -27,7 +27,7 @@ md5sum = c6c11db5372150019debb1ce519b907d
[template-pull-backup]
filename = instance-pull-backup.cfg.in
md5sum = 5f35d9c0d08a268ed0c0f0dbef114f42
md5sum = 85e777bd349a5c881d36e747882aee8a
[template-replicated]
filename = template-replicated.cfg.in
......
......@@ -136,13 +136,13 @@ notifier-url = http://[$${notifier:host}]:$${notifier:port}
slave-instance-list = $${slap-parameter:slave_instance_list}
ignore-known-hosts-file = $${slap-parameter:ignore-known-hosts-file}
# To get a verbose feed about PBS state
instance-root-name = $${instance-info-parameters:root-name}
instance-root-name = $${monitor-instance-parameter:root-instance-title}
log-url = $${monitor-publish-parameters:monitor-base-url}/private/notifier/
status-item-directory = $${directory:notifier-status-items}
[pbs-resilient-status-feed]
recipe = slapos.cookbook:wrapper
command-line = ${buildout:directory}/bin/generatefeed --output $${:feed-path} --status-item-path $${pbs:status-item-directory} --title "Status feed for $${instance-info-parameters:root-name}-PBS" --link $${pbs:log-url}
command-line = ${buildout:directory}/bin/generatefeed --output $${:feed-path} --status-item-path $${pbs:status-item-directory} --title "Status feed for $${monitor-instance-parameter:root-instance-title}-PBS" --link $${pbs:log-url}
feed-path = $${directory:monitor-resilient}/pbs-status-rss
wrapper-path = $${rootdirectory:bin}/resilient-genstatusrss.py
......
......@@ -135,7 +135,7 @@ command = ${buildout:bin-directory}/pubsubserver --callbacks $${directory:notifi
notifier-binary = ${buildout:bin-directory}/pubsubnotifier
host = $${slap-network-information:global-ipv6}
port = $${notifier-port:port}
instance-root-name = $${instance-info-parameters:root-name}
instance-root-name = $${monitor-instance-parameter:root-instance-title}
log-url = $${monitor-publish-parameters:monitor-base-url}/resilient/notifier-status-rss
status-item-directory = $${directory:notifier-status-items}
context =
......
......@@ -131,8 +131,8 @@ pyparsing = 2.2.0
pytz = 2016.10
requests = 2.13.0
six = 1.10.0
slapos.cookbook = 1.0.59
slapos.core = 1.4.4
slapos.cookbook = 1.0.62
slapos.core = 1.4.7
slapos.extension.strip = 0.4
slapos.libnetworkcache = 0.15
slapos.rebootstrap = 4.1
......@@ -144,7 +144,7 @@ xml-marshaller = 0.9.7
paramiko = 2.1.3
# Required by:
# slapos.core==1.4.4
# slapos.core==1.4.7
Flask = 0.12
# Required by:
......@@ -160,11 +160,11 @@ functools32 = 3.2.3.post2
ipaddress = 1.0.18
# Required by:
# slapos.cookbook==1.0.59
# slapos.cookbook==1.0.62
jsonschema = 2.6.0
# Required by:
# slapos.core==1.4.4
# slapos.core==1.4.7
# XXX 'slapos node format' raises an exception with netifaces 0.10.5.
netifaces = 0.10.4
......@@ -177,15 +177,15 @@ packaging = 16.8
pycparser = 2.17
# Required by:
# slapos.core==1.4.4
# slapos.core==1.4.7
supervisor = 3.3.3
# Required by:
# slapos.core==1.4.4
# slapos.core==1.4.7
uritemplate = 3.0.0
# Required by:
# slapos.core==1.4.4
# slapos.core==1.4.7
zope.interface = 4.3.3
[networkcache]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment