Commit f6a30dcc authored by Rafael Monnerat's avatar Rafael Monnerat

Update Release Candidate

parents dc7ae622 7ce498dd
......@@ -8,6 +8,8 @@ parts =
[gowork]
# Caddy 1.x+ uses go modules, for which gowork does not work yet
golang = ${golang1.12:location}
gcc-bin-directory = ${golang1.12:gcc-bin-directory}
install =
[gowork.goinstall]
......
......@@ -17,10 +17,12 @@ configure-command = :
location = @@LOCATION@@
make-binary =
make-targets= cd src && ./all.bash && cp -alf .. ${:location}
# gcc version to use
gcc-bin-directory = ${gcc-8.2:location}/bin
# some testdata files have an issue with slapos.extension.strip.
post-install = ${findutils:location}/bin/find ${:location}/src -type d -name testdata -exec rm -rf {} \; || true
environment =
PATH=${gcc-8.2:location}/bin:%(PATH)s
PATH=${:gcc-bin-directory}:%(PATH)s
GOROOT_FINAL=${:location}
${:environment-extra}
......@@ -104,6 +106,9 @@ depends = ${gowork.goinstall:recipe}
# go version used for the workspace (possible to override in applications)
golang = ${golang1.10:location}
# gcc version must be compatible with go version selected
gcc-bin-directory = ${golang1.10:gcc-bin-directory}
# no special build flags by default
buildflags =
......@@ -123,7 +128,7 @@ recipe = slapos.recipe.template
url = ${:_profile_base_location_}/goenv.sh.in
output = ${gowork:directory}/env.sh
depends = ${gowork.mkdir:recipe}
md5sum = 7a067a3974c446c3eaa0e82818ba1adb
md5sum = 7eaad1f9aabd3cfad554975098c5d4c3
[gowork.mkdir]
# NOTE do not use slapos.cookbook:mkdirectory here - if anything in software (not instance)
......
......@@ -3,7 +3,7 @@
# ---- 8< ---- (buildout substitution here)
# PATH so that go & friends work out of the box
export PATH=${gowork:golang}/bin:${git:location}/bin:${pkgconfig:location}/bin:${buildout:bin-directory}:$PATH
export PATH=${gowork:golang}/bin:${git:location}/bin:${pkgconfig:location}/bin:${buildout:bin-directory}:${gowork:gcc-bin-directory}:$PATH
X=${gowork:directory}
export PKG_CONFIG_PATH=$(echo -n "${gowork:cpkgpath}" |tr '\n' ':'):$PKG_CONFIG_PATH
......
......@@ -15,8 +15,8 @@ extends =
[groonga]
recipe = slapos.recipe.cmmi
shared = false
url = https://packages.groonga.org/source/groonga/groonga-9.1.0.tar.gz
md5sum = 81722e0bc8cc0cfdddcafde2087b71df
url = https://packages.groonga.org/source/groonga/groonga-9.1.2.tar.gz
md5sum = 5266c49b758dde744854cb8cfe025812
# temporary patch to respect more tokens in natural language mode.
patches =
${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b
......
......@@ -26,7 +26,7 @@ install =
lab.nexedi.com/nexedi/helloweb/go/...
golang = ${golang1.12:location}
gcc-bin-directory = ${golang1.12:gcc-bin-directory}
# -*- go -*-
[helloweb-go]
......
......@@ -29,8 +29,8 @@ parts =
[mariadb]
recipe = slapos.recipe.cmmi
url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
version = 10.3.21
md5sum = c5b9ed98f2f57f7b809d4ec9c1515107
version = 10.3.22
md5sum = f712a5e6fde038d0c9c6d2a2cd88b84e
location = ${buildout:parts-directory}/${:_buildout_section_name_}
pre-configure =
set -e '\bSET(PLUGIN_AUTH_PAM YES)' cmake/build_configurations/mysql_release.cmake
......@@ -84,8 +84,8 @@ post-install =
# mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
# http://mroonga.github.com/
recipe = slapos.recipe.cmmi
url = https://packages.groonga.org/source/mroonga/mroonga-9.10.tar.gz
md5sum = 30a3010fd5ef11a8bd440053f1624bdb
url = https://packages.groonga.org/source/mroonga/mroonga-9.12.tar.gz
md5sum = d0af673f1bad3b9ccf33870bb2344a25
pre-configure = set -e
rm -rf fake_mariadb_source
mkdir -p fake_mariadb_source
......
......@@ -57,8 +57,8 @@ openssl = ${openssl:location}/bin/openssl
[openssl-1.0]
recipe = slapos.recipe.cmmi
shared = true
url = https://www.openssl.org/source/openssl-1.0.2t.tar.gz
md5sum = ef66581b80f06eae42f5268bc0b50c6d
url = https://www.openssl.org/source/openssl-1.0.2u.tar.gz
md5sum = cdc2638f789ecc2db2c91488265686c1
location = @@LOCATION@@
# 'prefix' option to override --openssldir/--prefix (which is useful
# when combined with INSTALL_PREFIX). Used by slapos.package.git/obs
......
......@@ -36,8 +36,14 @@ long_description = open("README.rst").read() + "\n" + \
for f in sorted(glob.glob(os.path.join('slapos', 'recipe', 'README.*.rst'))):
long_description += '\n' + open(f).read() + '\n'
# extras_requires are not used because of
# https://bugs.launchpad.net/zc.buildout/+bug/85604
extras_require = {
'test': (
'jsonschema',
'mock',
'testfixtures',
),
}
setup(name=name,
version=version,
description="SlapOS recipes.",
......@@ -201,10 +207,7 @@ setup(name=name,
'kumo = slapos.recipe.nosqltestbed.kumo:KumoTestBed',
],
},
extras_require=extras_require,
test_suite='slapos.test',
tests_require=[
'jsonschema',
'mock',
'testfixtures',
],
tests_require=extras_require['test'],
)
# grafana / telegraf / influxdb
This is an experimental integration, mainly to evaluate these solutions.
## Custom telegraf plugins
......@@ -7,24 +8,20 @@
See https://github.com/influxdata/telegraf to learn about plugins.
Useful plugins in this context are probably
[exec](https://github.com/influxdata/telegraf/tree/1.5.1/plugins/inputs/exec)
[exec](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/exec),
[logparser](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/logparser)
or
[httpjson](https://github.com/influxdata/telegraf/tree/1.5.1/plugins/inputs/httpjson).
[http](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/http).
Telegraf will save in the `telegraf` database from the embedded influxdb server.
## Grafana
You'll have to add yourself the influxdb data source in grafana, using the
parameters published by the slapos instance.
A default user is created, username and password are published as connection
parameters. You can add more users in grafana interface.
http://docs.grafana.org/features/datasources/influxdb/
When adding datasource, use *proxy* option, otherwise Grafana makes your
browser query influxdb directly, which also uses a self signed certificate.
One workaround is to configure your browser to also accept influxdb certificate
before using grafana, but using proxy seems easier.
Datasources should be automatically added.
## Influxdb
......@@ -34,8 +31,32 @@ One important thing to notice is that the backup protocol is enabled on ipv4
provided by slapos, so make sure this ip is not reachable from untrusted
sources.
## TODO
# Ingesting/Visualizing logs
Eventhough main feature is visualizing metrics, Grafana has a feature called "Explore" to view logs for a time frame.
The following backend can be used:
## Loki
See `TestLoki` in test for an example.
## Influxdb
Influxdb logs only have tags and there does not seem to be a way to search (except than tag and time frame).
To inject log files containing:
```
INFO the message
WARN another message
```
use config like:
```
[[inputs.logparser]]
files = ["/tmp/x*.log", "/tmp/aaa.log"]
* influxdb and telegraf runs with very low priority, this could become an option
* make one partition for each service and use switch software type
* make it easier to add custom configuration (how ?)
[inputs.logparser.grok]
measurement = "logs"
patterns = ['^%{WORD:level:tag} %{GREEDYDATA:message:string}']
```
......@@ -15,7 +15,7 @@
[instance-profile]
filename = instance.cfg.in
md5sum = 461d515da03de5e422e6f75189d09184
md5sum = 214a756a6a9f73455411760b2daa25b7
[influxdb-config-file]
filename = influxdb-config-file.cfg.in
......@@ -28,3 +28,15 @@ md5sum = a1a9c22c2a7829c66a49fc2504604d21
[grafana-config-file]
filename = grafana-config-file.cfg.in
md5sum = 8244d430905b968795c7946049bed9e3
[grafana-provisioning-config-file]
filename = grafana-provisioning-config-file.cfg.in
md5sum = 3aa0f1ed752b2a59ea2b5e7c1733daf3
[loki-config-file]
filename = loki-config-file.cfg.in
md5sum = 02ba5acf23fcf88f5594919f46838533
[promtail-config-file]
filename = promtail-config-file.cfg.in
md5sum = c77788d0a3cc654ad9393eb4b1f31e94
This diff is collapsed.
# https://grafana.com/docs/administration/provisioning/#example-datasource-config-file
apiVersion: 1
datasources:
- name: telegraf
type: influxdb
access: proxy
url: {{ influxdb['url'] }}
user: {{ influxdb['auth-username'] }}
database: telegraf
isDefault: true
jsonData:
tlsSkipVerify: true
secureJsonData:
password: {{ influxdb['auth-password'] }}
version: 1
editable: false
- name: loki
type: loki
access: proxy
url: {{ loki['url'] }}
version: 1
editable: false
......@@ -28,6 +28,11 @@
"description": "Name used in From: header of emails",
"default": "Grafana",
"type": "string"
},
"promtail-extra-scrape-config": {
"description": "Raw promtail config (experimental parameter, see https://github.com/grafana/loki/blob/v0.3.0/docs/promtail.md#scrape-configs for detail)",
"default": "",
"type": "string"
}
}
}
......@@ -3,8 +3,6 @@ parts =
promises
publish-connection-parameter
extends = {{ monitor_template }}
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
......@@ -35,17 +33,21 @@ etc = ${:home}/etc
var = ${:home}/var
srv = ${:home}/srv
service = ${:etc}/service
promise = ${:etc}/promise
influxdb-data-dir = ${:srv}/influxdb
grafana-dir = ${:srv}/grafana
grafana-data-dir = ${:grafana-dir}/data
grafana-logs-dir = ${:var}/log
grafana-plugins-dir = ${:grafana-dir}/plugins
grafana-provisioning-config-dir = ${:grafana-dir}/provisioning-config
grafana-provisioning-datasources = ${:grafana-provisioning-config-dir}/datasources
grafana-provisioning-dashboards = ${:grafana-provisioning-config-dir}/dashboards
grafana-provisioning-datasources-dir = ${:grafana-provisioning-config-dir}/datasources
grafana-provisioning-dashboards-dir = ${:grafana-provisioning-config-dir}/dashboards
telegraf-dir = ${:srv}/telegraf
telegraf-extra-config-dir = ${:telegraf-dir}/extra-config
loki-dir = ${:srv}/loki
loki-storage-boltdb-dir = ${:loki-dir}/index/
loki-storage-filesystem-dir = ${:loki-dir}/chunks/
promtail-dir = ${:srv}/promtail
# macros
[generate-certificate]
......@@ -71,11 +73,14 @@ mode = 0644
extensions = jinja2.ext.do
[check-port-listening-promise]
<= monitor-base-promise
module = check_port_listening
name = ${:_buildout_section_name_}.py
recipe = slapos.cookbook:check_port_listening
path = ${directory:promise}/${:_buildout_section_name_}
[check-url-available-promise]
recipe = slapos.cookbook:check_url_available
path = ${directory:promise}/${:_buildout_section_name_}
dash_path = {{ dash_bin }}
curl_path = {{ curl_bin }}
[influxdb]
ipv6 = ${instance-parameter:ipv6-random}
......@@ -112,27 +117,27 @@ username = influxdb
[influxdb-listen-promise]
<= check-port-listening-promise
config-hostname = ${influxdb:ipv6}
config-port = ${influxdb:http-port}
hostname = ${influxdb:ipv6}
port = ${influxdb:http-port}
[influxdb-password-promise]
<= monitor-base-promise
module = check_command_execute
name = ${:_buildout_section_name_}.py
config-command =
recipe = slapos.cookbook:wrapper
command-line =
{{ influx_bin }} -username ${influxdb:auth-username} -password ${influxdb:auth-password} -socket ${influxdb:unix-socket} -execute "CREATE USER ${influxdb:auth-username} WITH PASSWORD '${influxdb:auth-password}' WITH ALL PRIVILEGES"
wrapper-path = ${directory:promise}/${:_buildout_section_name_}
[grafana]
ipv6 = ${instance-parameter:ipv6-random}
port = 8080
port = 8180
url = https://[${:ipv6}]:${:port}
data-dir = ${directory:grafana-data-dir}
logs-dir = ${directory:grafana-logs-dir}
plugins-dir = ${directory:grafana-plugins-dir}
provisioning-config-dir = ${directory:grafana-provisioning-config-dir}
provisioning-datasources-dir = ${directory:grafana-provisioning-datasources-dir}
admin-user = ${grafana-password:username}
admin-password = ${grafana-password:passwd}
secret-key = ${grafana-secret-key:passwd}
......@@ -160,11 +165,20 @@ context =
section grafana grafana
section apache_frontend apache-frontend
key slapparameter_dict slap-configuration:configuration
depends =
${grafana-provisioning-config-file:rendered}
[grafana-provisioning-config-file]
<= config-file
rendered = ${grafana:provisioning-datasources-dir}/datasource.yaml
context =
section influxdb influxdb
section loki loki
[grafana-listen-promise]
<= check-port-listening-promise
config-hostname= ${grafana:ipv6}
config-port = ${grafana:port}
hostname= ${grafana:ipv6}
port = ${grafana:port}
......@@ -183,6 +197,53 @@ context =
section telegraf telegraf
[loki]
recipe = slapos.cookbook:wrapper
command-line =
bash -c 'nice -19 chrt --idle 0 ionice -c3 {{ loki_bin }} -config.file=${loki-config-file:rendered}'
wrapper-path = ${directory:service}/loki
storage-boltdb-dir = ${directory:loki-storage-boltdb-dir}
storage-filesystem-dir = ${directory:loki-storage-filesystem-dir}
ip = ${instance-parameter:ipv4-random}
port = 3100
url = http://${:ip}:${:port}
[loki-config-file]
<= config-file
context =
section loki loki
[loki-listen-promise]
<= check-url-available-promise
url = ${loki:url}/ready
[promtail]
recipe = slapos.cookbook:wrapper
command-line =
bash -c 'nice -19 chrt --idle 0 ionice -c3 {{ promtail_bin }} -config.file=${promtail-config-file:rendered}'
wrapper-path = ${directory:service}/promtail
dir = ${directory:promtail-dir}
http_port = 19080
ip = ${instance-parameter:ipv4-random}
url = http://${:ip}:${:http_port}
[promtail-config-file]
<= config-file
context =
section promtail promtail
section loki loki
key slapparameter_dict slap-configuration:configuration
[promtail-listen-promise]
<= check-port-listening-promise
hostname= ${promtail:ip}
port = ${promtail:http_port}
[apache-frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
......@@ -201,6 +262,8 @@ instance-promises =
${influxdb-listen-promise:path}
${influxdb-password-promise:wrapper-path}
${grafana-listen-promise:path}
${loki-listen-promise:path}
${promtail-listen-promise:path}
[publish-connection-parameter]
......@@ -213,4 +276,6 @@ telegraf-extra-config-dir = ${telegraf:extra-config-dir}
grafana-url = ${grafana:url}
grafana-username = ${grafana:admin-user}
grafana-password = ${grafana:admin-password}
loki-url = ${loki:url}
promtail-url = ${promtail:url}
url = ${apache-frontend:connection-secure_access}
auth_enabled: false
server:
http_listen_port: {{ loki['port'] }}
ingester:
lifecycler:
address: {{ loki['ip'] }}
ring:
kvstore:
store: inmemory
replication_factor: 1
chunk_idle_period: 15m
schema_config:
configs:
- from: 2018-04-15
store: boltdb
object_store: filesystem
schema: v9
index:
prefix: index_
period: 168h
storage_config:
boltdb:
directory: {{ loki['storage-boltdb-dir'] }}
filesystem:
directory: {{ loki['storage-filesystem-dir'] }}
limits_config:
enforce_metric_name: false
reject_old_samples: true
reject_old_samples_max_age: 168h
chunk_store_config:
max_look_back_period: 0
table_manager:
chunk_tables_provisioning:
inactive_read_throughput: 0
inactive_write_throughput: 0
provisioned_read_throughput: 0
provisioned_write_throughput: 0
index_tables_provisioning:
inactive_read_throughput: 0
inactive_write_throughput: 0
provisioned_read_throughput: 0
provisioned_write_throughput: 0
retention_deletes_enabled: false
retention_period: 0
# https://github.com/grafana/loki/blob/master/docs/logentry/processing-log-lines.md
server:
http_listen_port: {{ promtail['http_port'] }}
# XXX this external_url does not work ... promtail still listen on all IPs
external_url: {{ promtail['url'] }}
grpc_listen_port: 0
positions:
filename: {{ promtail['dir'] }}/positions.yaml
clients:
- url: {{ loki['url'] }}/api/prom/push
scrape_configs:
- job_name: test
static_configs:
- targets:
- localhost
labels:
job: grafanalogs
__path__: ./var/log/*log
{{ slapparameter_dict.get('promtail-extra-scrape-config', '') }}
......@@ -2,10 +2,11 @@
extends =
../../stack/slapos.cfg
../../stack/nodejs.cfg
../../stack/monitor/buildout.cfg
../../component/make/buildout.cfg
../../component/golang/buildout.cfg
../../component/openssl/buildout.cfg
../../component/curl/buildout.cfg
../../component/dash/buildout.cfg
buildout.hash.cfg
gowork.cfg
......@@ -17,21 +18,24 @@ parts =
influxdb-config-file
telegraf-config-file
grafana-config-file
grafana-provisioning-config-file
loki-config-file
promtail-config-file
[nodejs]
<= nodejs-8.6.0
<= nodejs-10.6.0
[yarn]
# this could become a component, but it needs to be invoked from nodejs explicitly,
# otherwise it uses system's nodejs
recipe = slapos.recipe.build:download-unpacked
url = https://github.com/yarnpkg/yarn/releases/download/v1.3.2/yarn-v1.3.2.tar.gz
md5sum = db82fa09c996e9318f2f1d2ab99228f9
url = https://github.com/yarnpkg/yarn/releases/download/v1.16.0/yarn-v1.16.0.tar.gz
md5sum = 46790033c23803387890f545e4040690
[gowork]
# All the softwares installed in the go work have "non standard" installation
# All the softwares installed in the go workspace have "non standard" installation
# methods, so we install them in specific parts with custom commands.
# They will be installed because they are dependencies of ${gowork.goinstall}
install =
......@@ -41,6 +45,12 @@ influx-bin = ${:bin}/influx
influxd-bin = ${:bin}/influxd
grafana-bin = ${:bin}/grafana-server
grafana-homepath = ${go_github.com_grafana_grafana:location}
loki-bin = ${:bin}/loki
promtail-bin = ${:bin}/promtail
# use recent go
golang = ${golang1.12:location}
gcc-bin-directory = ${golang1.12:gcc-bin-directory}
[gowork.goinstall]
command = :
......@@ -48,17 +58,22 @@ depends =
${influxdb-install:recipe}
${telegraf-install:recipe}
${grafana-install:recipe}
${loki-install:recipe}
${promtail-install:recipe}
[influxdb-install]
<= gowork.goinstall
command = bash -c ". ${gowork:env.sh} && \
go install -v github.com/golang/dep/cmd/dep && \
cd ${gowork:directory}/src/github.com/influxdata/influxdb && \
dep ensure && \
go install ./cmd/..."
update-command =
[telegraf-install]
<= gowork.goinstall
command = bash -c ". ${gowork:env.sh} && \
go install -v github.com/golang/dep/cmd/dep && \
cd ${gowork:directory}/src/github.com/influxdata/telegraf && \
${make:location}/bin/make &&
cp telegraf ${gowork:bin}"
......@@ -70,12 +85,32 @@ update-command =
command = bash -c "export PATH=${nodejs:location}/bin/:$PATH && \
. ${gowork:env.sh} && \
cd ${gowork:directory}/src/github.com/grafana/grafana && \
${gowork:golang}/bin/go run build.go setup && \
${gowork:golang}/bin/go run build.go build && \
go run build.go setup && \
go run build.go build && \
${yarn:location}/bin/yarn install --pure-lockfile && \
${nodejs:location}/bin/npm run build"
update-command =
[loki-install]
<= gowork.goinstall
# loki also uses nodejs
command = bash -c "export PATH=${nodejs:location}/bin/:$PATH && \
. ${gowork:env.sh} && \
go install -v github.com/golang/dep/cmd/dep && \
cd ${gowork:directory}/src/github.com/grafana/loki && \
go install ./cmd/loki"
update-command =
[promtail-install]
<= gowork.goinstall
# CGO_ENABLED is to disable systemd support (did not compile in my case)
command = bash -c "export CGO_ENABLED=0 && \
. ${gowork:env.sh} && \
go install -v github.com/golang/dep/cmd/dep && \
cd ${gowork:directory}/src/github.com/grafana/loki && \
go install ./cmd/promtail"
update-command =
[download-file-base]
recipe = slapos.recipe.build:download
......@@ -92,6 +127,15 @@ mode = 0644
[grafana-config-file]
<= download-file-base
[grafana-provisioning-config-file]
<= download-file-base
[loki-config-file]
<= download-file-base
[promtail-config-file]
<= download-file-base
[instance-profile]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:filename}
......@@ -106,10 +150,13 @@ context =
key influx_bin gowork:influx-bin
key grafana_bin gowork:grafana-bin
key grafana_homepath gowork:grafana-homepath
key monitor_template monitor2-template:rendered
key loki_bin gowork:loki-bin
key promtail_bin gowork:promtail-bin
key curl_bin :curl-bin
key dash_bin :dash-bin
curl-bin = ${curl:location}/bin/curl
dash-bin = ${dash:location}/bin/dash
[versions]
slapos.recipe.template = 4.2
inotifyx = 0.2.2
Tests for Grafana software release
##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.grafana'
long_description = open("README.md").read()
setup(
name=name,
version=version,
description="Test for SlapOS' Grafana",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'erp5.util',
'requests',
'supervisor',
'psutil',
],
zip_safe=True,
test_suite='test',
)
##############################################################################
#
# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import textwrap
import logging
import tempfile
import time
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class GrafanaTestCase(SlapOSInstanceTestCase):
"""Base test case for grafana.
Since the instances takes timte to start and stop,
we increate as lot the number of retries.
"""
report_max_retry = 30
instance_max_retry = 30
class TestGrafana(GrafanaTestCase):
def setUp(self):
self.grafana_url = self.computer_partition.getConnectionParameterDict(
)['grafana-url']
def test_grafana_available(self):
resp = requests.get(self.grafana_url, verify=False)
self.assertEqual(requests.codes.ok, resp.status_code)
def test_grafana_api(self):
# check API is usable
api_org_url = '{self.grafana_url}/api/org'.format(**locals())
resp = requests.get(api_org_url, verify=False)
self.assertEqual(requests.codes.unauthorized, resp.status_code)
connection_params = self.computer_partition.getConnectionParameterDict()
resp = requests.get(
api_org_url,
verify=False,
auth=requests.auth.HTTPBasicAuth(
connection_params['grafana-username'],
connection_params['grafana-password'],
))
self.assertEqual(requests.codes.ok, resp.status_code)
self.assertEqual(1, resp.json()['id'])
def test_grafana_datasource_povisinonned(self):
# data sources are provisionned
connection_params = self.computer_partition.getConnectionParameterDict()
resp = requests.get(
'{self.grafana_url}/api/datasources'.format(**locals()),
verify=False,
auth=requests.auth.HTTPBasicAuth(
connection_params['grafana-username'],
connection_params['grafana-password'],
))
self.assertEqual(requests.codes.ok, resp.status_code)
self.assertEqual(
sorted(['influxdb', 'loki']),
sorted([ds['type'] for ds in resp.json()]))
class TestInfluxDb(GrafanaTestCase):
def setUp(self):
self.influxdb_url = self.computer_partition.getConnectionParameterDict(
)['influxdb-url']
def test_influxdb_available(self):
ping_url = '{self.influxdb_url}/ping'.format(**locals())
resp = requests.get(ping_url, verify=False)
self.assertEqual(requests.codes.no_content, resp.status_code)
def test_influxdb_api(self):
query_url = '{self.influxdb_url}/query'.format(**locals())
connection_params = self.computer_partition.getConnectionParameterDict()
for i in range(10):
# retry, as it may take a little delay to create databases
resp = requests.get(
query_url,
verify=False,
params=dict(
q='SHOW DATABASES',
u=connection_params['influxdb-username'],
p=connection_params['influxdb-password']))
self.assertEqual(requests.codes.ok, resp.status_code)
result, = resp.json()['results']
if result['series'] and 'values' in result['series'][0]:
break
time.sleep(0.5 * i)
self.assertIn(
[connection_params['influxdb-database']], result['series'][0]['values'])
class TestTelegraf(GrafanaTestCase):
def test_telegraf_running(self):
with self.slap.instance_supervisor_rpc as supervisor:
all_process_info = supervisor.getAllProcessInfo()
process_info, = [p for p in all_process_info if 'telegraf' in p['name']]
self.assertEqual('RUNNING', process_info['statename'])
class TestLoki(GrafanaTestCase):
@classmethod
def getInstanceParameterDict(cls):
cls._logfile = tempfile.NamedTemporaryFile(suffix='log')
return {
'promtail-extra-scrape-config':
textwrap.dedent(
r'''
- job_name: {cls.__name__}
pipeline_stages:
- regex:
expression: "^(?P<timestamp>.*) - (?P<name>\\S+) - (?P<level>\\S+) - (?P<message>.*)"
- timestamp:
format: 2006-01-02T15:04:05Z00:00
source: timestamp
- labels:
level:
name:
static_configs:
- targets:
- localhost
labels:
job: {cls.__name__}
__path__: {cls._logfile.name}
''').format(**locals())
}
@classmethod
def tearDownClass(cls):
cls._logfile.close()
super(TestLoki, cls).tearDownClass()
def setUp(self):
self.loki_url = self.computer_partition.getConnectionParameterDict(
)['loki-url']
def test_loki_available(self):
self.assertEqual(
requests.codes.ok,
requests.get('{self.loki_url}/ready'.format(**locals()),
verify=False).status_code)
def test_log_ingested(self):
# create a logger logging to the file that we have
# configured in instance parameter.
test_logger = logging.getLogger(self.id())
test_logger.setLevel(logging.INFO)
test_handler = logging.FileHandler(filename=self._logfile.name)
test_handler.setFormatter(
logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
test_logger.addHandler(test_handler)
test_logger.info("testing message")
test_logger.info("testing another message")
test_logger.warning("testing warn")
# Check our messages have been ingested
# we retry a few times, because there's a short delay until messages are
# ingested and returned.
for i in range(10):
resp = requests.get(
'{self.loki_url}/api/prom/query?query={{job="TestLoki"}}'.format(
**locals()),
verify=False).json()
if not resp:
time.sleep(0.5 * i)
continue
warn_stream_list = [stream for stream in resp['streams'] if 'level="WARNING"' in stream['labels']]
self.assertEqual(1, len(warn_stream_list), resp['streams'])
warn_stream, = warn_stream_list
self.assertIn("testing warn", warn_stream['entries'][0]['line'])
info_stream_list = [stream for stream in resp['streams'] if 'level="INFO"' in stream['labels']]
self.assertEqual(1, len(info_stream_list), resp['streams'])
info_stream, = info_stream_list
self.assertTrue(
[
line for line in info_stream['entries']
if "testing message" in line['line']
])
self.assertTrue(
[
line for line in info_stream['entries']
if "testing another message" in line['line']
])
# The labels we have configued are also available
resp = requests.get(
'{self.loki_url}/api/prom/label'.format(**locals()),
verify=False).json()
self.assertIn('level', resp['values'])
self.assertIn('name', resp['values'])
......@@ -15,4 +15,4 @@
[template]
filename = instance.cfg
md5sum = 4664f7dae66d3f582e34cec2ca627501
md5sum = 1cbab58e896ff63575f6a67db530d183
......@@ -28,7 +28,7 @@ bin = $${buildout:directory}/bin
working-dir = $${buildout:directory}/tmp
[test-list]
path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup},${slapos.test.turnserver-setup:setup},${slapos.test.theia-setup:setup}
path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup},${slapos.test.turnserver-setup:setup},${slapos.test.theia-setup:setup},${slapos.test.grafana-setup:setup}
[slapos-test-runner]
recipe = slapos.cookbook:wrapper
......
......@@ -24,7 +24,7 @@ recipe = zc.recipe.egg:develop
[slapos.cookbook-setup]
<= setup-develop-egg
egg = slapos.cookbook
egg = slapos.cookbook[test]
setup = ${slapos-repository:location}/
[slapos.test.caddy-frontend-setup]
......@@ -107,6 +107,11 @@ setup = ${slapos-repository:location}/software/turnserver/test/
egg = slapos.test.theia
setup = ${slapos-repository:location}/software/theia/test/
[slapos.test.grafana-setup]
<= setup-develop-egg
egg = slapos.test.grafana
setup = ${slapos-repository:location}/software/grafana/test/
[slapos.core-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.core.git
......@@ -210,3 +215,7 @@ Django = 1.11
urllib3 = 1.24.1
backports.lzma = 0.0.13
mock = 2.0.0
testfixtures = 6.11
funcsigs = 1.0.2
......@@ -55,14 +55,13 @@ depends = ${slapos.core-setup:egg}
[slapos.cookbook-setup]
<= setup-develop-egg
# XXX slapos.cookbook does not have `test` extra require, `mock` is only listed in `tests_require` and is listed explicitly
egg = slapos.cookbook
egg = slapos.cookbook[test]
setup = ${slapos.cookbook-repository:location}
depends = ${slapos.core-setup:egg}
[slapos.core-setup]
<= setup-develop-egg
# XXX slapos.cookbook does not have `test` extra require, `mock`, `pyflakes` and `httmock` are only listed in `tests_require` and are listed explicitly
# XXX slapos.core does not have `test` extra require, `mock`, `pyflakes` and `httmock` are only listed in `tests_require` and are listed explicitly
egg = slapos.core
setup = ${slapos.core-repository:location}
......@@ -201,6 +200,7 @@ gitdb2 = 2.0.4
httmock = 1.2.6
manuel = 1.9.0
mock = 2.0.0
testfixtures = 6.11.0
pem = 18.1.0
pycurl = 7.43.0.2
pyflakes = 2.0.0
......
......@@ -5,12 +5,8 @@ extends =
ztk-versions.cfg
zope-versions.cfg
buildout.hash.cfg
../../component/cups/buildout.cfg
../../component/dbus/buildout.cfg
../../component/file/buildout.cfg
../../component/fonts/buildout.cfg
../../component/gcc/buildout.cfg
../../component/ghostscript/buildout.cfg
../../component/git/buildout.cfg
../../component/graphviz/buildout.cfg
../../component/gzip/buildout.cfg
......@@ -19,13 +15,9 @@ extends =
../../component/findutils/buildout.cfg
../../component/librsvg/buildout.cfg
../../component/imagemagick/buildout.cfg
../../component/inkscape/buildout.cfg
../../component/kumo/buildout.cfg
../../component/libdmtx/buildout.cfg
../../component/libffi/buildout.cfg
../../component/libpng/buildout.cfg
../../component/matplotlib/buildout.cfg
../../component/mesa/buildout.cfg
../../component/numpy/buildout.cfg
../../component/statsmodels/buildout.cfg
../../component/h5py/buildout.cfg
......@@ -44,7 +36,6 @@ extends =
../../component/subversion/buildout.cfg
../../component/tesseract/buildout.cfg
../../component/w3m/buildout.cfg
../../component/xorg/buildout.cfg
../../component/poppler/buildout.cfg
../../component/zabbix/buildout.cfg
../../component/sed/buildout.cfg
......@@ -56,11 +47,9 @@ extends =
../../component/aspell/buildout.cfg
../../component/jsl/buildout.cfg
../../component/6tunnel/buildout.cfg
../../component/findutils/buildout.cfg
../../component/userhosts/buildout.cfg
../../component/postfix/buildout.cfg
../../component/zbarlight/buildout.cfg
../../component/perl/buildout.cfg
../../component/pylint/buildout.cfg
../../stack/caucase/buildout.cfg
../../software/jupyter/software.cfg
......@@ -71,52 +60,20 @@ parts +=
erp5-util-develop
slapos-cookbook
rdiff-backup
aspell
aspell-en-dictionary
apache
apache-antiloris
file
findutils
graphviz
haproxy
jsl
w3m
poppler
libpng
ghostscript
mroonga-mariadb
imagemagick
inkscape
libdmtx
dmtx-utils
kumo
tesseract
tesseract-eng-traineddata
tesseract-osd-traineddata
scipy
ocropy
hookbox
percona-toolkit
zabbix-agent
dash
bash
wget
userhosts
postfix
# Buildoutish
eggs
eggs-all-scripts
testrunner
test_suite_runner
# basic Xorg
libXdmcp
libXext
libXau
libSM
libXrender
# fonts
liberation-fonts
ipaex-fonts
......@@ -124,9 +81,7 @@ parts +=
ocrb-fonts
android-fonts
# get git repositories
erp5
genbt5list
# some additional utils
......@@ -136,9 +91,7 @@ parts +=
template
# jupyter
jupyter
jupyter-notebook-initialized-scripts
instance-jupyter-notebook
# override python2.7 to add SlapOS libstdc++ in RPATH.
[python2.7]
......@@ -225,47 +178,24 @@ context =
key mariadb_link_binary template-mariadb:link-binary
key zope_link_binary template-zope:link-binary
key apache_location apache:location
key aspell_location aspell:location
key bin_directory buildout:bin-directory
key buildout_bin_directory buildout:bin-directory
key cairo_location cairo:location
key caucase_jinja2_library caucase-jinja2-library:target
key coreutils_location coreutils:location
key cups_location cups:location
key curl_location curl:location
key cyrus_sasl_location cyrus-sasl:location
key dash_location dash:location
key bash_location bash:location
key dbus_glib_location dbus-glib:location
key dbus_location dbus:location
key dcron_location dcron:location
key default_cloudooo_url erp5-defaults:cloudooo-connection-url
key erp5_location erp5:location
key file_location file:location
key findutils_location findutils:location
key fontconfig_location fontconfig:location
key fonts_location fonts:location
key freetype_location freetype:location
key glib_location glib:location
key glu_location glu:location
key gzip_location gzip:location
key haproxy_location haproxy:location
key imagemagick_location imagemagick:location
key instance_common_cfg instance-common:rendered
key jsl_location jsl:location
key jupyter_enable_default erp5-defaults:jupyter-enable-default
key kumo_location kumo:location
key libICE_location libICE:location
key libSM_location libSM:location
key libX11_location libX11:location
key libXau_location libXau:location
key libXdmcp_location libXdmcp:location
key libXext_location libXext:location
key libXrender_location libXrender:location
key libexpat_location libexpat:location
key libffi_location libffi:location
key librsvg_location librsvg:location
key libxcb_location libxcb:location
key local_bt5_repository local-bt5-repository:list
key logrotate_location logrotate:location
key mariadb_location mariadb:location
......@@ -273,13 +203,10 @@ context =
key mariadb_slow_query_report_script mariadb-slow-query-report-script:target
key mariadb_start_clone_from_backup mariadb-start-clone-from-backup:target
key matplotlibrc_location matplotlibrc:location
key mesa_location mesa:location
key parts_directory buildout:parts-directory
key openssl_location openssl:location
key percona_toolkit_location percona-toolkit:location
key perl_dbd_mariadb_path perl-DBD-mariadb:perl-PATH
key pixman_location pixman:location
key poppler_location poppler:location
key postfix_location postfix:location
key root_common root-common:target
key site_zcml site-zcml:target
......@@ -306,9 +233,6 @@ context =
key userhosts_location userhosts:location
key unixodbc_location unixodbc:location
key wget_location wget:location
key xdamage_location xdamage:location
key xfixes_location xfixes:location
key zlib_location zlib:location
key extra_path_list eggs:extra-paths
[template-erp5]
......
......@@ -138,7 +138,7 @@ pytz = 2016.10
requests = 2.13.0
six = 1.12.0
slapos.cookbook = 1.0.124
slapos.core = 1.5.7
slapos.core = 1.5.8
slapos.extension.strip = 0.4
slapos.extension.shared = 1.0
slapos.libnetworkcache = 0.20
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment