Skip to content

slapos
slapos
Commit 6e65bb9d authored by Łukasz Nowak's avatar Łukasz Nowak
Browse files
Options

monitor: Edge testing of urls 

Adds edgetest and edgebot software types, where edgetest is a software type
to request by user and requests by itself edgebot, which does the monitoring
work.

Based on Rafael Monnerat <rafael@nexedi.com>
parent a796981d
Show whitespace changes
Inline Side-by-side
Showing with 804 additions and 107 deletions
software/monitor/README.edgetest.rst 0 → 100644
View file @ 6e65bb9d
============
Edge testing
============
``edgetest`` is a special software type of monitor software release used for website monitoring by using bots.
It uses `surykatka <https://lab.nexedi.com/nexedi/surykatka>`_ and `check_surykatka_json <https://lab.nexedi.com/nexedi/slapos.toolbox/blob/master/slapos/promise/plugin/check_surykatka_json.py>`_ to monitor websites.
``surykatka`` provides a bot to query list of hosts and a JSON reporting system.
``check_surykatka_json`` is used in promises to provide monitoring information about the websites.
In order to monitor an url one need to:
* request a monitor software release with ``edgetest`` software type, configured as described in ``instance-edgetest-input-schema.json``,
* request a slave to monitor with ``edgetest`` software type, configured as described in ``instance-edgetest-slave-input-schema.json``.
software/monitor/buildout.hash.cfg
View file @ 6e65bb9d
......@@ -14,16 +14,32 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = 1b7d2d097f208f6641bf98a17df079c8
md5sum = cb814297f2f76dc8e08014a7d662439f
[template-monitor]
_update_hash_filename_ = instance-monitor.cfg.jinja2
md5sum = 373c79480e6425c20480fc911a56c3fd
[template-monitor-distributor]
_update_hash_filename_ = instance-monitor-distributor.cfg.jinja2
md5sum = 61c0bfdfc0a2b51ba15fe4a49baf6091
md5sum = 165a15672fc85981f68b9af2d6253254
[json-test-template]
_update_hash_filename_ = json-test-template.json.in.jinja2
md5sum = 2eb5596544d9c341acf653d4f7ce2680
[template-monitor-edgetest]
_update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2
md5sum = 9e237dbdda59e788202f0da194a57d41
[template-monitor-edgebot]
_update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
md5sum = cd4c5d2ecf8285ea8f0905ed1150d8a0
[network-bench-cfg]
filename = network_bench.cfg.in
md5sum = cfcbf2002b8eff5153e2bf68ed24b720
[monitor-collect-csv-dump]
filename = script/collect_csv_dump.py
md5sum = cad2402bbd21907cfed6bc5af8c5d3ab
[template-surykatka-ini]
_update_hash_filename_ = surykatka.ini.jinja2
md5sum = 40870921e05d93b5843ab34abd7e3902
software/monitor/instance-default-input-schema.json 0 → 100644
View file @ 6e65bb9d
{
"$schema": "http://json-schema.org/draft-04/schema#",
"properties": {
"dummy": {
"title": "dummy",
"description": "Dummy",
"type": "string"
}
}
}
software/monitor/instance-default-output-schema.json 0 → 100644
View file @ 6e65bb9d
{
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Values returned by Re6st Master instanciation",
"properties": {
"re6stry-url": {
"description": "ipv6 url to access your re6st registry service",
"type": "string"
}
},
"type": "object"
}
software/monitor/instance-edgetest-input-schema.json 0 → 100644
View file @ 6e65bb9d
{
"$schema": "http://json-schema.org/draft-04/schema#",
"properties": {
"nameserver": {
"default": "",
"title": "Nameserver",
"description": "Space separated list of name servers to use.",
"type": "string"
},
"check-status-code": {
"default": "200",
"title": "Default Check HTTP Code",
"description": "Default HTTP code to check against (default: 200).",
"type": "string"
},
"check-frontend-ip": {
"default": "",
"title": "Default space separated list of Frontend IPs to check",
"description": "Default list of Frontend IPs to check, if empty no constraint is used.",
"type": "string"
}
}
}
software/monitor/instance-edgetest-slave-input-schema.json 0 → 100644
View file @ 6e65bb9d
{
"$schema": "http://json-schema.org/draft-04/schema#",
"properties": {
"url": {
"title": "URL to check",
"description": "URL to check, like https://example.com",
"type": "string"
},
"check-status-code": {
"default": "Master default",
"title": "Default Check HTTP Code.",
"description": "HTTP code to check against (default: comes from master partition).",
"type": "string"
},
"check-frontend-ip": {
"default": "Master default",
"title": "Space separated list of Frontend IPs to check",
"description": "List of Frontend IPs to check, if empty no constraint is used (default: comes from master partition).",
"type": "string"
}
}
}
software/monitor/instance-monitor-distributor.cfg.jinja2 deleted 100644 → 0
View file @ a796981d
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:filename}
extra-context =
context =
import json_module json
${:extra-context}
[slave-test-configuration]
<=jinja2-template-base
template = {{ template_json_distributor_test }}
filename = srv/monitor/private/test.json
extensions = jinja2.ext.do
extra-context =
section slave_information slap-configuration
{% set part_list = [] -%}
# Publish information for each slave
{% set directory_list = [] -%}
{% for slave_instance in slave_instance_list -%}
{% set publish_section_title = 'publish-%s' % slave_instance.get('slave_reference') -%}
{% do part_list.append(publish_section_title) -%}
[{{ publish_section_title }}]
recipe = slapos.cookbook:publish
-slave-reference = {{ slave_instance.get('slave_reference') }}
log-access-url = ${monitor-frontend:connection-site_url}/{{ slave_instance.get('slave_reference') }}
log-access-url-v6 = ${monitor-httpd-conf-parameter:url}/{{ slave_instance.get('slave_reference') }}
{% endfor %}
{% set data_source_dict = slapparameter_dict.get('data-source', None) -%}
{% set cron_min_count = 0 -%}
{% if data_source_dict %}
{% for entry in data_source_dict -%}
{% set cron_min_count = cron_min_count + 1 -%}
{% set cron_min = cron_min_count%60 -%}
{% do part_list.append('cron-crawl-' + entry) -%}
{% do directory_list.append(entry) -%}
[cron-crawl-{{ entry }}]
<= cron
recipe = slapos.cookbook:cron.d
name = cron-crawler-{{ entry }}
frequency = * * * * *
command = cd ${monitor-directory:crawl-log}/{{ entry }} && ${crawler-bin:wrapper-path} {{ data_source_dict.get(entry) }}
{% endfor %}
{% endif %}
[monitor-directory]
crawl-log = ${:srv}/crawlog
network-user-logs = ${:private}/network-user-logs/
{% for slave_instance in slave_instance_list -%}
user-log-{{ slave_instance.get('slave_reference') }}-folder = ${:private}/network-user-logs/{{ slave_instance.get('slave_reference') }}
user-log-{{ slave_instance.get('slave_reference') }}-ping-folder = ${:private}/network-user-logs/{{ slave_instance.get('slave_reference') }}/ping
user-log-{{ slave_instance.get('slave_reference') }}-ping6-folder = ${:private}/network-user-logs/{{ slave_instance.get('slave_reference') }}/ping6
{% endfor -%}
{% for directory in directory_list %}
{{ '%s = ${:crawl-log}/%s' % (directory, directory) }}
{% endfor %}
[crawler-bin]
recipe = slapos.cookbook:wrapper
command-line =
{{ wget_bin }} --no-check-certificate -l1 -r -nd --timestamp
wrapper-path = ${monitor-directory:bin}/log-crawler
[buildout]
extends = {{ instance_base_monitor }}
parts +=
slave-test-configuration
{% for part in part_list %}
{{ ' %s' % part }}
{% endfor %}
software/monitor/instance-monitor-edgebot.cfg.jinja2 0 → 100644
View file @ 6e65bb9d
{%- if slap_software_type == software_type %}
{%- set CONFIGURATION = {} %}
{%- for k, v in sorted(slap_configuration.items()) %}
{%- if k.startswith('configuration.') %}
{%- do CONFIGURATION.__setitem__(k[14:], v) %}
{%- endif %}
{%- endfor %}
{%- set slave_instance_list = [] %}
{%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %}
{%- if extra_slave_instance_list %}
{#- Create slaves to process with setting up defaults #}
{%- for slave in sorted(json_module.loads(extra_slave_instance_list)) %}
{%- if 'check-status-code' not in slave %}
{%- do slave.__setitem__('check-status-code', CONFIGURATION['check-status-code']) %}
{%- endif %}
{%- if 'check-frontend-ip' not in slave %}
{%- do slave.__setitem__('check-frontend-ip', CONFIGURATION['check-frontend-ip']) %}
{%- endif %}
{%- if 'url' in slave %}
{%- do slave_instance_list.append(slave) %}
{%- endif %}
{%- endfor %}
{%- endif %}
{%- set part_list = [] %}
{%- for slave in sorted(slave_instance_list) %}
{%- set part_id = 'http-query-' ~ slave['slave_reference'] ~ '-promise' %}
{%- do part_list.append(part_id) %}
{%- set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}
[{{part_id}}]
<= monitor-promise-base
module = check_surykatka_json
name = {{ safe_name }}.py
config-report = http_query
config-url = {{ slave['url'] }}
config-status-code = {{ slave['check-status-code'] }}
config-ip-list = {{ slave['check-frontend-ip'] }}
config-json-file = ${surykatka-config:json}
{% endfor %}
[surykatka-bot-promise]
<= monitor-promise-base
module = check_surykatka_json
name = surykatka-bot-promise.py
config-report = bot_status
config-json-file = ${surykatka-config:json}
[buildout]
extends = {{ monitor_template_output }}
parts =
cron
cron-entry-surykatka-status
monitor-base
publish-connection-information
surykatka
surykatka-bot-promise
{% for part_id in sorted(part_list) %}
{{ part_id }}
{% endfor %}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[surykatka-config]
recipe = slapos.recipe.template:jinja2
db = ${directory:srv}/surykatka.db
rendered = ${directory:etc}/surykatka.ini
template = {{ template_surykatka_ini }}
slave_instance_list = {{ dumps(slave_instance_list) }}
nameserver = {{ dumps(CONFIGURATION['nameserver']) }}
json = ${directory:srv}/surykatka.json
context =
import json_module json
key db :db
key nameserver :nameserver
key slave_instance_list :slave_instance_list
[surykatka]
recipe = slapos.cookbook:wrapper
config = ${surykatka-config:rendered}
command-line =
{{ surykatka_binary }} --run crawl --reload --configuration ${:config}
wrapper-path = ${monitor-directory:service}/${:_buildout_section_name_}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[surykatka-status-json]
recipe = slapos.recipe.template:jinja2
template = inline:#!/bin/sh
if {{ surykatka_binary }} --run status --configuration ${surykatka:config} --output json > ${surykatka-config:json}.tmp ; then
mv -f ${surykatka-config:json}.tmp ${surykatka-config:json}
else
rm -f ${surykatka-config:json}.tmp
fi
rendered = ${monitor-directory:bin}/${:_buildout_section_name_}
mode = 0755
[cron-entry-surykatka-status]
recipe = slapos.cookbook:cron.d
cron-entries = ${directory:etc}/cron.d
name = surykatka-status
frequency = */2 * * * *
command = ${surykatka-status-json:rendered}
[publish-connection-information]
recipe = slapos.cookbook:publish.serialised
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
monitor-url = ${monitor-publish-parameters:monitor-url}
monitor-user = ${monitor-publish-parameters:monitor-user}
monitor-password = ${monitor-publish-parameters:monitor-password}
[monitor-instance-parameter]
monitor-httpd-port = {{ slapparameter_dict['monitor-httpd-port'] }}
cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
{% if slapparameter_dict.get('monitor-username', '') -%}
username = {{ slapparameter_dict['monitor-username'] }}
{% endif -%}
{% if slapparameter_dict.get('monitor-password', '') -%}
password = {{ slapparameter_dict['monitor-password'] }}
{% endif -%}
interface-url = {{ slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') }}
[monitor-directory]
service = ${buildout:directory}/etc/service
var = ${buildout:directory}/var
srv = ${buildout:directory}/srv
server-log = ${:private}/server-log
monitor-log = ${:private}/monitor-log
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[slap-parameter]
{%- endif %}
software/monitor/instance-monitor-edgetest.cfg.jinja2 0 → 100644
View file @ 6e65bb9d
{%- if slap_software_type == software_type %}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:filename}
extra-context =
context =
import json_module json
${:extra-context}
[slave-test-configuration]
<=jinja2-template-base
template = {{ template_json_edgetest_test }}
filename = srv/monitor/private/test.json
extensions = jinja2.ext.do
extra-context =
section slave_information slap-configuration
{% set part_list = [] -%}
# Publish information for each slave
{%- set edgebot_software_type = 'edgebot' %}
{%- set edgebot_quantity = slapparameter_dict.pop('edgebot-quantity', '1') | int %}
{%- set edgebot_list = [] %}
{%- set edgebot_section_list = [] %}
{%- set slave_list_name = 'extra_slave_instance_list' %}
{%- set request_dict = {} %}
{%- set namebase = "edgebot" %}
{%- set authorized_slave_list = [] %}
{%- set monitor_base_url_dict = {} -%}
{%- for slave in sorted(slave_instance_list) %}
{%- do authorized_slave_list.append(slave) %}
{%- endfor %}
{%- set monitor_base_port = int(slap_configuration['configuration.monitor-base-port']) %}
{%- for i in range(1, edgebot_quantity + 1) %}
{%- set edgebot_name = "%s-%s" % (namebase, i) %}
{%- set request_section_title = 'request-%s' % edgebot_name %}
{%- do edgebot_list.append(edgebot_name) %}
{%- do edgebot_section_list.append(request_section_title) %}
{%- do part_list.append(request_section_title) %}
{%- do request_dict.__setitem__(request_section_title,
{
'config': {'monitor-httpd-port': monitor_base_port + i},
'name': edgebot_name,
'sla': {},
'state': 'started',
}) %}
{%- endfor %}
[replicate]
<= slap-connection
recipe = slapos.cookbook:request.serialised
config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-password = ${monitor-htpasswd:passwd}
software-url = ${slap-connection:software-release-url}
software-type = {{edgebot_software_type}}
return = monitor-base-url
{% for section, edgebot_request in request_dict.iteritems() %}
[{{section}}]
<= replicate
name = {{ edgebot_request.get('name') }}
{%- if edgebot_request.get('state') %}
state = {{ edgebot_request.get('state') }}
{%- endif%}
{%- set slave_configuration_dict = slapparameter_dict %}
{%- do slave_configuration_dict.update(edgebot_request.get('config')) %}
{%- do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list)) %}
{%- for config_key, config_value in slave_configuration_dict.iteritems() %}
config-{{ config_key }} = {{ dumps(config_value) }}
{% endfor -%}
{%- if edgebot_request.get('sla') %}
{%- for parameter, value in edgebot_request.get('sla').iteritems() %}
sla-{{ parameter }} = {{ value }}
{%- endfor %}
{%- else %}
# As no SLA was provided, by default it is requested on the same computer
sla-computer_guid = ${slap-connection:computer-id}
{% endif %}
{%- do monitor_base_url_dict.__setitem__(section, '${' ~ section ~ ':connection-monitor-base-url}') -%}
{%- endfor %}
{%- set directory_list = [] -%}
{%- for slave_instance in slave_instance_list -%}
{%- set publish_section_title = 'publish-%s' % slave_instance.get('slave_reference') -%}
{%- do part_list.append(publish_section_title) %}
[{{ publish_section_title }}]
recipe = slapos.cookbook:publish
-slave-reference = {{ slave_instance.get('slave_reference') }}
{% endfor %}
[monitor-conf-parameters]
monitor-title = Monitor
password = ${monitor-htpasswd:passwd}
[monitor-base-url-dict]
{% for key, value in monitor_base_url_dict.items() -%}
{{ key }} = {{ value }}
{% endfor %}
[buildout]
extends = {{ instance_base_monitor }}
parts +=
slave-test-configuration
{% for part in part_list %}
{{ ' %s' % part }}
{%- endfor %}
{%- endif %}
software/monitor/instance-monitor.cfg.jinja2
View file @ 6e65bb9d
......@@ -57,14 +57,15 @@ recipe = slapos.cookbook:generate.password
user = admin
bytes = 16
[monitor-instance-parameter]
monitor-httpd-port = {{ slap_configuration['configuration.monitor-base-port'] }}
[monitor-directory]
service = ${buildout:directory}/etc/service
var = ${buildout:directory}/var
srv = ${buildout:directory}/srv
server-log = ${:private}/server-log
monitor-log = ${:private}/monitor-log
cache = ${:var}/cache
mod-ssl = ${:cache}/httpd_mod_ssl
system-log = ${:private}/system-log
consumption = ${:log}/consumption
......
software/monitor/instance.cfg
View file @ 6e65bb9d
......@@ -8,38 +8,65 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
[switch_softwaretype]
recipe = slapos.cookbook:softwaretype
default = $${instance-base-monitor:rendered}
distributor = $${instance-base-distributor:rendered}
edgetest = $${instance-base-edgetest:rendered}
edgebot = $${instance-base-edgebot:rendered}
[instance-base-monitor]
recipe = slapos.recipe.template:jinja2
template = ${template-monitor:destination}
template = ${template-monitor:target}
rendered = $${buildout:directory}/template-base-monitor.cfg
extensions = jinja2.ext.do
context = key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
section slap_configuration slap-configuration
raw buildout_bin ${buildout:bin-directory}
raw monitor_template_output ${monitor-template:output}
raw network_benck_cfg_output ${network-bench-cfg:output}
raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
mode = 0644
[instance-base-distributor]
[instance-base-edgetest]
recipe = slapos.recipe.template:jinja2
template = ${template-monitor-distributor:destination}
rendered = $${buildout:directory}/template-monitor-base-distributor.cfg
template = ${template-monitor-edgetest:target}
rendered = $${buildout:directory}/template-monitor-base-edgetest.cfg
extensions = jinja2.ext.do
context = import json_module json
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
key slap_software_type slap-configuration:slap-software-type
section slap_configuration slap-configuration
raw software_type edgetest
key instance_base_monitor instance-base-monitor:rendered
key slave_instance_list slap-configuration:slave-instance-list
raw buildout_bin ${buildout:bin-directory}
raw template_json_distributor_test ${json-test-template:destination}
raw wget_bin ${wget:location}/bin/wget
raw template_json_edgetest_test ${json-test-template:target}
mode = 0644
[instance-base-edgebot]
recipe = slapos.recipe.template:jinja2
template = ${template-monitor-edgebot:target}
rendered = $${buildout:directory}/template-monitor-edgebot.cfg
extensions = jinja2.ext.do
surykatka-binary = ${surykatka:executable}
template-surykatka-ini = ${template-surykatka-ini:target}
context = import json_module json
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
section slap_configuration slap-configuration
key slapparameter_dict slap-configuration:configuration
key slap_software_type slap-configuration:slap-software-type
raw software_type edgebot
key surykatka_binary :surykatka-binary
key template_surykatka_ini :template-surykatka-ini
raw buildout_bin ${buildout:bin-directory}
raw monitor_template_output ${monitor-template:output}
raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
mode = 0644
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap-connection:computer-id}
......@@ -47,3 +74,12 @@ partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
# Defaults
configuration.check-status-code = 200
configuration.nameserver =
configuration.check-frontend-ip =
# use monitor-base-port to have monitor listening on each instance
# on different port and also on different port than other services
# it makes it possible to instantiate it correctly on signle IP, for
# example in case of webrunner
configuration.monitor-base-port = 9700
software/monitor/software.cfg
View file @ 6e65bb9d
......@@ -4,17 +4,16 @@ extends =
buildout.hash.cfg
../../component/pycurl/buildout.cfg
../../component/python-cryptography/buildout.cfg
../../component/wget/buildout.cfg
../../component/surykatka/buildout.cfg
../../stack/monitor/buildout.cfg
../../stack/slapos.cfg
parts =
wget
slapos-cookbook
network-bench-cfg
json-test-template
template
template-monitor-distributor
template-monitor-edgetest
template-monitor
monitor-collect-csv-dump
......@@ -27,26 +26,31 @@ mode = 0644
[template-monitor]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
destination = ${buildout:directory}/template-base-monitor.cfg
mode = 0644
[template-monitor-distributor]
[template-monitor-edgetest]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
destination = ${buildout:directory}/template-monitor-base-distributor.cfg
mode = 0644
[template-monitor-edgebot]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
[template-surykatka-ini]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
[json-test-template]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
destination = ${buildout:directory}/json-test-template.json.in.jinja2
mode = 0644
[network-bench-cfg]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/network_bench.cfg.in
md5sum = cfcbf2002b8eff5153e2bf68ed24b720
output = ${buildout:directory}/template-network-bench-cfg.in
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:parts-directory}/${:_buildout_section_name_}
mode = 0644
[monitor-collect-csv-dump]
......@@ -54,7 +58,6 @@ mode = 0644
url = ${:_profile_base_location_}/script/${:filename}
filename = collect_csv_dump.py
output = ${:destination}/${:filename}
md5sum = cad2402bbd21907cfed6bc5af8c5d3ab
[extra-eggs]
<= monitor-eggs
......@@ -83,6 +86,5 @@ eggs +=
hexagonit.recipe.download
plone.recipe.command
[versions]
slapos.recipe.template = 4.3
software/monitor/software.cfg.json 0 → 100644
View file @ 6e65bb9d
{
"name": "Monitor",
"description": "Software release for Monitoring purpose",
"serialisation": "xml",
"software-type": {
"default": {
"title": "Default",
"description": "Standalone Monitor",
"request": "instance-default-input-schema.json",
"response": "instance-default-output-schema.json",
"index": 0
},
"edgetest": {
"title": "Edge Test",
"description": "Cluster of bots to perform a distributed monitoring ",
"request": "instance-edgetest-input-schema.json",
"response": "instance-default-output-schema.json",
"index": 1
},
"edgetest-slave": {
"title": "Edge Test Slave",
"software-type": "edgetest",
"description": "Cluster of bots to perform a distributed monitoring ",
"request": "instance-edgetest-slave-input-schema.json",
"response": "instance-default-output-schema.json",
"index": 2
}
}
}
software/monitor/surykatka.ini.jinja2 0 → 100644
View file @ 6e65bb9d
[SURYKATKA]
INTERVAL = 120
SQLITE = {{ db }}
{%- set nameserver_list = nameserver.split() %}
{%- if len(nameserver_list) > 0 %}
NAMESERVER =
{%- for nameserver_entry in sorted(nameserver_list) %}
{{ nameserver_entry }}
{%- endfor %}
{% endif %}
URL =
{%- for slave in sorted(slave_instance_list) %}
{%- if 'url' in slave %}
{{ slave['url'] }}
{%- endif -%}
{% endfor %}
software/monitor/test/test.py
View file @ 6e65bb9d
......@@ -25,10 +25,12 @@
#
##############################################################################
import json
import os
import re
import requests
import xml.etree.ElementTree as ET
from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
......@@ -59,3 +61,346 @@ class ServicesTestCase(SlapOSInstanceTestCase):
expected_process_name = name.format(hash=h)
self.assertIn(expected_process_name, process_names)
class MonitorTestMixin(object):
monitor_setup_url_key = 'monitor-setup-url'
def test_monitor_setup(self):
connection_parameter_dict = self\
.computer_partition.getConnectionParameterDict()
self.assertTrue(
self.monitor_setup_url_key in connection_parameter_dict,
'%s not in %s' % (self.monitor_setup_url_key, connection_parameter_dict))
monitor_setup_url_value = connection_parameter_dict[
self.monitor_setup_url_key]
monitor_url_match = re.match(r'.*url=(.*)', monitor_setup_url_value)
self.assertNotEqual(
None, monitor_url_match, '%s not parsable' % (monitor_setup_url_value,))
self.assertEqual(1, len(monitor_url_match.groups()))
monitor_url = monitor_url_match.groups()[0]
monitor_url_split = monitor_url.split('&')
self.assertEqual(
3, len(monitor_url_split), '%s not splitabble' % (monitor_url,))
self.monitor_url = monitor_url_split[0]
monitor_username = monitor_url_split[1].split('=')
self.assertEqual(
2, len(monitor_username), '%s not splittable' % (monitor_username))
monitor_password = monitor_url_split[2].split('=')
self.assertEqual(
2, len(monitor_password), '%s not splittable' % (monitor_password))
self.monitor_username = monitor_username[1]
self.monitor_password = monitor_password[1]
opml_text = requests.get(self.monitor_url, verify=False).text
opml = ET.fromstring(opml_text)
body = opml[1]
self.assertEqual('body', body.tag)
outline_list = body[0].findall('outline')
self.assertEqual(
self.monitor_configuration_list,
[q.attrib for q in outline_list]
)
expected_status_code_list = []
got_status_code_list = []
for monitor_configuration in self.monitor_configuration_list:
status_code = requests.get(
monitor_configuration['url'],
verify=False,
auth=(self.monitor_username, self.monitor_password)
).status_code
expected_status_code_list.append(
{
'url': monitor_configuration['url'],
'status_code': 200
}
)
got_status_code_list.append(
{
'url': monitor_configuration['url'],
'status_code': status_code
}
)
self.assertEqual(
expected_status_code_list,
got_status_code_list
)
class EdgeSlaveMixin(MonitorTestMixin):
__partition_reference__ = 'edge'
instance_max_retry = 20
@classmethod
def getInstanceSoftwareType(cls):
return 'edgetest'
def requestEdgetestSlave(self, partition_reference, partition_parameter_kw):
software_url = self.getSoftwareURL()
self.slap.request(
software_release=software_url,
software_type='edgetest',
partition_reference=partition_reference,
partition_parameter_kw=partition_parameter_kw,
shared=True
)
def setUp(self):
self.bot_partition_path = os.path.join(
self.slap.instance_directory,
self.__partition_reference__ + '1')
self.surykatka_json = os.path.join(
self.bot_partition_path, 'srv', 'surykatka.json')
self.surykatka_status_json = os.path.join(
self.bot_partition_path, 'bin', 'surykatka-status-json')
self.monitor_configuration_list = [
{
'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
'version': 'RSS',
'title': 'testing partition 0',
'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
'text': 'testing partition 0',
'type': 'rss',
'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
},
{
'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
'version': 'RSS',
'title': 'edgebot-1',
'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
'text': 'edgebot-1',
'type': 'rss',
'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
}
]
def assertSurykatkaIni(self):
surykatka_ini = open(
os.path.join(
self.bot_partition_path, 'etc', 'surykatka.ini')).read().strip()
expected = self.surykatka_ini % dict(
partition_path=self.bot_partition_path)
self.assertEqual(
expected.strip(),
surykatka_ini)
def assertPromiseContent(self, name, content):
promise = open(
os.path.join(
self.bot_partition_path, 'etc', 'plugin', name
)).read().strip()
self.assertTrue(content in promise)
def assertSurykatkaBotPromise(self):
self.assertPromiseContent(
'surykatka-bot-promise.py',
"'report': 'bot_status'")
self.assertPromiseContent(
'surykatka-bot-promise.py',
"'json-file': '%s'" % (self.surykatka_json,)
)
def assertSurykatkaCron(self):
surykatka_cron = open(
os.path.join(
self.bot_partition_path, 'etc', 'cron.d', 'surykatka-status')
).read().strip()
self.assertEqual(
'*/2 * * * * %s' % (self.surykatka_status_json,),
surykatka_cron
)
def initiateSurykatkaRun(self):
try:
self.slap.waitForInstance(max_retry=2)
except Exception:
pass
def assertSurykatkaStatusJSON(self):
if os.path.exists(self.surykatka_json):
os.unlink(self.surykatka_json)
self.assertEqual(0, os.system(self.surykatka_status_json))
self.assertTrue(os.path.exists(self.surykatka_json))
with open(self.surykatka_json) as fh:
status_json = json.load(fh)
self.assertTrue('bot_status' in status_json)
def test(self):
# Note: Those tests do not run surykatka and do not do real checks, as
# this depends too much on the environment and is really hard to
# mock
# So it is possible that some bugs might slip under the radar
# Nevertheless the surykatka and check_surykatka_json are heavily
# unit tested, and configuration created by the profiles is asserted
# here, so it shall be enough as reasonable status
self.requestEdgetestSlaves()
self.initiateSurykatkaRun()
self.assertSurykatkaStatusJSON()
self.assertSurykatkaIni()
self.assertSurykatkaBotPromise()
self.assertSurykatkaPromises()
self.assertSurykatkaCron()
class TestEdge(EdgeSlaveMixin, SlapOSInstanceTestCase):
surykatka_ini = """[SURYKATKA]
INTERVAL = 120
SQLITE = %(partition_path)s/srv/surykatka.db
URL =
https://www.erp5.com/
https://www.erp5.org/"""
def assertSurykatkaPromises(self):
self.assertPromiseContent(
'http-query-backend-300-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-300-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-300-promise.py',
"'status-code': '300'")
self.assertPromiseContent(
'http-query-backend-300-promise.py',
"'url': 'https://www.erp5.org/'")
self.assertPromiseContent(
'http-query-backend-300-promise.py',
"'json-file': '%s'" % (self.surykatka_json,)
)
self.assertPromiseContent(
'http-query-backend-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'url': 'https://www.erp5.com/'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'json-file': '%s'" % (self.surykatka_json,)
)
def requestEdgetestSlaves(self):
self.requestEdgetestSlave(
'backend',
{'url': 'https://www.erp5.com/'},
)
self.requestEdgetestSlave(
'backend-300',
{'url': 'https://www.erp5.org/', 'check-status-code': '300'},
)
class TestEdgeNameserverCheckFrontendIp(
EdgeSlaveMixin, SlapOSInstanceTestCase):
surykatka_ini = """[SURYKATKA]
INTERVAL = 120
SQLITE = %(partition_path)s/srv/surykatka.db
NAMESERVER =
127.0.1.1
127.0.1.2
URL =
https://www.erp5.com/"""
@classmethod
def getInstanceParameterDict(cls):
return {
'nameserver': '127.0.1.1 127.0.1.2',
'check-frontend-ip': '127.0.0.1 127.0.0.2',
}
def assertSurykatkaPromises(self):
self.assertPromiseContent(
'http-query-backend-promise.py',
"'ip-list': '127.0.0.1 127.0.0.2'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'url': 'https://www.erp5.com/'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'json-file': '%s'" % (self.surykatka_json,)
)
def requestEdgetestSlaves(self):
self.requestEdgetestSlave(
'backend',
{'url': 'https://www.erp5.com/'},
)
class TestEdgeCheckStatusCode(EdgeSlaveMixin, SlapOSInstanceTestCase):
surykatka_ini = """[SURYKATKA]
INTERVAL = 120
SQLITE = %(partition_path)s/srv/surykatka.db
URL =
https://www.erp5.com/
https://www.erp5.org/"""
@classmethod
def getInstanceParameterDict(cls):
return {
'check-status-code': '500',
}
def assertSurykatkaPromises(self):
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'status-code': '501'")
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'url': 'https://www.erp5.org/'")
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'json-file': '%s'" % (self.surykatka_json,)
)
self.assertPromiseContent(
'http-query-backend-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'status-code': '500'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'url': 'https://www.erp5.com/'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'json-file': '%s'" % (self.surykatka_json,)
)
def requestEdgetestSlaves(self):
self.requestEdgetestSlave(
'backend',
{'url': 'https://www.erp5.com/'},
)
self.requestEdgetestSlave(
'backend-501',
{'url': 'https://www.erp5.org/', 'check-status-code': '501'},
)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7