Commit 6e65bb9d authored by Łukasz Nowak's avatar Łukasz Nowak

monitor: Edge testing of urls

Adds edgetest and edgebot software types, where edgetest is a software type
to request by user and requests by itself edgebot, which does the monitoring
work.

Based on Rafael Monnerat <rafael@nexedi.com>
parent a796981d
Pipeline #7164 canceled with stage
in 0 seconds
============
Edge testing
============
``edgetest`` is a special software type of monitor software release used for website monitoring by using bots.
It uses `surykatka <https://lab.nexedi.com/nexedi/surykatka>`_ and `check_surykatka_json <https://lab.nexedi.com/nexedi/slapos.toolbox/blob/master/slapos/promise/plugin/check_surykatka_json.py>`_ to monitor websites.
``surykatka`` provides a bot to query list of hosts and a JSON reporting system.
``check_surykatka_json`` is used in promises to provide monitoring information about the websites.
In order to monitor an url one need to:
* request a monitor software release with ``edgetest`` software type, configured as described in ``instance-edgetest-input-schema.json``,
* request a slave to monitor with ``edgetest`` software type, configured as described in ``instance-edgetest-slave-input-schema.json``.
......@@ -14,16 +14,32 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = 1b7d2d097f208f6641bf98a17df079c8
md5sum = cb814297f2f76dc8e08014a7d662439f
[template-monitor]
_update_hash_filename_ = instance-monitor.cfg.jinja2
md5sum = 373c79480e6425c20480fc911a56c3fd
[template-monitor-distributor]
_update_hash_filename_ = instance-monitor-distributor.cfg.jinja2
md5sum = 61c0bfdfc0a2b51ba15fe4a49baf6091
md5sum = 165a15672fc85981f68b9af2d6253254
[json-test-template]
_update_hash_filename_ = json-test-template.json.in.jinja2
md5sum = 2eb5596544d9c341acf653d4f7ce2680
[template-monitor-edgetest]
_update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2
md5sum = 9e237dbdda59e788202f0da194a57d41
[template-monitor-edgebot]
_update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
md5sum = cd4c5d2ecf8285ea8f0905ed1150d8a0
[network-bench-cfg]
filename = network_bench.cfg.in
md5sum = cfcbf2002b8eff5153e2bf68ed24b720
[monitor-collect-csv-dump]
filename = script/collect_csv_dump.py
md5sum = cad2402bbd21907cfed6bc5af8c5d3ab
[template-surykatka-ini]
_update_hash_filename_ = surykatka.ini.jinja2
md5sum = 40870921e05d93b5843ab34abd7e3902
{
"$schema": "http://json-schema.org/draft-04/schema#",
"properties": {
"dummy": {
"title": "dummy",
"description": "Dummy",
"type": "string"
}
}
}
{
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Values returned by Re6st Master instanciation",
Please register or sign in to reply
"properties": {
"re6stry-url": {
"description": "ipv6 url to access your re6st registry service",
"type": "string"
}
},
"type": "object"
}
{
"$schema": "http://json-schema.org/draft-04/schema#",
"properties": {
"nameserver": {
"default": "",
"title": "Nameserver",
"description": "Space separated list of name servers to use.",
"type": "string"
},
"check-status-code": {
"default": "200",
"title": "Default Check HTTP Code",
"description": "Default HTTP code to check against (default: 200).",
"type": "string"
},
"check-frontend-ip": {
"default": "",
"title": "Default space separated list of Frontend IPs to check",
"description": "Default list of Frontend IPs to check, if empty no constraint is used.",
"type": "string"
}
}
}
{
"$schema": "http://json-schema.org/draft-04/schema#",
"properties": {
"url": {
"title": "URL to check",
"description": "URL to check, like https://example.com",
"type": "string"
},
"check-status-code": {
"default": "Master default",
"title": "Default Check HTTP Code.",
"description": "HTTP code to check against (default: comes from master partition).",
"type": "string"
},
"check-frontend-ip": {
"default": "Master default",
"title": "Space separated list of Frontend IPs to check",
"description": "List of Frontend IPs to check, if empty no constraint is used (default: comes from master partition).",
"type": "string"
}
}
}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:filename}
extra-context =
context =
import json_module json
${:extra-context}
[slave-test-configuration]
<=jinja2-template-base
template = {{ template_json_distributor_test }}
filename = srv/monitor/private/test.json
extensions = jinja2.ext.do
extra-context =
section slave_information slap-configuration
{% set part_list = [] -%}
# Publish information for each slave
{% set directory_list = [] -%}
{% for slave_instance in slave_instance_list -%}
{% set publish_section_title = 'publish-%s' % slave_instance.get('slave_reference') -%}
{% do part_list.append(publish_section_title) -%}
[{{ publish_section_title }}]
recipe = slapos.cookbook:publish
-slave-reference = {{ slave_instance.get('slave_reference') }}
log-access-url = ${monitor-frontend:connection-site_url}/{{ slave_instance.get('slave_reference') }}
log-access-url-v6 = ${monitor-httpd-conf-parameter:url}/{{ slave_instance.get('slave_reference') }}
{% endfor %}
{% set data_source_dict = slapparameter_dict.get('data-source', None) -%}
{% set cron_min_count = 0 -%}
{% if data_source_dict %}
{% for entry in data_source_dict -%}
{% set cron_min_count = cron_min_count + 1 -%}
{% set cron_min = cron_min_count%60 -%}
{% do part_list.append('cron-crawl-' + entry) -%}
{% do directory_list.append(entry) -%}
[cron-crawl-{{ entry }}]
<= cron
recipe = slapos.cookbook:cron.d
name = cron-crawler-{{ entry }}
frequency = * * * * *
command = cd ${monitor-directory:crawl-log}/{{ entry }} && ${crawler-bin:wrapper-path} {{ data_source_dict.get(entry) }}
{% endfor %}
{% endif %}
[monitor-directory]
crawl-log = ${:srv}/crawlog
network-user-logs = ${:private}/network-user-logs/
{% for slave_instance in slave_instance_list -%}
user-log-{{ slave_instance.get('slave_reference') }}-folder = ${:private}/network-user-logs/{{ slave_instance.get('slave_reference') }}
user-log-{{ slave_instance.get('slave_reference') }}-ping-folder = ${:private}/network-user-logs/{{ slave_instance.get('slave_reference') }}/ping
user-log-{{ slave_instance.get('slave_reference') }}-ping6-folder = ${:private}/network-user-logs/{{ slave_instance.get('slave_reference') }}/ping6
{% endfor -%}
{% for directory in directory_list %}
{{ '%s = ${:crawl-log}/%s' % (directory, directory) }}
{% endfor %}
[crawler-bin]
recipe = slapos.cookbook:wrapper
command-line =
{{ wget_bin }} --no-check-certificate -l1 -r -nd --timestamp
wrapper-path = ${monitor-directory:bin}/log-crawler
[buildout]
extends = {{ instance_base_monitor }}
parts +=
slave-test-configuration
{% for part in part_list %}
{{ ' %s' % part }}
{% endfor %}
{%- if slap_software_type == software_type %}
{%- set CONFIGURATION = {} %}
{%- for k, v in sorted(slap_configuration.items()) %}
{%- if k.startswith('configuration.') %}
{%- do CONFIGURATION.__setitem__(k[14:], v) %}
{%- endif %}
{%- endfor %}
{%- set slave_instance_list = [] %}
{%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %}
{%- if extra_slave_instance_list %}
{#- Create slaves to process with setting up defaults #}
{%- for slave in sorted(json_module.loads(extra_slave_instance_list)) %}
{%- if 'check-status-code' not in slave %}
{%- do slave.__setitem__('check-status-code', CONFIGURATION['check-status-code']) %}
{%- endif %}
{%- if 'check-frontend-ip' not in slave %}
{%- do slave.__setitem__('check-frontend-ip', CONFIGURATION['check-frontend-ip']) %}
{%- endif %}
{%- if 'url' in slave %}
{%- do slave_instance_list.append(slave) %}
{%- endif %}
{%- endfor %}
{%- endif %}
{%- set part_list = [] %}
{%- for slave in sorted(slave_instance_list) %}
{%- set part_id = 'http-query-' ~ slave['slave_reference'] ~ '-promise' %}
{%- do part_list.append(part_id) %}
{%- set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}
[{{part_id}}]
<= monitor-promise-base
module = check_surykatka_json
name = {{ safe_name }}.py
config-report = http_query
config-url = {{ slave['url'] }}
config-status-code = {{ slave['check-status-code'] }}
config-ip-list = {{ slave['check-frontend-ip'] }}
config-json-file = ${surykatka-config:json}
{% endfor %}
[surykatka-bot-promise]
<= monitor-promise-base
module = check_surykatka_json
name = surykatka-bot-promise.py
config-report = bot_status
config-json-file = ${surykatka-config:json}
[buildout]
extends = {{ monitor_template_output }}
parts =
cron
cron-entry-surykatka-status
monitor-base
publish-connection-information
surykatka
surykatka-bot-promise
{% for part_id in sorted(part_list) %}
{{ part_id }}
{% endfor %}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[surykatka-config]
recipe = slapos.recipe.template:jinja2
db = ${directory:srv}/surykatka.db
rendered = ${directory:etc}/surykatka.ini
template = {{ template_surykatka_ini }}
slave_instance_list = {{ dumps(slave_instance_list) }}
nameserver = {{ dumps(CONFIGURATION['nameserver']) }}
json = ${directory:srv}/surykatka.json
context =
import json_module json
key db :db
key nameserver :nameserver
key slave_instance_list :slave_instance_list
[surykatka]
recipe = slapos.cookbook:wrapper
config = ${surykatka-config:rendered}
command-line =
{{ surykatka_binary }} --run crawl --reload --configuration ${:config}
wrapper-path = ${monitor-directory:service}/${:_buildout_section_name_}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[surykatka-status-json]
recipe = slapos.recipe.template:jinja2
template = inline:#!/bin/sh
if {{ surykatka_binary }} --run status --configuration ${surykatka:config} --output json > ${surykatka-config:json}.tmp ; then
mv -f ${surykatka-config:json}.tmp ${surykatka-config:json}
else
rm -f ${surykatka-config:json}.tmp
fi
rendered = ${monitor-directory:bin}/${:_buildout_section_name_}
mode = 0755
[cron-entry-surykatka-status]
recipe = slapos.cookbook:cron.d
cron-entries = ${directory:etc}/cron.d
name = surykatka-status
frequency = */2 * * * *
command = ${surykatka-status-json:rendered}
[publish-connection-information]
recipe = slapos.cookbook:publish.serialised
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
monitor-url = ${monitor-publish-parameters:monitor-url}
monitor-user = ${monitor-publish-parameters:monitor-user}
monitor-password = ${monitor-publish-parameters:monitor-password}
[monitor-instance-parameter]
monitor-httpd-port = {{ slapparameter_dict['monitor-httpd-port'] }}
cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
{% if slapparameter_dict.get('monitor-username', '') -%}
username = {{ slapparameter_dict['monitor-username'] }}
{% endif -%}
{% if slapparameter_dict.get('monitor-password', '') -%}
password = {{ slapparameter_dict['monitor-password'] }}
{% endif -%}
interface-url = {{ slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') }}
[monitor-directory]
service = ${buildout:directory}/etc/service
var = ${buildout:directory}/var
srv = ${buildout:directory}/srv
server-log = ${:private}/server-log
monitor-log = ${:private}/monitor-log
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[slap-parameter]
{%- endif %}
{%- if slap_software_type == software_type %}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:filename}
extra-context =
context =
import json_module json
${:extra-context}
[slave-test-configuration]
<=jinja2-template-base
template = {{ template_json_edgetest_test }}
filename = srv/monitor/private/test.json
extensions = jinja2.ext.do
extra-context =
section slave_information slap-configuration
{% set part_list = [] -%}
# Publish information for each slave
{%- set edgebot_software_type = 'edgebot' %}
{%- set edgebot_quantity = slapparameter_dict.pop('edgebot-quantity', '1') | int %}
{%- set edgebot_list = [] %}
{%- set edgebot_section_list = [] %}
{%- set slave_list_name = 'extra_slave_instance_list' %}
{%- set request_dict = {} %}
{%- set namebase = "edgebot" %}
{%- set authorized_slave_list = [] %}
{%- set monitor_base_url_dict = {} -%}
{%- for slave in sorted(slave_instance_list) %}
  • What is the reason to sort this list? (this is a list of dicts)

  • To minimise amount of updates on the server which happens below in

    {%-   do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list)) %}

    AFAIK then the request processing on server side will not update data, as long as the list is stable.

Please register or sign in to reply
{%- do authorized_slave_list.append(slave) %}
{%- endfor %}
{%- set monitor_base_port = int(slap_configuration['configuration.monitor-base-port']) %}
{%- for i in range(1, edgebot_quantity + 1) %}
{%- set edgebot_name = "%s-%s" % (namebase, i) %}
{%- set request_section_title = 'request-%s' % edgebot_name %}
{%- do edgebot_list.append(edgebot_name) %}
{%- do edgebot_section_list.append(request_section_title) %}
{%- do part_list.append(request_section_title) %}
{%- do request_dict.__setitem__(request_section_title,
{
'config': {'monitor-httpd-port': monitor_base_port + i},
'name': edgebot_name,
'sla': {},
'state': 'started',
}) %}
{%- endfor %}
[replicate]
<= slap-connection
recipe = slapos.cookbook:request.serialised
config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-password = ${monitor-htpasswd:passwd}
software-url = ${slap-connection:software-release-url}
software-type = {{edgebot_software_type}}
return = monitor-base-url
{% for section, edgebot_request in request_dict.iteritems() %}
[{{section}}]
<= replicate
name = {{ edgebot_request.get('name') }}
{%- if edgebot_request.get('state') %}
state = {{ edgebot_request.get('state') }}
{%- endif%}
{%- set slave_configuration_dict = slapparameter_dict %}
{%- do slave_configuration_dict.update(edgebot_request.get('config')) %}
{%- do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list)) %}
{%- for config_key, config_value in slave_configuration_dict.iteritems() %}
config-{{ config_key }} = {{ dumps(config_value) }}
{% endfor -%}
{%- if edgebot_request.get('sla') %}
{%- for parameter, value in edgebot_request.get('sla').iteritems() %}
sla-{{ parameter }} = {{ value }}
{%- endfor %}
{%- else %}
# As no SLA was provided, by default it is requested on the same computer
sla-computer_guid = ${slap-connection:computer-id}
{% endif %}
{%- do monitor_base_url_dict.__setitem__(section, '${' ~ section ~ ':connection-monitor-base-url}') -%}
{%- endfor %}
{%- set directory_list = [] -%}
{%- for slave_instance in slave_instance_list -%}
{%- set publish_section_title = 'publish-%s' % slave_instance.get('slave_reference') -%}
{%- do part_list.append(publish_section_title) %}
[{{ publish_section_title }}]
recipe = slapos.cookbook:publish
-slave-reference = {{ slave_instance.get('slave_reference') }}
{% endfor %}
[monitor-conf-parameters]
monitor-title = Monitor
password = ${monitor-htpasswd:passwd}
[monitor-base-url-dict]
{% for key, value in monitor_base_url_dict.items() -%}
{{ key }} = {{ value }}
{% endfor %}
[buildout]
extends = {{ instance_base_monitor }}
parts +=
slave-test-configuration
{% for part in part_list %}
{{ ' %s' % part }}
{%- endfor %}
{%- endif %}
......@@ -57,14 +57,15 @@ recipe = slapos.cookbook:generate.password
user = admin
bytes = 16
[monitor-instance-parameter]
monitor-httpd-port = {{ slap_configuration['configuration.monitor-base-port'] }}
[monitor-directory]
service = ${buildout:directory}/etc/service
var = ${buildout:directory}/var
srv = ${buildout:directory}/srv
server-log = ${:private}/server-log
monitor-log = ${:private}/monitor-log
cache = ${:var}/cache
mod-ssl = ${:cache}/httpd_mod_ssl
system-log = ${:private}/system-log
consumption = ${:log}/consumption
......
......@@ -8,38 +8,65 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
[switch_softwaretype]
recipe = slapos.cookbook:softwaretype
default = $${instance-base-monitor:rendered}
distributor = $${instance-base-distributor:rendered}
edgetest = $${instance-base-edgetest:rendered}
edgebot = $${instance-base-edgebot:rendered}
[instance-base-monitor]
recipe = slapos.recipe.template:jinja2
template = ${template-monitor:destination}
template = ${template-monitor:target}
rendered = $${buildout:directory}/template-base-monitor.cfg
extensions = jinja2.ext.do
context = key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
section slap_configuration slap-configuration
raw buildout_bin ${buildout:bin-directory}
raw monitor_template_output ${monitor-template:output}
raw network_benck_cfg_output ${network-bench-cfg:output}
raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
mode = 0644
[instance-base-distributor]
[instance-base-edgetest]
recipe = slapos.recipe.template:jinja2
template = ${template-monitor-distributor:destination}
rendered = $${buildout:directory}/template-monitor-base-distributor.cfg
template = ${template-monitor-edgetest:target}
rendered = $${buildout:directory}/template-monitor-base-edgetest.cfg
extensions = jinja2.ext.do
context = import json_module json
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
key slap_software_type slap-configuration:slap-software-type
section slap_configuration slap-configuration
raw software_type edgetest
key instance_base_monitor instance-base-monitor:rendered
key slave_instance_list slap-configuration:slave-instance-list
raw buildout_bin ${buildout:bin-directory}
raw template_json_distributor_test ${json-test-template:destination}
raw wget_bin ${wget:location}/bin/wget
raw template_json_edgetest_test ${json-test-template:target}
mode = 0644
[instance-base-edgebot]
recipe = slapos.recipe.template:jinja2
template = ${template-monitor-edgebot:target}
rendered = $${buildout:directory}/template-monitor-edgebot.cfg
extensions = jinja2.ext.do
surykatka-binary = ${surykatka:executable}
template-surykatka-ini = ${template-surykatka-ini:target}
context = import json_module json
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
section slap_configuration slap-configuration
key slapparameter_dict slap-configuration:configuration
key slap_software_type slap-configuration:slap-software-type
raw software_type edgebot
key surykatka_binary :surykatka-binary
key template_surykatka_ini :template-surykatka-ini
raw buildout_bin ${buildout:bin-directory}
raw monitor_template_output ${monitor-template:output}
raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
mode = 0644
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap-connection:computer-id}
......@@ -47,3 +74,12 @@ partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
# Defaults
configuration.check-status-code = 200
configuration.nameserver =
configuration.check-frontend-ip =
# use monitor-base-port to have monitor listening on each instance
# on different port and also on different port than other services
# it makes it possible to instantiate it correctly on signle IP, for
# example in case of webrunner
configuration.monitor-base-port = 9700
......@@ -4,17 +4,16 @@ extends =
buildout.hash.cfg
../../component/pycurl/buildout.cfg
../../component/python-cryptography/buildout.cfg
../../component/wget/buildout.cfg
../../component/surykatka/buildout.cfg
../../stack/monitor/buildout.cfg
../../stack/slapos.cfg
parts =
wget
slapos-cookbook
network-bench-cfg
json-test-template
template
template-monitor-distributor
template-monitor-edgetest
template-monitor
monitor-collect-csv-dump
......@@ -27,26 +26,31 @@ mode = 0644
[template-monitor]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
destination = ${buildout:directory}/template-base-monitor.cfg
mode = 0644
[template-monitor-distributor]
[template-monitor-edgetest]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
destination = ${buildout:directory}/template-monitor-base-distributor.cfg
mode = 0644
[template-monitor-edgebot]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
[template-surykatka-ini]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
[json-test-template]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
destination = ${buildout:directory}/json-test-template.json.in.jinja2
mode = 0644
[network-bench-cfg]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/network_bench.cfg.in
md5sum = cfcbf2002b8eff5153e2bf68ed24b720
output = ${buildout:directory}/template-network-bench-cfg.in
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:parts-directory}/${:_buildout_section_name_}
mode = 0644
[monitor-collect-csv-dump]
......@@ -54,7 +58,6 @@ mode = 0644
url = ${:_profile_base_location_}/script/${:filename}
filename = collect_csv_dump.py
output = ${:destination}/${:filename}
md5sum = cad2402bbd21907cfed6bc5af8c5d3ab
[extra-eggs]
<= monitor-eggs
......@@ -83,6 +86,5 @@ eggs +=
hexagonit.recipe.download
plone.recipe.command
[versions]
slapos.recipe.template = 4.3
{
"name": "Monitor",
"description": "Software release for Monitoring purpose",
"serialisation": "xml",
"software-type": {
"default": {
"title": "Default",
"description": "Standalone Monitor",
"request": "instance-default-input-schema.json",
"response": "instance-default-output-schema.json",
"index": 0
},
"edgetest": {
"title": "Edge Test",
"description": "Cluster of bots to perform a distributed monitoring ",
"request": "instance-edgetest-input-schema.json",
"response": "instance-default-output-schema.json",
"index": 1
},
"edgetest-slave": {
"title": "Edge Test Slave",
"software-type": "edgetest",
"description": "Cluster of bots to perform a distributed monitoring ",
"request": "instance-edgetest-slave-input-schema.json",
"response": "instance-default-output-schema.json",
"index": 2
}
}
}
[SURYKATKA]
INTERVAL = 120
SQLITE = {{ db }}
{%- set nameserver_list = nameserver.split() %}
{%- if len(nameserver_list) > 0 %}
NAMESERVER =
{%- for nameserver_entry in sorted(nameserver_list) %}
{{ nameserver_entry }}
{%- endfor %}
{% endif %}
URL =
{%- for slave in sorted(slave_instance_list) %}
{%- if 'url' in slave %}
{{ slave['url'] }}
{%- endif -%}
{% endfor %}
......@@ -25,10 +25,12 @@
#
##############################################################################
import json
import os
import re
import requests
import xml.etree.ElementTree as ET
from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
......@@ -59,3 +61,346 @@ class ServicesTestCase(SlapOSInstanceTestCase):
expected_process_name = name.format(hash=h)
self.assertIn(expected_process_name, process_names)
class MonitorTestMixin(object):
monitor_setup_url_key = 'monitor-setup-url'
def test_monitor_setup(self):
connection_parameter_dict = self\
.computer_partition.getConnectionParameterDict()
self.assertTrue(
self.monitor_setup_url_key in connection_parameter_dict,
'%s not in %s' % (self.monitor_setup_url_key, connection_parameter_dict))
monitor_setup_url_value = connection_parameter_dict[
self.monitor_setup_url_key]
monitor_url_match = re.match(r'.*url=(.*)', monitor_setup_url_value)
self.assertNotEqual(
None, monitor_url_match, '%s not parsable' % (monitor_setup_url_value,))
self.assertEqual(1, len(monitor_url_match.groups()))
monitor_url = monitor_url_match.groups()[0]
monitor_url_split = monitor_url.split('&')
self.assertEqual(
3, len(monitor_url_split), '%s not splitabble' % (monitor_url,))
self.monitor_url = monitor_url_split[0]
monitor_username = monitor_url_split[1].split('=')
self.assertEqual(
2, len(monitor_username), '%s not splittable' % (monitor_username))
monitor_password = monitor_url_split[2].split('=')
self.assertEqual(
2, len(monitor_password), '%s not splittable' % (monitor_password))
self.monitor_username = monitor_username[1]
self.monitor_password = monitor_password[1]
opml_text = requests.get(self.monitor_url, verify=False).text
opml = ET.fromstring(opml_text)
body = opml[1]
self.assertEqual('body', body.tag)
outline_list = body[0].findall('outline')
self.assertEqual(
self.monitor_configuration_list,
[q.attrib for q in outline_list]
)
expected_status_code_list = []
got_status_code_list = []
for monitor_configuration in self.monitor_configuration_list:
status_code = requests.get(
monitor_configuration['url'],
verify=False,
auth=(self.monitor_username, self.monitor_password)
).status_code
expected_status_code_list.append(
{
'url': monitor_configuration['url'],
'status_code': 200
}
)
got_status_code_list.append(
{
'url': monitor_configuration['url'],
'status_code': status_code
}
)
self.assertEqual(
expected_status_code_list,
got_status_code_list
)
class EdgeSlaveMixin(MonitorTestMixin):
__partition_reference__ = 'edge'
instance_max_retry = 20
@classmethod
def getInstanceSoftwareType(cls):
return 'edgetest'
def requestEdgetestSlave(self, partition_reference, partition_parameter_kw):
software_url = self.getSoftwareURL()
self.slap.request(
software_release=software_url,
software_type='edgetest',
partition_reference=partition_reference,
partition_parameter_kw=partition_parameter_kw,
shared=True
)
def setUp(self):
self.bot_partition_path = os.path.join(
self.slap.instance_directory,
self.__partition_reference__ + '1')
self.surykatka_json = os.path.join(
self.bot_partition_path, 'srv', 'surykatka.json')
self.surykatka_status_json = os.path.join(
self.bot_partition_path, 'bin', 'surykatka-status-json')
self.monitor_configuration_list = [
{
'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
'version': 'RSS',
'title': 'testing partition 0',
'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
'text': 'testing partition 0',
'type': 'rss',
'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
},
{
'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
'version': 'RSS',
'title': 'edgebot-1',
'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
'text': 'edgebot-1',
'type': 'rss',
'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
}
]
def assertSurykatkaIni(self):
surykatka_ini = open(
os.path.join(
self.bot_partition_path, 'etc', 'surykatka.ini')).read().strip()
expected = self.surykatka_ini % dict(
partition_path=self.bot_partition_path)
self.assertEqual(
expected.strip(),
surykatka_ini)
def assertPromiseContent(self, name, content):
promise = open(
os.path.join(
self.bot_partition_path, 'etc', 'plugin', name
)).read().strip()
self.assertTrue(content in promise)
def assertSurykatkaBotPromise(self):
self.assertPromiseContent(
'surykatka-bot-promise.py',
"'report': 'bot_status'")
self.assertPromiseContent(
'surykatka-bot-promise.py',
"'json-file': '%s'" % (self.surykatka_json,)
)
def assertSurykatkaCron(self):
surykatka_cron = open(
os.path.join(
self.bot_partition_path, 'etc', 'cron.d', 'surykatka-status')
).read().strip()
self.assertEqual(
'*/2 * * * * %s' % (self.surykatka_status_json,),
surykatka_cron
)
def initiateSurykatkaRun(self):
try:
self.slap.waitForInstance(max_retry=2)
except Exception:
pass
def assertSurykatkaStatusJSON(self):
if os.path.exists(self.surykatka_json):
os.unlink(self.surykatka_json)
self.assertEqual(0, os.system(self.surykatka_status_json))
self.assertTrue(os.path.exists(self.surykatka_json))
with open(self.surykatka_json) as fh:
status_json = json.load(fh)
self.assertTrue('bot_status' in status_json)
def test(self):
# Note: Those tests do not run surykatka and do not do real checks, as
# this depends too much on the environment and is really hard to
# mock
# So it is possible that some bugs might slip under the radar
# Nevertheless the surykatka and check_surykatka_json are heavily
# unit tested, and configuration created by the profiles is asserted
# here, so it shall be enough as reasonable status
self.requestEdgetestSlaves()
self.initiateSurykatkaRun()
self.assertSurykatkaStatusJSON()
self.assertSurykatkaIni()
self.assertSurykatkaBotPromise()
self.assertSurykatkaPromises()
self.assertSurykatkaCron()
class TestEdge(EdgeSlaveMixin, SlapOSInstanceTestCase):
surykatka_ini = """[SURYKATKA]
INTERVAL = 120
SQLITE = %(partition_path)s/srv/surykatka.db
URL =
https://www.erp5.com/
https://www.erp5.org/"""
def assertSurykatkaPromises(self):
self.assertPromiseContent(
'http-query-backend-300-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-300-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-300-promise.py',
"'status-code': '300'")
self.assertPromiseContent(
'http-query-backend-300-promise.py',
"'url': 'https://www.erp5.org/'")
self.assertPromiseContent(
'http-query-backend-300-promise.py',
"'json-file': '%s'" % (self.surykatka_json,)
)
self.assertPromiseContent(
'http-query-backend-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'url': 'https://www.erp5.com/'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'json-file': '%s'" % (self.surykatka_json,)
)
def requestEdgetestSlaves(self):
self.requestEdgetestSlave(
'backend',
{'url': 'https://www.erp5.com/'},
)
self.requestEdgetestSlave(
'backend-300',
{'url': 'https://www.erp5.org/', 'check-status-code': '300'},
)
class TestEdgeNameserverCheckFrontendIp(
EdgeSlaveMixin, SlapOSInstanceTestCase):
surykatka_ini = """[SURYKATKA]
INTERVAL = 120
SQLITE = %(partition_path)s/srv/surykatka.db
NAMESERVER =
127.0.1.1
127.0.1.2
URL =
https://www.erp5.com/"""
@classmethod
def getInstanceParameterDict(cls):
return {
'nameserver': '127.0.1.1 127.0.1.2',
'check-frontend-ip': '127.0.0.1 127.0.0.2',
}
def assertSurykatkaPromises(self):
self.assertPromiseContent(
'http-query-backend-promise.py',
"'ip-list': '127.0.0.1 127.0.0.2'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'url': 'https://www.erp5.com/'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'json-file': '%s'" % (self.surykatka_json,)
)
def requestEdgetestSlaves(self):
self.requestEdgetestSlave(
'backend',
{'url': 'https://www.erp5.com/'},
)
class TestEdgeCheckStatusCode(EdgeSlaveMixin, SlapOSInstanceTestCase):
surykatka_ini = """[SURYKATKA]
INTERVAL = 120
SQLITE = %(partition_path)s/srv/surykatka.db
URL =
https://www.erp5.com/
https://www.erp5.org/"""
@classmethod
def getInstanceParameterDict(cls):
return {
'check-status-code': '500',
}
def assertSurykatkaPromises(self):
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'status-code': '501'")
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'url': 'https://www.erp5.org/'")
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'json-file': '%s'" % (self.surykatka_json,)
)
self.assertPromiseContent(
'http-query-backend-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'status-code': '500'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'url': 'https://www.erp5.com/'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'json-file': '%s'" % (self.surykatka_json,)
)
def requestEdgetestSlaves(self):
self.requestEdgetestSlave(
'backend',
{'url': 'https://www.erp5.com/'},
)
self.requestEdgetestSlave(
'backend-501',
{'url': 'https://www.erp5.org/', 'check-status-code': '501'},
)
  • It seems test is failing with

    test (test.TestEdge) ... Traceback (most recent call last):
      File "/srv/slapgrid/slappart15/srv/testnode/cqc/inst/test0-0/tmp/soft/d2d3c2d572a2b34f70dd66f8b7ab75c9/parts/surykatka/bin/surykatka", line 5, in <module>
        import re
      File "/srv/slapgrid/slappart15/srv/testnode/cqc/inst/test0-0/tmp/soft/d2d3c2d572a2b34f70dd66f8b7ab75c9/parts/python3.7/lib/python3.7/re.py", line 143, in <module>
        class RegexFlag(enum.IntFlag):
    AttributeError: module 'enum' has no attribute 'IntFlag'

    this is an error that happens when the enum34 module ( see https://pypi.org/project/enum34 it's a backport of python 3.4 enum module ) is installed on python >= 3.6, because the enum module included in python3.6 is different. The fix is in theory to not install enum34 if using python > 3.4, but something strange seems to happen with monitor plugins.

    I have not really investigated in details, but it's one of the problems we are addressing in slapos.core!146 (closed) . With jerome/slapos.core@a8bd3756 we could workaround because subprocess32 is another module name in python, but enum is enum.

    /cc @alain.takoudjou @romain

  • It seems test is failing with

    I overlooked this problem, it didn't happen in my local test runs (I am confirming now), and I merged despite the failures fixed in 6acdc8ca we had, they hidden the problem. I'll check it out.

    Bizarre, that the SR correctly installs and instantiates on all machines.

  • For now I can confirm, that the problem appears only in case if test is run python_for_test setup.py test, so exactly like in the test runner. If I run it with my environment, I have no such problem, neither it happens on real environment.

    I suspect the incompatibility is in the test infrastructure, I'm working on this.

  • This error happens when slapos-sr-testing run with python setup.py test is calling slapos node instance. So it seems that something bad is leaking to the supervisord started by testing system.

    I am investigating it.

  • I am investigating it.

    It was problem in my code, fixed in 1f59ba1b

    It seemed like originated in sueprvisor configuration, which resulted in small improvement slapos.core!170 (merged)

  • great ! Do we know what sets PYTHONPATH ?

  • great ! Do we know what sets PYTHONPATH ?

    IMHO python setup.py test, but I didn't check internally what exactly there.

  • Oh you are right:

    $ cat setup.py 
    import setuptools
    
    setuptools.setup(name='test_python_path', test_suite='tests')
    
    $ cat tests.py 
    
    import unittest
    import os
    
    
    class T(unittest.TestCase):
        def test(self):
            self.assertFalse(os.environ.get('PYTHONPATH'))
    $ PYTHONPATH= python setup.py test
    running test
    running egg_info
    writing test_python_path.egg-info/PKG-INFO
    writing dependency_links to test_python_path.egg-info/dependency_links.txt
    writing top-level names to test_python_path.egg-info/top_level.txt
    reading manifest file 'test_python_path.egg-info/SOURCES.txt'
    writing manifest file 'test_python_path.egg-info/SOURCES.txt'
    running build_ext
    test (tests.T) ... FAIL
    
    ======================================================================
    FAIL: test (tests.T)
    ----------------------------------------------------------------------
    Traceback (most recent call last):
      File "/tmp/test_python_path/tests.py", line 8, in test
        self.assertFalse(os.environ.get('PYTHONPATH'))
    AssertionError: '/tmp/test_python_path' is not false
    
    ----------------------------------------------------------------------
    Ran 1 test in 0.000s
    
    FAILED (failures=1)
    Test failed: <unittest.runner.TextTestResult run=1 errors=0 failures=1>
    error: Test failed: <unittest.runner.TextTestResult run=1 errors=0 failures=1>
    $ PYTHONPATH= pytest tests.py 
    ======================================================== test session starts ========================================================
    platform linux -- Python 3.6.6, pytest-5.3.2, py-1.8.0, pluggy-0.13.1
    rootdir: /tmp/test_python_path
    plugins: metadata-1.8.0, html-2.0.1, cov-2.8.1, remove-stale-bytecode-4.0
    collected 1 item                                                                                                                    
    
    tests.py .                                                                                                                    [100%]
    
    ========================================================= 1 passed in 0.02s =========================================================
    $ 

    this might be here

    For the records, python setup.py test is deprecated since 41.5.0 see https://github.com/pypa/setuptools/issues/1684 . They recommand tox ( in https://setuptools.readthedocs.io/en/latest/setuptools.html#test-build-package-and-run-a-unittest-suite ), but I don't think we can use tox with slapos installed eggs. Well we can, but we can only install eggs on the python running slapos, so we cannot use tox feature of running tests on multiple pythons versions (because we can only install eggs on one python )

  • Also, slapos.core unset $PYTHONPATH when running buildout but if I read correctly, that does not happen when slapos node instance invokes promises.

  • mentioned in merge request slapos.core!146 (closed)

    Toggle commit list
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment