Commit 40cd2ab0 authored by Łukasz Nowak's avatar Łukasz Nowak

monitor: Implement check-maximum-elapsed-time

Allows to configure maximum time for which the site is considered correct.

For each class (the maximum elapsed time value) separate surykatka process
is started, in order to not impact fast sites checking by long timeouts on
slow sites.
parent 98fac475
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[template] [template]
filename = instance.cfg filename = instance.cfg
md5sum = d778b6f436ae6864819eb2ff2d12a86f md5sum = 1e64cee5a2249d19f6241312f94d06b7
[template-monitor] [template-monitor]
_update_hash_filename_ = instance-monitor.cfg.jinja2 _update_hash_filename_ = instance-monitor.cfg.jinja2
...@@ -30,7 +30,7 @@ md5sum = 9e237dbdda59e788202f0da194a57d41 ...@@ -30,7 +30,7 @@ md5sum = 9e237dbdda59e788202f0da194a57d41
[template-monitor-edgebot] [template-monitor-edgebot]
_update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2 _update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
md5sum = 8786e4245db0d27dfa4815222d970e52 md5sum = b8f3c467cfd988e29be2ba70aa7a059e
[network-bench-cfg] [network-bench-cfg]
filename = network_bench.cfg.in filename = network_bench.cfg.in
...@@ -42,4 +42,4 @@ md5sum = cad2402bbd21907cfed6bc5af8c5d3ab ...@@ -42,4 +42,4 @@ md5sum = cad2402bbd21907cfed6bc5af8c5d3ab
[template-surykatka-ini] [template-surykatka-ini]
_update_hash_filename_ = surykatka.ini.jinja2 _update_hash_filename_ = surykatka.ini.jinja2
md5sum = 40870921e05d93b5843ab34abd7e3902 md5sum = a2de719a5a65438c8c3ee5195442beb6
...@@ -24,6 +24,12 @@ ...@@ -24,6 +24,12 @@
"title": "Default certificate expiration days check", "title": "Default certificate expiration days check",
"description": "Default amount of days to consider certificate as being to-be-expired (default: 15).", "description": "Default amount of days to consider certificate as being to-be-expired (default: 15).",
"type": "string" "type": "string"
},
"check-maximum-elapsed-time": {
"default": "2",
"title": "Default maximum elapsed time for a site to reply (seconds)",
"description": "Default maximum elapsed time for a site to reply to be considered good (default: 2s).",
"type": "string"
} }
} }
} }
...@@ -23,6 +23,12 @@ ...@@ -23,6 +23,12 @@
"title": "Certificate expiration days check", "title": "Certificate expiration days check",
"description": "Amount of days to consider certificate as being to-be-expired (default: comes from master partition).", "description": "Amount of days to consider certificate as being to-be-expired (default: comes from master partition).",
"type": "string" "type": "string"
},
"check-maximum-elapsed-time": {
"default": "Master default",
"title": "Maximum elapsed time for a site to reply (seconds)",
"description": "Maximum elapsed time for a site to reply to be considered good.(default: comes from master partition).",
"type": "string"
} }
} }
} }
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
{%- do CONFIGURATION.__setitem__(k[14:], v) %} {%- do CONFIGURATION.__setitem__(k[14:], v) %}
{%- endif %} {%- endif %}
{%- endfor %} {%- endfor %}
{%- set slave_instance_list = [] %} {%- set slave_instance_dict = {} %}
{%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %} {%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %}
{%- if extra_slave_instance_list %} {%- if extra_slave_instance_list %}
{#- Create slaves to process with setting up defaults #} {#- Create slaves to process with setting up defaults #}
...@@ -16,15 +16,24 @@ ...@@ -16,15 +16,24 @@
{%- if 'check-certificate-expiration-days' not in slave %} {%- if 'check-certificate-expiration-days' not in slave %}
{%- do slave.__setitem__('check-certificate-expiration-days', CONFIGURATION['check-certificate-expiration-days']) %} {%- do slave.__setitem__('check-certificate-expiration-days', CONFIGURATION['check-certificate-expiration-days']) %}
{%- endif %} {%- endif %}
{%- if 'check-maximum-elapsed-time' not in slave %}
{%- do slave.__setitem__('check-maximum-elapsed-time', CONFIGURATION['check-maximum-elapsed-time']) %}
{%- endif %}
{%- if 'check-frontend-ip' not in slave %} {%- if 'check-frontend-ip' not in slave %}
{%- do slave.__setitem__('check-frontend-ip', CONFIGURATION['check-frontend-ip']) %} {%- do slave.__setitem__('check-frontend-ip', CONFIGURATION['check-frontend-ip']) %}
{%- endif %} {%- endif %}
{%- if 'url' in slave %} {%- if 'url' in slave %}
{%- do slave_instance_list.append(slave) %} {%- set class = slave['check-maximum-elapsed-time'] %}
{%- if class not in slave_instance_dict %}
{%- do slave_instance_dict.__setitem__(class, []) %}
{%- endif %}
{%- do slave_instance_dict[class].append(slave) %}
{%- endif %} {%- endif %}
{%- endfor %} {%- endfor %}
{%- endif %} {%- endif %}
{%- set part_list = [] %} {%- set part_list = [] %}
{%- for class, slave_instance_list in slave_instance_dict.items() %}
{#- class is used to separate surykatka with different timeouts #}
{%- for slave in sorted(slave_instance_list) %} {%- for slave in sorted(slave_instance_list) %}
{%- set part_id = 'http-query-' ~ slave['slave_reference'] ~ '-promise' %} {%- set part_id = 'http-query-' ~ slave['slave_reference'] ~ '-promise' %}
{%- do part_list.append(part_id) %} {%- do part_list.append(part_id) %}
...@@ -37,75 +46,80 @@ config-report = http_query ...@@ -37,75 +46,80 @@ config-report = http_query
config-url = {{ slave['url'] }} config-url = {{ slave['url'] }}
config-status-code = {{ slave['check-status-code'] }} config-status-code = {{ slave['check-status-code'] }}
config-certificate-expiration-days = {{ slave['check-certificate-expiration-days'] }} config-certificate-expiration-days = {{ slave['check-certificate-expiration-days'] }}
config-maximum-elapsed-time = {{ slave['check-maximum-elapsed-time'] }}
config-ip-list = {{ slave['check-frontend-ip'] }} config-ip-list = {{ slave['check-frontend-ip'] }}
config-json-file = ${surykatka-config:json} config-json-file = ${surykatka-config-{{ class }}:json}
{% endfor %} {%- endfor %}
[surykatka-bot-promise] [surykatka-bot-promise-{{ class }}]
<= monitor-promise-base <= monitor-promise-base
module = check_surykatka_json module = check_surykatka_json
name = surykatka-bot-promise.py name = surykatka-bot-promise-{{ class }}.py
config-report = bot_status config-report = bot_status
config-json-file = ${surykatka-config:json} config-json-file = ${surykatka-config-{{ class }}:json}
[buildout]
extends = {{ monitor_template_output }}
parts =
cron
cron-entry-surykatka-status
monitor-base
publish-connection-information
surykatka
surykatka-bot-promise
{% for part_id in sorted(part_list) %}
{{ part_id }}
{% endfor %}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[surykatka-config] [surykatka-config-{{ class }}]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
db = ${directory:srv}/surykatka.db db = ${directory:srv}/surykatka-{{ class }}.db
rendered = ${directory:etc}/surykatka.ini rendered = ${directory:etc}/surykatka-{{ class }}.ini
template = {{ template_surykatka_ini }} template = {{ template_surykatka_ini }}
slave_instance_list = {{ dumps(slave_instance_list) }} slave_instance_list = {{ dumps(slave_instance_list) }}
nameserver = {{ dumps(CONFIGURATION['nameserver']) }} nameserver = {{ dumps(CONFIGURATION['nameserver']) }}
json = ${directory:srv}/surykatka.json json = ${directory:srv}/surykatka-{{ class }}.json
{#- timeout is just a bit bigger than class time #}
timeout = {{ int(class) + 2 }}
context = context =
import json_module json import json_module json
key db :db key db :db
key nameserver :nameserver key nameserver :nameserver
key slave_instance_list :slave_instance_list key slave_instance_list :slave_instance_list
key timeout :timeout
[surykatka] [surykatka-{{ class }}]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
config = ${surykatka-config:rendered} config = ${surykatka-config-{{ class }}:rendered}
command-line = command-line =
{{ surykatka_binary }} --run crawl --reload --configuration ${:config} {{ surykatka_binary }} --run crawl --reload --configuration ${:config}
wrapper-path = ${monitor-directory:service}/${:_buildout_section_name_} wrapper-path = ${monitor-directory:service}/${:_buildout_section_name_}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[surykatka-status-json] [surykatka-status-json-{{ class }}]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
json = ${surykatka-config-{{ class }}:json}
template = inline:#!/bin/sh template = inline:#!/bin/sh
if {{ surykatka_binary }} --run status --configuration ${surykatka:config} --output json > ${surykatka-config:json}.tmp ; then if {{ surykatka_binary }} --run status --configuration ${surykatka-{{ class }}:config} --output json > ${:json}.tmp ; then
mv -f ${surykatka-config:json}.tmp ${surykatka-config:json} mv -f ${:json}.tmp ${:json}
else else
rm -f ${surykatka-config:json}.tmp rm -f ${:json}.tmp
fi fi
rendered = ${monitor-directory:bin}/${:_buildout_section_name_} rendered = ${monitor-directory:bin}/${:_buildout_section_name_}
mode = 0755 mode = 0755
[cron-entry-surykatka-status] [cron-entry-surykatka-status-{{ class }}]
recipe = slapos.cookbook:cron.d recipe = slapos.cookbook:cron.d
cron-entries = ${directory:etc}/cron.d cron-entries = ${directory:etc}/cron.d
name = surykatka-status name = surykatka-status-{{ class }}
frequency = */2 * * * * frequency = */2 * * * *
command = ${surykatka-status-json:rendered} command = ${surykatka-status-json-{{ class }}:rendered}
{%- do part_list.append('surykatka-' + class) %}
{%- do part_list.append('surykatka-bot-promise-' + class) %}
{%- do part_list.append('cron-entry-surykatka-status-' + class) %}
{%- endfor %}
[buildout]
extends = {{ monitor_template_output }}
parts =
cron
monitor-base
publish-connection-information
{% for part_id in sorted(part_list) %}
{{ part_id }}
{% endfor %}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[publish-connection-information] [publish-connection-information]
recipe = slapos.cookbook:publish.serialised recipe = slapos.cookbook:publish.serialised
......
...@@ -79,6 +79,7 @@ configuration.check-status-code = 200 ...@@ -79,6 +79,7 @@ configuration.check-status-code = 200
configuration.nameserver = configuration.nameserver =
configuration.check-frontend-ip = configuration.check-frontend-ip =
configuration.check-certificate-expiration-days = 15 configuration.check-certificate-expiration-days = 15
configuration.check-maximum-elapsed-time = 2
# use monitor-base-port to have monitor listening on each instance # use monitor-base-port to have monitor listening on each instance
# on different port and also on different port than other services # on different port and also on different port than other services
# it makes it possible to instantiate it correctly on signle IP, for # it makes it possible to instantiate it correctly on signle IP, for
......
[SURYKATKA] [SURYKATKA]
INTERVAL = 120 INTERVAL = 120
TIMEOUT = {{ timeout }}
SQLITE = {{ db }} SQLITE = {{ db }}
{%- set nameserver_list = nameserver.split() %} {%- set nameserver_list = nameserver.split() %}
{%- if len(nameserver_list) > 0 %} {%- if len(nameserver_list) > 0 %}
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment