Skip to content

slapos
slapos
Commit 6b4d1124 authored by Łukasz Nowak's avatar Łukasz Nowak
Browse files
Options

software/monitor: edgetest-basic implementation 

Instead of using complex mechanism with slaves (which are not monitored
anyway), allow the user to express monitoring with simple UI on normal
instance.

Additionally support for simple domains, not only URLs, is added.
parent 4dbf7fdf
Pipeline #17434 failed with stage
    in 0 seconds
    Hide whitespace changes
    Inline Side-by-side
    Showing with 714 additions and 105 deletions
    software/monitor/buildout.hash.cfg
    View file @ 6b4d1124
    ......@@ -14,7 +14,7 @@
    # not need these here).
    [template]
    filename = instance.cfg
    md5sum = 2114ae8c1e92bd33ef1347f36f567c74
    md5sum = b6c2df0d4a62473d6dae26b10c0a4adc
    [template-monitor]
    _update_hash_filename_ = instance-monitor.cfg.jinja2
    ......@@ -24,13 +24,17 @@ md5sum = 165a15672fc85981f68b9af2d6253254
    _update_hash_filename_ = json-test-template.json.in.jinja2
    md5sum = 2eb5596544d9c341acf653d4f7ce2680
    [template-monitor-edgetest-basic]
    _update_hash_filename_ = instance-monitor-edgetest-basic.cfg.jinja2
    md5sum = 61309a48f7b0135cba21b09247a2d8fd
    [template-monitor-edgetest]
    _update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2
    md5sum = 3c8ab4e78f66c974eb95afc595a13514
    [template-monitor-edgebot]
    _update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
    md5sum = 4b19eea828dc1d9b393a1537f1d8d5f5
    md5sum = 2ac74559d6108ca0dbabb872f1071e44
    [network-bench-cfg]
    filename = network_bench.cfg.in
    ......@@ -42,4 +46,4 @@ md5sum = d3cfa1f6760e3fa64ccd64acf213bdfb
    [template-surykatka-ini]
    _update_hash_filename_ = surykatka.ini.jinja2
    md5sum = 609c6cca763b73a80fa05ee56475eb20
    md5sum = 7e9b874c20faaa8190d2bf2b74caa727
    software/monitor/instance-edgetest-basic-input-schema.json 0 → 100644
    View file @ 6b4d1124
    {
    "$schema": "http://json-schema.org/draft-04/schema#",
    "type": "object",
    "properties": {
    "nameserver-list": {
    "default": [],
    "title": "Nameservers",
    "description": "List of nameservers IPs (like 1.2.3.4) to use.",
    "type": "array"
    },
    "check-frontend-ip-list": {
    "default": [],
    "title": "Default Frontend IPs to check",
    "description": "List of default frontend IPs (like 1.2.3.4) to check, if empty no constraint is used.",
    "type": "array"
    },
    "check-dict": {
    "title": "Configuration of checks",
    "description": "Configure checks.",
    "patternProperties": {
    ".*": {
    "properties": {
    "url-list": {
    "default": [],
    "title": "URLs to check",
    "description": "List of URLs to check, can be domain (example.com) or URL (https://example.com/, http://example.com/path)",
    "type": "array"
    },
    "check-status-code": {
    "title": "HTTP Code Check",
    "description": "Expected response HTTP Code. Defaults to global configuration.",
    "type": "number",
    "minimum": 100,
    "maximum": 599
    },
    "check-frontend-ip-list": {
    "title": "Frontend IPs to check (optional)",
    "description": "List of frontend IPs (like 1.2.3.4) to check. Defaults to global configuration.",
    "type": "array"
    },
    "check-certificate-expiration-days": {
    "title": "Certificate Expiration Check (days)",
    "description": "Amount of days to consider certificate as being to-be-expired. Defaults to global configuration.",
    "type": "number",
    "minimum": 1
    },
    "check-maximum-elapsed-time": {
    "title": "Maximum Elapsed Check (seconds)",
    "description": "Maximum elapsed time for a site to reply to be considered good. Defaults to global configuration.",
    "type": "number",
    "minimum": 1
    },
    "failure-amount": {
    "title": "Failure Amount",
    "description": "Amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages. Defaults to global configuration.",
    "type": "number",
    "minimum": 1
    },
    "check-http-header-dict": {
    "title": "[UI UNSUPPORTED] HTTP Header Check",
    "description": "[UI UNSUPPORTED] JSON object of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}. Note: Shall be expressed directly as object, without any additional qouting.",
    "type": "object",
    "default": {}
    }
    },
    "type": "object"
    }
    },
    "type": "object",
    "default": {}
    },
    "check-status-code": {
    "title": "Default HTTP Code Check",
    "description": "Default expected response HTTP Code.",
    "type": "number",
    "default": 200,
    "minimum": 100,
    "maximum": 599
    },
    "check-certificate-expiration-days": {
    "title": "Default Certificate Expiration Check (days)",
    "description": "Default amount of days to consider certificate as being to-be-expired.",
    "type": "number",
    "default": 15,
    "minimum": 1
    },
    "check-maximum-elapsed-time": {
    "title": "Default Maximum Elapsed Check (seconds)",
    "description": "Default maximum elapsed time for a site to reply to be considered good.",
    "type": "number",
    "default": 2,
    "minimum": 1
    },
    "failure-amount": {
    "title": "Default Failure Amount",
    "description": "Default amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages.",
    "type": "number",
    "default": 2,
    "minimum": 1
    }
    }
    }
    software/monitor/instance-monitor-edgebot.cfg.jinja2
    View file @ 6b4d1124
    ......@@ -28,7 +28,9 @@
    {%- set part_list = [] %}
    {%- for class, slave_instance_list in slave_instance_dict.items() %}
    {#- class is used to separate surykatka with different timeouts #}
    {%- set URL_LIST = [] %}
    {%- for slave in slave_instance_list | sort(attribute='-slave-title') %}
    {%- do URL_LIST.append(slave['url']) %}
    {%- set part_id = 'http-query-' ~ hashlib_module.md5(slave['-slave-reference'].encode('utf-8')).hexdigest() ~ '-promise' %}
    {%- do part_list.append(part_id) %}
    {%- set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}
    ......@@ -59,7 +61,7 @@ recipe = slapos.recipe.template:jinja2
    db = ${directory:srv}/surykatka-{{ class }}.db
    rendered = ${directory:etc}/surykatka-{{ class }}.ini
    template = {{ template_surykatka_ini }}
    slave_instance_list = {{ dumps(slave_instance_list) }}
    url_list = {{ dumps(URL_LIST) }}
    nameserver_list = {{ dumps(CONFIGURATION['nameserver-list']) }}
    json = ${directory:srv}/surykatka-{{ class }}.json
    {#- timeout is just a bit bigger than class time #}
    ......@@ -69,7 +71,7 @@ context =
    import json_module json
    key db :db
    key nameserver_list :nameserver_list
    key slave_instance_list :slave_instance_list
    key url_list :url_list
    key timeout :timeout
    [surykatka-{{ class }}]
    ......
    software/monitor/instance-monitor-edgetest-basic.cfg.jinja2 0 → 100644
    View file @ 6b4d1124
    {#- PREPARE #}
    {%- set DEFAULT_DICT = {
    'check-status-code': 200,
    'check-http-header-dict': {},
    'check-certificate-expiration-days': 15,
    'failure-amount': 2,
    'check-maximum-elapsed-time': 2,
    'check-frontend-ip-list': [],
    'nameserver-list': []
    } %}
    {%- for default_key in ['check-frontend-ip-list', 'nameserver-list', 'check-status-code', 'check-certificate-expiration-days', 'check-maximum-elapsed-time', 'failure-amount'] %}
    {%- if default_key in slapparameter_dict %}
    {%- do DEFAULT_DICT.__setitem__(default_key, slapparameter_dict[default_key]) %}
    {%- endif %}
    {%- endfor %}
    {%- set CHECK_DICT = {} %}
    {%- for check_name, check_definition in slapparameter_dict['check-dict'].items() %}
    {%- if 'url-list' in check_definition %}
    {%- set check = DEFAULT_DICT.copy() %}
    {%- set url_list = [] %}
    {%- for url in check_definition.pop('url-list') %}
    {%- if len(url.strip()) > 0 %}
    {#- Support various URL entries #}
    {%- if url.startswith('http:') or url.startswith('https:') %}
    {%- if url not in url_list %}
    {%- do url_list.append(url) %}
    {%- endif %}
    {%- else %}
    {%- for schema in ['http://', 'https://'] %}
    {%- set add_url = '%s%s' % (schema, url) %}
    {%- if add_url not in url_list %}
    {%- do url_list.append('%s%s' % (schema, url)) %}
    {%- endif %}
    {%- endfor %}
    {%- endif %}
    {%- endif %}
    {%- endfor %}
    {%- do check.update(check_definition) %}
    {%- do check.__setitem__('url-list', url_list) %}
    {%- set class = check['check-maximum-elapsed-time'] %}
    {%- if class not in CHECK_DICT %}
    {%- do CHECK_DICT.__setitem__(class, []) %}
    {%- endif %}
    {%- do check.__setitem__('-name', check_name) %}
    {%- do CHECK_DICT[class].append(check) %}
    {%- endif %}
    {%- endfor %}
    {%- set PART_LIST = [] %}
    {%- for class, class_check_list in CHECK_DICT.items() %}
    {#- class is used to separate surykatka with different timeouts #}
    {%- set CLASS_URL_LIST = [] %}
    {%- for check in class_check_list | sort(attribute='-name') %}
    {%- do CLASS_URL_LIST.extend(check['url-list']) %}
    {%- for url in check['url-list'] %}
    {%- set promise_name = 'http-query-' ~ hashlib_module.md5(check['-name'].encode('utf-8')).hexdigest() ~ '-' ~ hashlib_module.md5(url.encode('utf-8')).hexdigest() %}
    {%- set part_name = promise_name ~ '-promise' %}
    {%- do PART_LIST.append(part_name) %}
    [{{part_name}}]
    <= monitor-promise-base
    module = check_surykatka_json
    name = {{ promise_name }}.py
    config-report = http_query
    config-url = {{ url }}
    config-status-code = {{ check['check-status-code'] }}
    config-http-header-dict = {{ json_module.dumps(check['check-http-header-dict']) }}
    config-certificate-expiration-days = {{ check['check-certificate-expiration-days'] }}
    config-failure-amount = {{ check['failure-amount'] }}
    config-maximum-elapsed-time = {{ check['check-maximum-elapsed-time'] }}
    config-ip-list = {{ ' '.join(check['check-frontend-ip-list']) }}
    config-json-file = ${surykatka-config-{{ class }}:json}
    {%- endfor %} {#- for url in check['url-list'] #}
    {%- endfor %} {#- for check in class_check_list | sort(attribute='-name') #}
    {%- do PART_LIST.append('surykatka-bot-%i-promise' % (class,)) %}
    [surykatka-bot-{{ class }}-promise]
    <= monitor-promise-base
    module = check_surykatka_json
    name = surykatka-bot-{{ class }}.py
    config-report = bot_status
    config-json-file = ${surykatka-config-{{ class }}:json}
    [surykatka-config-{{ class }}]
    recipe = slapos.recipe.template:jinja2
    db = ${directory:srv}/surykatka-{{ class }}.db
    rendered = ${directory:etc}/surykatka-{{ class }}.ini
    template = {{ template_surykatka_ini }}
    url_list = {{ dumps(CLASS_URL_LIST) }}
    nameserver_list = {{ dumps(DEFAULT_DICT['nameserver-list']) }}
    json = ${directory:srv}/surykatka-{{ class }}.json
    {#- timeout is just a bit bigger than class time #}
    timeout = {{ int(class) + 2 }}
    context =
    import json_module json
    key db :db
    key nameserver_list :nameserver_list
    key url_list :url_list
    key timeout :timeout
    {%- do PART_LIST.append('surykatka-%i'% (class,)) %}
    [surykatka-{{ class }}]
    recipe = slapos.cookbook:wrapper
    config = ${surykatka-config-{{ class }}:rendered}
    command-line =
    {{ surykatka_binary }} --run crawl --reload --configuration ${:config}
    wrapper-path = ${monitor-directory:service}/${:_buildout_section_name_}
    hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
    {%- do PART_LIST.append('surykatka-json-%i-promise'% (class,)) %}
    [surykatka-json-{{ class }}-promise]
    <= monitor-promise-base
    module = check_file_state
    name = surykatka-json-{{ class }}.py
    config-filename = ${surykatka-config-{{ class }}:json}
    config-state = not-empty
    # workaround for bug in check_file_state
    config-url =
    [surykatka-status-json-{{ class }}]
    recipe = slapos.recipe.template:jinja2
    json = ${surykatka-config-{{ class }}:json}
    template = inline:#!/bin/sh
    {{ surykatka_binary }} --run status --configuration ${surykatka-{{ class }}:config} --output json --stdout ${:json}
    rendered = ${monitor-directory:bin}/${:_buildout_section_name_}
    mode = 0755
    {%- do PART_LIST.append('cron-entry-surykatka-status-%i' % (class,)) %}
    [cron-entry-surykatka-status-{{ class }}]
    recipe = slapos.cookbook:cron.d
    cron-entries = ${directory:etc}/cron.d
    name = surykatka-status-{{ class }}
    frequency = */2 * * * *
    command = ${surykatka-status-json-{{ class }}:rendered}
    {%- endfor %} {#- for class, class_check_list in CHECK_DICT.items() #}
    [buildout]
    extends = {{ instance_base_monitor }}
    parts +=
    {% for part_id in sorted(PART_LIST) %}
    {{ part_id }}
    {% endfor %}
    software/monitor/instance.cfg
    View file @ 6b4d1124
    ......@@ -10,6 +10,7 @@ recipe = slapos.cookbook:switch-softwaretype
    default = instance-base-monitor:rendered
    edgetest = instance-base-edgetest:rendered
    edgebot = instance-base-edgebot:rendered
    edgetest-basic = instance-edgetest-basic:rendered
    RootSoftwareInstance = $${:default}
    [instance-base-monitor]
    ......@@ -27,6 +28,34 @@ context = key develop_eggs_directory buildout:develop-eggs-directory
    raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
    mode = 0644
    [instance-template]
    recipe = slapos.recipe.template:jinja2
    extensions = jinja2.ext.do
    rendered = $${buildout:directory}/$${:_buildout_section_name_}.cfg
    context =
    import json_module json
    import hashlib_module hashlib
    key develop_eggs_directory buildout:develop-eggs-directory
    key eggs_directory buildout:eggs-directory
    key slapparameter_dict slap-configuration:configuration
    key slap_software_type slap-configuration:slap-software-type
    key instance_base_monitor instance-base-monitor:rendered
    raw buildout_bin ${buildout:bin-directory}
    $${:extra-context}
    mode = 0644
    [surykatka]
    binary = ${buildout:bin-directory}/${surykatka:script-name}
    ini = ${template-surykatka-ini:target}
    [instance-edgetest-basic]
    <= instance-template
    template = ${template-monitor-edgetest-basic:target}
    extra-context =
    raw software_type edgetest-basic
    key template_surykatka_ini surykatka:ini
    key surykatka_binary surykatka:binary
    [instance-base-edgetest]
    recipe = slapos.recipe.template:jinja2
    template = ${template-monitor-edgetest:target}
    ......@@ -51,8 +80,6 @@ recipe = slapos.recipe.template:jinja2
    template = ${template-monitor-edgebot:target}
    rendered = $${buildout:directory}/template-monitor-edgebot.cfg
    extensions = jinja2.ext.do
    surykatka-binary = ${buildout:bin-directory}/${surykatka:script-name}
    template-surykatka-ini = ${template-surykatka-ini:target}
    context = import json_module json
    import hashlib_module hashlib
    ......@@ -62,8 +89,8 @@ context = import json_module json
    key slapparameter_dict slap-configuration:configuration
    key slap_software_type slap-configuration:slap-software-type
    raw software_type edgebot
    key surykatka_binary :surykatka-binary
    key template_surykatka_ini :template-surykatka-ini
    key surykatka_binary surykatka:binary
    key template_surykatka_ini surykatka:ini
    raw buildout_bin ${buildout:bin-directory}
    raw monitor_template_output ${monitor-template:output}
    raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
    ......
    software/monitor/software.cfg
    View file @ 6b4d1124
    ......@@ -31,6 +31,11 @@ recipe = slapos.recipe.build:download
    url = ${:_profile_base_location_}/${:_update_hash_filename_}
    mode = 0644
    [template-monitor-edgetest-basic]
    recipe = slapos.recipe.build:download
    url = ${:_profile_base_location_}/${:_update_hash_filename_}
    mode = 0644
    [template-monitor-edgetest]
    recipe = slapos.recipe.build:download
    url = ${:_profile_base_location_}/${:_update_hash_filename_}
    ......
    software/monitor/software.cfg.json
    View file @ 6b4d1124
    ......@@ -10,23 +10,31 @@
    "response": "instance-default-output-schema.json",
    "index": 0
    },
    "edgetest-basic": {
    "title": "Edge Test Basic",
    "description": "Basic URL monitoring configuration",
    "request": "instance-edgetest-basic-input-schema.json",
    "response": "instance-default-output-schema.json",
    "serialisation": "json-in-xml",
    "index": 1
    },
    "edgetest": {
    "title": "Edge Test",
    "description": "Cluster of bots to perform a distributed monitoring ",
    "title": "Edge Test [OBSOLETE]",
    "description": "Cluster of bots to perform a distributed monitoring. OBSOLETE: Use Edge Test Basic.",
    "request": "instance-edgetest-input-schema.json",
    "response": "instance-default-output-schema.json",
    "serialisation": "json-in-xml",
    "index": 1
    "index": 2
    },
    "edgetest-slave": {
    "title": "Edge Test Slave",
    "title": "Edge Test Slave [OBSOLETE]",
    "shared": true,
    "software-type": "edgetest",
    "description": "Cluster of bots to perform a distributed monitoring ",
    "description": "Cluster of bots to perform a distributed monitoring. OBSOLETE: Use Edge Test Basic.",
    "request": "instance-edgetest-slave-input-schema.json",
    "response": "instance-default-output-schema.json",
    "serialisation": "json-in-xml",
    "index": 2
    "index": 3
    }
    }
    }
    software/monitor/surykatka.ini.jinja2
    View file @ 6b4d1124
    ......@@ -9,9 +9,6 @@ NAMESERVER =
    {%- endfor %}
    {% endif %}
    URL =
    {%- for slave in slave_instance_list | sort(attribute='-slave-title') %}
    {%- if 'url' in slave %}
    {{ slave['url'] }}
    {%- endif -%}
    {%- for url in sorted(url_list) %}
    {{ url -}}
    {% endfor %}
    software/monitor/test/test.py
    View file @ 6b4d1124
    ......@@ -137,89 +137,13 @@ class MonitorTestMixin:
    )
    class EdgeSlaveMixin(MonitorTestMixin):
    class EdgeMixin(object):
    __partition_reference__ = 'edge'
    instance_max_retry = 20
    expected_connection_parameter_dict = {}
    @classmethod
    def setUpClass(cls):
    # XXX we run these tests with --all as a workaround for the fact that after
    # requesting new shared instances we don't have promise to wait for the
    # processing of these shared instances to be completed.
    # The sequence is something like this:
    # - `requestEdgetestSlaves` will request edgetest partition
    # - first `waitForInstance` will process the edgetest partition, which will
    # request a edgebot partition, but without promise to wait for the
    # processing to be finished, so the first run of `slapos node instance`
    # exits with success code and `waitForInstance` return.
    # - second `waitForInstance` process the edgebot partition.
    # Once we implement a promise (or something similar) here, we should not
    # have to use --all
    cls.slap._force_slapos_node_instance_all = True
    return super(EdgeSlaveMixin, cls).setUpClass()
    @classmethod
    def getInstanceSoftwareType(cls):
    return 'edgetest'
    def requestEdgetestSlave(self, partition_reference, partition_parameter_kw):
    software_url = self.getSoftwareURL()
    return self.slap.request(
    software_release=software_url,
    software_type='edgetest',
    partition_reference=partition_reference,
    partition_parameter_kw={'_': json.dumps(partition_parameter_kw)},
    shared=True
    )
    def updateSurykatkaDict(self):
    for instance_reference in self.surykatka_dict:
    for class_ in self.surykatka_dict[instance_reference]:
    update_dict = {}
    update_dict['ini-file'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'etc',
    'surykatka-%s.ini' % (class_,))
    update_dict['json-file'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'srv',
    'surykatka-%s.json' % (class_,))
    update_dict['status-json'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'bin',
    'surykatka-status-json-%s' % (class_,))
    update_dict['bot-promise'] = 'surykatka-bot-promise-%s.py' % (class_,)
    update_dict['status-cron'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'etc',
    'cron.d', 'surykatka-status-%s' % (class_,))
    update_dict['db_file'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'srv',
    'surykatka-%s.db' % (class_,))
    self.surykatka_dict[instance_reference][class_].update(update_dict)
    def setUpMonitorConfigurationList(self):
    self.monitor_configuration_list = [
    {
    'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
    'version': 'RSS',
    'title': 'testing partition 0',
    'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
    'text': 'testing partition 0',
    'type': 'rss',
    'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
    },
    {
    'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
    'version': 'RSS',
    'title': 'edgebot-1',
    'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
    'text': 'edgebot-1',
    'type': 'rss',
    'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
    }
    ]
    def setUp(self):
    self.updateSurykatkaDict()
    self.setUpMonitorConfigurationList()
    def assertSurykatkaIni(self):
    expected_init_path_list = []
    ......@@ -252,11 +176,6 @@ class EdgeSlaveMixin(MonitorTestMixin):
    self.assertTrue(content in promise)
    def assertHttpQueryPromiseContent(self, instance_reference, name, content):
    hashed = 'http-query-%s-promise.py' % (
    hashlib.md5(('_' + name).encode('utf-8')).hexdigest(),)
    self.assertPromiseContent(instance_reference, hashed, content)
    def assertSurykatkaBotPromise(self):
    for instance_reference in self.surykatka_dict:
    for info_dict in self.surykatka_dict[instance_reference].values():
    ......@@ -297,6 +216,25 @@ class EdgeSlaveMixin(MonitorTestMixin):
    status_json = json.load(fh)
    self.assertIn('bot_status', status_json)
    class EdgeSlaveMixin(EdgeMixin, MonitorTestMixin):
    @classmethod
    def setUpClass(cls):
    # XXX we run these tests with --all as a workaround for the fact that after
    # requesting new shared instances we don't have promise to wait for the
    # processing of these shared instances to be completed.
    # The sequence is something like this:
    # - `requestEdgetestSlaves` will request edgetest partition
    # - first `waitForInstance` will process the edgetest partition, which
    # will request a edgebot partition, but without promise to wait for the
    # processing to be finished, so the first run of `slapos node instance`
    # exits with success code and `waitForInstance` return.
    # - second `waitForInstance` process the edgebot partition.
    # Once we implement a promise (or something similar) here, we should not
    # have to use --all
    cls.slap._force_slapos_node_instance_all = True
    return super().setUpClass()
    def assertConnectionParameterDict(self):
    serialised = self.requestDefaultInstance().getConnectionParameterDict()
    connection_parameter_dict = json.loads(serialised['_'])
    ......@@ -309,6 +247,73 @@ class EdgeSlaveMixin(MonitorTestMixin):
    connection_parameter_dict
    )
    @classmethod
    def getInstanceSoftwareType(cls):
    return 'edgetest'
    def updateSurykatkaDict(self):
    for instance_reference in self.surykatka_dict:
    for class_ in self.surykatka_dict[instance_reference]:
    update_dict = {}
    update_dict['ini-file'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'etc',
    'surykatka-%s.ini' % (class_,))
    update_dict['json-file'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'srv',
    'surykatka-%s.json' % (class_,))
    update_dict['status-json'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'bin',
    'surykatka-status-json-%s' % (class_,))
    update_dict['bot-promise'] = 'surykatka-bot-promise-%s.py' % (class_,)
    update_dict['status-cron'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'etc',
    'cron.d', 'surykatka-status-%s' % (class_,))
    update_dict['db_file'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'srv',
    'surykatka-%s.db' % (class_,))
    self.surykatka_dict[instance_reference][class_].update(update_dict)
    def assertHttpQueryPromiseContent(self, instance_reference, name, content):
    hashed = 'http-query-%s-promise.py' % (
    hashlib.md5(('_' + name).encode('utf-8')).hexdigest(),)
    self.assertPromiseContent(instance_reference, hashed, content)
    def requestEdgetestSlave(self, partition_reference, partition_parameter_kw):
    software_url = self.getSoftwareURL()
    return self.slap.request(
    software_release=software_url,
    software_type='edgetest',
    partition_reference=partition_reference,
    partition_parameter_kw={'_': json.dumps(partition_parameter_kw)},
    shared=True
    )
    def setUpMonitorConfigurationList(self):
    self.monitor_configuration_list = [
    {
    'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
    'version': 'RSS',
    'title': 'testing partition 0',
    'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
    'text': 'testing partition 0',
    'type': 'rss',
    'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
    },
    {
    'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
    'version': 'RSS',
    'title': 'edgebot-1',
    'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
    'text': 'edgebot-1',
    'type': 'rss',
    'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
    }
    ]
    def setUp(self):
    super().setUp()
    self.setUpMonitorConfigurationList()
    def test(self):
    # Note: Those tests do not run surykatka and do not do real checks, as
    # this depends too much on the environment and is really hard to
    ......@@ -950,7 +955,7 @@ URL =
    self.surykatka_dict['edge3'][2]['json-file'],))
    def test(self):
    super(TestEdgeRegion, self).test()
    super().test()
    self.assertSlaveConnectionParameterDict()
    maxDiff = None
    ......@@ -1101,7 +1106,7 @@ class TestEdgeRegionDestroyed(TestEdgeRegion):
    ]
    def test(self):
    super(TestEdgeRegionDestroyed, self).test()
    super().test()
    # hack around @classmethod
    self.__class__.instance_parameter_dict[
    'region-dict']['Region Three']['state'] = 'destroyed'
    ......@@ -1256,7 +1261,7 @@ class TestEdgeRegionAdded(TestEdgeRegion):
    ]
    def test(self):
    super(TestEdgeRegionAdded, self).test()
    super().test()
    self.__class__.instance_parameter_dict['region-dict']['Region Four'] = {
    'sla-computer_guid': 'local',
    'state': 'started',
    ......@@ -1408,3 +1413,320 @@ URL =
    'status-code': '200',
    'url': 'https://www.all.org/'}""" % (
    self.surykatka_dict['edge4'][2]['json-file'],))
    class TestEdgeBasic(EdgeMixin, SlapOSInstanceTestCase):
    surykatka_dict = {}
    def assertConnectionParameterDict(self):
    connection_parameter_dict = self.requestDefaultInstance(
    ).getConnectionParameterDict()
    # tested elsewhere
    connection_parameter_dict.pop('monitor-setup-url', None)
    # comes from instance-monitor.cfg.jinja2, not needed here
    connection_parameter_dict.pop('server_log_url', None)
    self.assertEqual(
    self.expected_connection_parameter_dict,
    connection_parameter_dict
    )
    def assertHttpQueryPromiseContent(
    self, instance_reference, name, url, content):
    hashed = 'http-query-%s-%s.py' % (
    hashlib.md5((name).encode('utf-8')).hexdigest(),
    hashlib.md5((url).encode('utf-8')).hexdigest(),
    )
    self.assertPromiseContent(instance_reference, hashed, content)
    def updateSurykatkaDict(self):
    for instance_reference in self.surykatka_dict:
    for class_ in self.surykatka_dict[instance_reference]:
    update_dict = {}
    update_dict['ini-file'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'etc',
    'surykatka-%s.ini' % (class_,))
    update_dict['json-file'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'srv',
    'surykatka-%s.json' % (class_,))
    update_dict['status-json'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'bin',
    'surykatka-status-json-%s' % (class_,))
    update_dict['bot-promise'] = 'surykatka-bot-%s.py' % (class_,)
    update_dict['status-cron'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'etc',
    'cron.d', 'surykatka-status-%s' % (class_,))
    update_dict['db_file'] = os.path.join(
    self.slap.instance_directory, instance_reference, 'srv',
    'surykatka-%s.db' % (class_,))
    self.surykatka_dict[instance_reference][class_].update(update_dict)
    @classmethod
    def getInstanceParameterDict(cls):
    return {'_': json.dumps({
    'nameserver-list': ['127.0.1.1', '127.0.1.2'],
    'check-frontend-ip-list': ['127.0.0.1', '127.0.0.2'],
    "check-maximum-elapsed-time": 5,
    "check-certificate-expiration-days": 7,
    "check-status-code": 201,
    "failure-amount": 1,
    "check-dict": {
    "path-check": {
    "url-list": [
    "https://path.example.com/path",
    ]
    },
    "domain-check": {
    "url-list": [
    "domain.example.com",
    ]
    },
    "frontend-check": {
    "url-list": [
    "https://frontend.example.com",
    ],
    "check-frontend-ip-list": ['127.0.0.3'],
    },
    "frontend-empty-check": {
    "url-list": [
    "https://frontendempty.example.com",
    ],
    "check-frontend-ip-list": [],
    },
    "status-check": {
    "url-list": [
    "https://status.example.com",
    ],
    "check-status-code": 202,
    },
    "certificate-check": {
    "url-list": [
    "https://certificate.example.com",
    ],
    "check-certificate-expiration-days": 11,
    },
    "time-check": {
    "url-list": [
    "https://time.example.com",
    ],
    "check-maximum-elapsed-time": 11,
    },
    "failure-check": {
    "url-list": [
    "https://failure.example.com",
    ],
    "failure-amount": 3,
    },
    "header-check": {
    "url-list": [
    "https://header.example.com",
    ],
    'check-http-header-dict': {"A": "AAA"},
    },
    }
    })}
    surykatka_dict = {
    'edge0': {
    5: {'expected_ini': """[SURYKATKA]
    INTERVAL = 120
    TIMEOUT = 7
    SQLITE = %(db_file)s
    NAMESERVER =
    127.0.1.1
    127.0.1.2
    URL =
    http://domain.example.com
    https://certificate.example.com
    https://domain.example.com
    https://failure.example.com
    https://frontend.example.com
    https://frontendempty.example.com
    https://header.example.com
    https://path.example.com/path
    https://status.example.com
    """},
    11: {'expected_ini': """[SURYKATKA]
    INTERVAL = 120
    TIMEOUT = 13
    SQLITE = %(db_file)s
    NAMESERVER =
    127.0.1.1
    127.0.1.2
    URL =
    https://time.example.com
    """},
    }
    }
    @classmethod
    def getInstanceSoftwareType(cls):
    return 'edgetest-basic'
    def assertSurykatkaPromises(self):
    self.assertHttpQueryPromiseContent(
    'edge0',
    'path-check',
    'https://path.example.com/path',
    """extra_config_dict = { 'certificate-expiration-days': '7',
    'failure-amount': '1',
    'http-header-dict': '{}',
    'ip-list': '127.0.0.1 127.0.0.2',
    'json-file': '%s',
    'maximum-elapsed-time': '5',
    'report': 'http_query',
    'status-code': '201',
    'url': 'https://path.example.com/path'}""" % (
    self.surykatka_dict['edge0'][5]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge0',
    'domain-check',
    'https://domain.example.com',
    """extra_config_dict = { 'certificate-expiration-days': '7',
    'failure-amount': '1',
    'http-header-dict': '{}',
    'ip-list': '127.0.0.1 127.0.0.2',
    'json-file': '%s',
    'maximum-elapsed-time': '5',
    'report': 'http_query',
    'status-code': '201',
    'url': 'https://domain.example.com'}""" % (
    self.surykatka_dict['edge0'][5]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge0',
    'domain-check',
    'http://domain.example.com',
    """extra_config_dict = { 'certificate-expiration-days': '7',
    'failure-amount': '1',
    'http-header-dict': '{}',
    'ip-list': '127.0.0.1 127.0.0.2',
    'json-file': '%s',
    'maximum-elapsed-time': '5',
    'report': 'http_query',
    'status-code': '201',
    'url': 'http://domain.example.com'}""" % (
    self.surykatka_dict['edge0'][5]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge0',
    'frontend-check',
    'https://frontend.example.com',
    """extra_config_dict = { 'certificate-expiration-days': '7',
    'failure-amount': '1',
    'http-header-dict': '{}',
    'ip-list': '127.0.0.3',
    'json-file': '%s',
    'maximum-elapsed-time': '5',
    'report': 'http_query',
    'status-code': '201',
    'url': 'https://frontend.example.com'}""" % (
    self.surykatka_dict['edge0'][5]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge0',
    'frontend-empty-check',
    'https://frontendempty.example.com',
    """extra_config_dict = { 'certificate-expiration-days': '7',
    'failure-amount': '1',
    'http-header-dict': '{}',
    'ip-list': '',
    'json-file': '%s',
    'maximum-elapsed-time': '5',
    'report': 'http_query',
    'status-code': '201',
    'url': 'https://frontendempty.example.com'}""" % (
    self.surykatka_dict['edge0'][5]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge0',
    'status-check',
    'https://status.example.com',
    """extra_config_dict = { 'certificate-expiration-days': '7',
    'failure-amount': '1',
    'http-header-dict': '{}',
    'ip-list': '127.0.0.1 127.0.0.2',
    'json-file': '%s',
    'maximum-elapsed-time': '5',
    'report': 'http_query',
    'status-code': '202',
    'url': 'https://status.example.com'}""" % (
    self.surykatka_dict['edge0'][5]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge0',
    'certificate-check',
    'https://certificate.example.com',
    """extra_config_dict = { 'certificate-expiration-days': '11',
    'failure-amount': '1',
    'http-header-dict': '{}',
    'ip-list': '127.0.0.1 127.0.0.2',
    'json-file': '%s',
    'maximum-elapsed-time': '5',
    'report': 'http_query',
    'status-code': '201',
    'url': 'https://certificate.example.com'}""" % (
    self.surykatka_dict['edge0'][5]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge0',
    'time-check',
    'https://time.example.com',
    """extra_config_dict = { 'certificate-expiration-days': '7',
    'failure-amount': '1',
    'http-header-dict': '{}',
    'ip-list': '127.0.0.1 127.0.0.2',
    'json-file': '%s',
    'maximum-elapsed-time': '11',
    'report': 'http_query',
    'status-code': '201',
    'url': 'https://time.example.com'}""" % (
    self.surykatka_dict['edge0'][11]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge0',
    'failure-check',
    'https://failure.example.com',
    """extra_config_dict = { 'certificate-expiration-days': '7',
    'failure-amount': '3',
    'http-header-dict': '{}',
    'ip-list': '127.0.0.1 127.0.0.2',
    'json-file': '%s',
    'maximum-elapsed-time': '5',
    'report': 'http_query',
    'status-code': '201',
    'url': 'https://failure.example.com'}""" % (
    self.surykatka_dict['edge0'][5]['json-file'],))
    self.assertHttpQueryPromiseContent(
    'edge0',
    'header-check',
    'https://header.example.com',
    """extra_config_dict = { 'certificate-expiration-days': '7',
    'failure-amount': '1',
    'http-header-dict': '{"A": "AAA"}',
    'ip-list': '127.0.0.1 127.0.0.2',
    'json-file': '%s',
    'maximum-elapsed-time': '5',
    'report': 'http_query',
    'status-code': '201',
    'url': 'https://header.example.com'}""" % (
    self.surykatka_dict['edge0'][5]['json-file'],))
    def test(self):
    # Note: Those tests do not run surykatka and do not do real checks, as
    # this depends too much on the environment and is really hard to
    # mock
    # So it is possible that some bugs might slip under the radar
    # Nevertheless the surykatka and check_surykatka_json are heavily
    # unit tested, and configuration created by the profiles is asserted
    # here, so it shall be enough as reasonable status
    self.initiateSurykatkaRun()
    self.assertSurykatkaStatusJSON()
    self.assertSurykatkaIni()
    self.assertSurykatkaBotPromise()
    self.assertSurykatkaPromises()
    self.assertSurykatkaCron()
    self.assertConnectionParameterDict()
    Markdown is supported
    0% or
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment
    GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7