Commit 3edc447f authored by Łukasz Nowak's avatar Łukasz Nowak

Update Release Candidate

parents e892857f 6b4d1124
Pipeline #17432 failed with stage
in 0 seconds
...@@ -5,6 +5,7 @@ extends = ...@@ -5,6 +5,7 @@ extends =
../gcc/buildout.cfg ../gcc/buildout.cfg
../git/buildout.cfg ../git/buildout.cfg
../pkgconfig/buildout.cfg ../pkgconfig/buildout.cfg
../swig/buildout.cfg
./buildout.hash.cfg ./buildout.hash.cfg
parts = gowork go parts = gowork go
...@@ -21,6 +22,7 @@ make-targets= cd src && unset GOBIN && ./all.bash && cp -alf .. ${:location} ...@@ -21,6 +22,7 @@ make-targets= cd src && unset GOBIN && ./all.bash && cp -alf .. ${:location}
# some testdata files have an issue with slapos.extension.strip. # some testdata files have an issue with slapos.extension.strip.
post-install = ${findutils:location}/bin/find ${:location}/src -type d -name testdata -exec rm -rf {} \; || true post-install = ${findutils:location}/bin/find ${:location}/src -type d -name testdata -exec rm -rf {} \; || true
environment = environment =
PATH=${swig:location}/bin:%(PATH)s
GOROOT_FINAL=${:location} GOROOT_FINAL=${:location}
${:environment-extra} ${:environment-extra}
...@@ -42,15 +44,6 @@ md5sum = 6b607fc795391dc609ffd79ebf41f080 ...@@ -42,15 +44,6 @@ md5sum = 6b607fc795391dc609ffd79ebf41f080
environment-extra = environment-extra =
GOROOT_BOOTSTRAP=${golang14:location} GOROOT_BOOTSTRAP=${golang14:location}
[golang1.15]
<= golang-common
url = https://golang.org/dl/go1.15.12.src.tar.gz
md5sum = b3a5dafa7028fa00083ce4aeaf5bcb9a
# go1.15 needs go1.4 to bootstrap
environment-extra =
GOROOT_BOOTSTRAP=${golang14:location}
[golang1.16] [golang1.16]
<= golang-common <= golang-common
url = https://golang.org/dl/go1.16.8.src.tar.gz url = https://golang.org/dl/go1.16.8.src.tar.gz
...@@ -60,6 +53,14 @@ md5sum = 92e69a5e1bb6ea5e7498d12d03160032 ...@@ -60,6 +53,14 @@ md5sum = 92e69a5e1bb6ea5e7498d12d03160032
environment-extra = environment-extra =
GOROOT_BOOTSTRAP=${golang14:location} GOROOT_BOOTSTRAP=${golang14:location}
[golang1.17]
<= golang-common
url = https://golang.org/dl/go1.17.1.src.tar.gz
md5sum = a78205838c2a7054522cb91c12982f26
# go1.17 needs go1.4 to bootstrap
environment-extra =
GOROOT_BOOTSTRAP=${golang14:location}
# ---- infrastructure to build Go workspaces / projects ---- # ---- infrastructure to build Go workspaces / projects ----
......
...@@ -15,4 +15,4 @@ ...@@ -15,4 +15,4 @@
[gowork-env.sh] [gowork-env.sh]
filename = goenv.sh.in filename = goenv.sh.in
md5sum = f6d0fc24e3f08f27bc2bd85003ca5ec2 md5sum = 67a211c786a2351867c3c38f675370f7
...@@ -14,7 +14,7 @@ done <<-EOF ...@@ -14,7 +14,7 @@ done <<-EOF
EOF EOF
# PATH so that go & friends work out of the box # PATH so that go & friends work out of the box
export PATH=${gowork:golang}/bin:${git:location}/bin:${pkgconfig:location}/bin:${buildout:bin-directory}:${gcc:prefix}/bin:$PATH export PATH=${gowork:golang}/bin:${git:location}/bin:${pkgconfig:location}/bin:${buildout:bin-directory}:${gcc:prefix}/bin:${swig:location}/bin:$PATH
X=${gowork:directory} X=${gowork:directory}
export PKG_CONFIG_PATH=$(echo -n "${gowork:cpkgpath}" |tr '\n' ':'):$PKG_CONFIG_PATH export PKG_CONFIG_PATH=$(echo -n "${gowork:cpkgpath}" |tr '\n' ':'):$PKG_CONFIG_PATH
......
# Implementation of OPC UA (OPC Unified Architecture). https://open62541.org/
[buildout]
parts = open62541
extends =
../cmake/buildout.cfg
[open62541]
recipe = slapos.recipe.cmmi
shared = true
url = https://github.com/open62541/open62541/archive/refs/tags/v1.2.2.tar.gz
md5sum = 2883bde165bc9bc3d459ccbb47acf7f4
configure-command = ${cmake:location}/bin/cmake
configure-options =
-Bbuild
-DCMAKE_INSTALL_PREFIX=@@LOCATION@@
make-options = -C build
...@@ -12,3 +12,7 @@ ...@@ -12,3 +12,7 @@
repository = https://lab.nexedi.com/kirr/wendelin.core.git repository = https://lab.nexedi.com/kirr/wendelin.core.git
branch = t branch = t
revision = revision =
# Wendelin.core 2 targets Go1.17
[gowork]
golang = ${golang1.17:location}
...@@ -26,7 +26,7 @@ md5sum = 51087ac7615bd7cc01e60eb23701f625 ...@@ -26,7 +26,7 @@ md5sum = 51087ac7615bd7cc01e60eb23701f625
[profile-caddy-replicate] [profile-caddy-replicate]
filename = instance-apache-replicate.cfg.in filename = instance-apache-replicate.cfg.in
md5sum = b6fc5a004a1235ffad3af0b4cb0e661f md5sum = 99741e618b1c249bd17c9e02778d74ee
[profile-slave-list] [profile-slave-list]
_update_hash_filename_ = templates/apache-custom-slave-list.cfg.in _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
......
...@@ -103,10 +103,12 @@ context = ...@@ -103,10 +103,12 @@ context =
{% set authorized_slave_string_list = [] %} {% set authorized_slave_string_list = [] %}
{% set authorized_slave_list = [] %} {% set authorized_slave_list = [] %}
{% set rejected_slave_dict = {} %} {% set rejected_slave_dict = {} %}
{% set critical_rejected_slave_dict = {} %}
{% set warning_slave_dict = {} %} {% set warning_slave_dict = {} %}
{% set used_host_list = [] %} {% set used_host_list = [] %}
{% for slave in sorted(instance_parameter_dict['slave-instance-list']) %} {% for slave in sorted(instance_parameter_dict['slave-instance-list']) %}
{% set slave_error_list = [] %} {% set slave_error_list = [] %}
{% set slave_critical_error_list = [] %}
{% set slave_warning_list = [] %} {% set slave_warning_list = [] %}
{% set slave_server_alias_unclashed = [] %} {% set slave_server_alias_unclashed = [] %}
{% set slave_type = slave.get('type') %} {% set slave_type = slave.get('type') %}
...@@ -165,7 +167,9 @@ context = ...@@ -165,7 +167,9 @@ context =
{% endif %} {% endif %}
{% set custom_domain = slave.get('custom_domain') %} {% set custom_domain = slave.get('custom_domain') %}
{% if custom_domain and custom_domain in used_host_list %} {% if custom_domain and custom_domain in used_host_list %}
{% do slave_error_list.append('custom_domain %r clashes' % (custom_domain,)) %} {% set message = 'custom_domain %r clashes' % (custom_domain,) %}
{% do slave_error_list.append(message) %}
{% do slave_critical_error_list.append(message) %}
{% else %} {% else %}
{% do used_host_list.append(custom_domain) %} {% do used_host_list.append(custom_domain) %}
{% endif %} {% endif %}
...@@ -182,7 +186,9 @@ context = ...@@ -182,7 +186,9 @@ context =
{% if slave_alias in slave_server_alias_unclashed or slave_alias == custom_domain %} {% if slave_alias in slave_server_alias_unclashed or slave_alias == custom_domain %}
{# optionally do something about reporting back that server-alias has been unclashed #} {# optionally do something about reporting back that server-alias has been unclashed #}
{% elif slave_alias in used_host_list %} {% elif slave_alias in used_host_list %}
{% do slave_error_list.append('server-alias \'%s\' clashes' % (slave_alias,)) %} {% set message = 'server-alias \'%s\' clashes' % (slave_alias,) %}
{% do slave_error_list.append(message) %}
{% do slave_critical_error_list.append(message) %}
{% else %} {% else %}
{% do slave_server_alias_unclashed.append(slave_alias) %} {% do slave_server_alias_unclashed.append(slave_alias) %}
{% do used_host_list.append(slave_alias) %} {% do used_host_list.append(slave_alias) %}
...@@ -251,6 +257,9 @@ context = ...@@ -251,6 +257,9 @@ context =
{% else %} {% else %}
{% do rejected_slave_dict.__setitem__(slave.get('slave_reference'), sorted(slave_error_list)) %} {% do rejected_slave_dict.__setitem__(slave.get('slave_reference'), sorted(slave_error_list)) %}
{% endif %} {% endif %}
{% if len(slave_critical_error_list) > 0 %}
{% do critical_rejected_slave_dict.__setitem__(slave.get('slave_reference'), sorted(slave_critical_error_list)) %}
{% endif %}
{% if len(slave_warning_list) > 0 %} {% if len(slave_warning_list) > 0 %}
{% do warning_slave_dict.__setitem__(slave.get('slave_reference'), sorted(slave_warning_list)) %} {% do warning_slave_dict.__setitem__(slave.get('slave_reference'), sorted(slave_warning_list)) %}
{% endif %} {% endif %}
...@@ -752,9 +761,9 @@ filename = rejected-slave.json ...@@ -752,9 +761,9 @@ filename = rejected-slave.json
directory = ${directory:promise-output} directory = ${directory:promise-output}
rendered = ${:directory}/${:filename} rendered = ${:directory}/${:filename}
template = {{ software_parameter_dict['template_empty'] }} template = {{ software_parameter_dict['template_empty'] }}
{% if rejected_slave_dict %} {% if critical_rejected_slave_dict %}
{# sort_keys are important in order to avoid shuffling parameters on each run #} {# sort_keys are important in order to avoid shuffling parameters on each run #}
content = {{ dumps(json_module.dumps(rejected_slave_dict, indent=2, sort_keys=True)) }} content = {{ dumps(json_module.dumps(critical_rejected_slave_dict, indent=2, sort_keys=True)) }}
{% else %} {% else %}
content = content =
{% endif %} {% endif %}
......
...@@ -795,7 +795,7 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase): ...@@ -795,7 +795,7 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
with cls.slap.instance_supervisor_rpc as instance_supervisor: with cls.slap.instance_supervisor_rpc as instance_supervisor:
return getattr(instance_supervisor, method)(*args, **kwargs) return getattr(instance_supervisor, method)(*args, **kwargs)
def assertRejectedSlavePromiseWithPop(self, parameter_dict): def assertRejectedSlavePromiseEmptyWithPop(self, parameter_dict):
rejected_slave_promise_url = parameter_dict.pop( rejected_slave_promise_url = parameter_dict.pop(
'rejected-slave-promise-url') 'rejected-slave-promise-url')
...@@ -806,7 +806,7 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase): ...@@ -806,7 +806,7 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
else: else:
result_json = result.json() result_json = result.json()
self.assertEqual( self.assertEqual(
parameter_dict['rejected-slave-dict'], {},
result_json result_json
) )
except AssertionError: except AssertionError:
...@@ -1238,7 +1238,7 @@ class TestMasterRequestDomain(HttpFrontendTestCase, TestDataMixin): ...@@ -1238,7 +1238,7 @@ class TestMasterRequestDomain(HttpFrontendTestCase, TestDataMixin):
self.assertKeyWithPop('monitor-setup-url', parameter_dict) self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertBackendHaproxyStatisticUrl(parameter_dict) self.assertBackendHaproxyStatisticUrl(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-') self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertRejectedSlavePromiseWithPop(parameter_dict) self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
self.assertEqual( self.assertEqual(
{ {
...@@ -1269,7 +1269,7 @@ class TestMasterRequest(HttpFrontendTestCase, TestDataMixin): ...@@ -1269,7 +1269,7 @@ class TestMasterRequest(HttpFrontendTestCase, TestDataMixin):
self.assertKeyWithPop('monitor-setup-url', parameter_dict) self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertBackendHaproxyStatisticUrl(parameter_dict) self.assertBackendHaproxyStatisticUrl(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-') self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertRejectedSlavePromiseWithPop(parameter_dict) self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
self.assertEqual( self.assertEqual(
{ {
'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address, 'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
...@@ -1707,7 +1707,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): ...@@ -1707,7 +1707,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertKeyWithPop('monitor-setup-url', parameter_dict) self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertBackendHaproxyStatisticUrl(parameter_dict) self.assertBackendHaproxyStatisticUrl(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-') self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertRejectedSlavePromiseWithPop(parameter_dict) self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
expected_parameter_dict = { expected_parameter_dict = {
'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address, 'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
...@@ -5286,7 +5286,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility( ...@@ -5286,7 +5286,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
self.assertKeyWithPop('monitor-setup-url', parameter_dict) self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertBackendHaproxyStatisticUrl(parameter_dict) self.assertBackendHaproxyStatisticUrl(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-') self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertRejectedSlavePromiseWithPop(parameter_dict) self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
expected_parameter_dict = { expected_parameter_dict = {
'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address, 'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
...@@ -5296,10 +5296,6 @@ class TestSlaveSlapOSMasterCertificateCompatibility( ...@@ -5296,10 +5296,6 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
'rejected-slave-amount': '0', 'rejected-slave-amount': '0',
'slave-amount': '12', 'slave-amount': '12',
'rejected-slave-dict': { 'rejected-slave-dict': {
# u"_ssl_ca_crt_only":
# [u"ssl_ca_crt is present, so ssl_crt and ssl_key are required"],
# u"_ssl_key-ssl_crt-unsafe":
# [u"slave ssl_key and ssl_crt does not match"]
}, },
'warning-list': [ 'warning-list': [
u'apache-certificate is obsolete, please use master-key-upload-url', u'apache-certificate is obsolete, please use master-key-upload-url',
...@@ -5948,7 +5944,7 @@ class TestSlaveSlapOSMasterCertificateCompatibilityUpdate( ...@@ -5948,7 +5944,7 @@ class TestSlaveSlapOSMasterCertificateCompatibilityUpdate(
self.assertKeyWithPop('monitor-setup-url', parameter_dict) self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertBackendHaproxyStatisticUrl(parameter_dict) self.assertBackendHaproxyStatisticUrl(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-') self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertRejectedSlavePromiseWithPop(parameter_dict) self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
expected_parameter_dict = { expected_parameter_dict = {
'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address, 'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
...@@ -6053,7 +6049,7 @@ class TestSlaveCiphers(SlaveHttpFrontendTestCase, TestDataMixin): ...@@ -6053,7 +6049,7 @@ class TestSlaveCiphers(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertKeyWithPop('monitor-setup-url', parameter_dict) self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertBackendHaproxyStatisticUrl(parameter_dict) self.assertBackendHaproxyStatisticUrl(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-') self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertRejectedSlavePromiseWithPop(parameter_dict) self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
expected_parameter_dict = { expected_parameter_dict = {
'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address, 'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
...@@ -6293,6 +6289,29 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase): ...@@ -6293,6 +6289,29 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
} }
} }
def assertRejectedSlavePromiseWithPop(self, parameter_dict):
rejected_slave_promise_url = parameter_dict.pop(
'rejected-slave-promise-url')
try:
result = requests.get(rejected_slave_promise_url, verify=False)
if result.text == '':
result_json = {}
else:
result_json = result.json()
self.assertEqual(
{
u'_SITE_4': [u"custom_domain 'duplicate.example.com' clashes"],
u'_SITE_2': [u"custom_domain 'duplicate.example.com' clashes"],
u'_SITE_3': [u"server-alias 'duplicate.example.com' clashes"]
},
result_json
)
except AssertionError:
raise
except Exception as e:
self.fail(e)
def test_master_partition_state(self): def test_master_partition_state(self):
parameter_dict = self.parseConnectionParameterDict() parameter_dict = self.parseConnectionParameterDict()
self.assertKeyWithPop('monitor-setup-url', parameter_dict) self.assertKeyWithPop('monitor-setup-url', parameter_dict)
......
...@@ -32,6 +32,9 @@ parts = ...@@ -32,6 +32,9 @@ parts =
[python] [python]
part = python3 part = python3
[gowork]
golang = ${golang1.17:location}
# Macro for jinja templates. The filename is set in buildout.hash.cfg # Macro for jinja templates. The filename is set in buildout.hash.cfg
# in the section using this template # in the section using this template
[jinja-template] [jinja-template]
......
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[template] [template]
filename = instance.cfg filename = instance.cfg
md5sum = 2114ae8c1e92bd33ef1347f36f567c74 md5sum = b6c2df0d4a62473d6dae26b10c0a4adc
[template-monitor] [template-monitor]
_update_hash_filename_ = instance-monitor.cfg.jinja2 _update_hash_filename_ = instance-monitor.cfg.jinja2
...@@ -24,13 +24,17 @@ md5sum = 165a15672fc85981f68b9af2d6253254 ...@@ -24,13 +24,17 @@ md5sum = 165a15672fc85981f68b9af2d6253254
_update_hash_filename_ = json-test-template.json.in.jinja2 _update_hash_filename_ = json-test-template.json.in.jinja2
md5sum = 2eb5596544d9c341acf653d4f7ce2680 md5sum = 2eb5596544d9c341acf653d4f7ce2680
[template-monitor-edgetest-basic]
_update_hash_filename_ = instance-monitor-edgetest-basic.cfg.jinja2
md5sum = 61309a48f7b0135cba21b09247a2d8fd
[template-monitor-edgetest] [template-monitor-edgetest]
_update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2 _update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2
md5sum = 3c8ab4e78f66c974eb95afc595a13514 md5sum = 3c8ab4e78f66c974eb95afc595a13514
[template-monitor-edgebot] [template-monitor-edgebot]
_update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2 _update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
md5sum = 365a6cc6831267a73fa5ebd56ad394ee md5sum = 2ac74559d6108ca0dbabb872f1071e44
[network-bench-cfg] [network-bench-cfg]
filename = network_bench.cfg.in filename = network_bench.cfg.in
...@@ -42,4 +46,4 @@ md5sum = d3cfa1f6760e3fa64ccd64acf213bdfb ...@@ -42,4 +46,4 @@ md5sum = d3cfa1f6760e3fa64ccd64acf213bdfb
[template-surykatka-ini] [template-surykatka-ini]
_update_hash_filename_ = surykatka.ini.jinja2 _update_hash_filename_ = surykatka.ini.jinja2
md5sum = 609c6cca763b73a80fa05ee56475eb20 md5sum = 7e9b874c20faaa8190d2bf2b74caa727
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {
"nameserver-list": {
"default": [],
"title": "Nameservers",
"description": "List of nameservers IPs (like 1.2.3.4) to use.",
"type": "array"
},
"check-frontend-ip-list": {
"default": [],
"title": "Default Frontend IPs to check",
"description": "List of default frontend IPs (like 1.2.3.4) to check, if empty no constraint is used.",
"type": "array"
},
"check-dict": {
"title": "Configuration of checks",
"description": "Configure checks.",
"patternProperties": {
".*": {
"properties": {
"url-list": {
"default": [],
"title": "URLs to check",
"description": "List of URLs to check, can be domain (example.com) or URL (https://example.com/, http://example.com/path)",
"type": "array"
},
"check-status-code": {
"title": "HTTP Code Check",
"description": "Expected response HTTP Code. Defaults to global configuration.",
"type": "number",
"minimum": 100,
"maximum": 599
},
"check-frontend-ip-list": {
"title": "Frontend IPs to check (optional)",
"description": "List of frontend IPs (like 1.2.3.4) to check. Defaults to global configuration.",
"type": "array"
},
"check-certificate-expiration-days": {
"title": "Certificate Expiration Check (days)",
"description": "Amount of days to consider certificate as being to-be-expired. Defaults to global configuration.",
"type": "number",
"minimum": 1
},
"check-maximum-elapsed-time": {
"title": "Maximum Elapsed Check (seconds)",
"description": "Maximum elapsed time for a site to reply to be considered good. Defaults to global configuration.",
"type": "number",
"minimum": 1
},
"failure-amount": {
"title": "Failure Amount",
"description": "Amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages. Defaults to global configuration.",
"type": "number",
"minimum": 1
},
"check-http-header-dict": {
"title": "[UI UNSUPPORTED] HTTP Header Check",
"description": "[UI UNSUPPORTED] JSON object of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}. Note: Shall be expressed directly as object, without any additional qouting.",
"type": "object",
"default": {}
}
},
"type": "object"
}
},
"type": "object",
"default": {}
},
"check-status-code": {
"title": "Default HTTP Code Check",
"description": "Default expected response HTTP Code.",
"type": "number",
"default": 200,
"minimum": 100,
"maximum": 599
},
"check-certificate-expiration-days": {
"title": "Default Certificate Expiration Check (days)",
"description": "Default amount of days to consider certificate as being to-be-expired.",
"type": "number",
"default": 15,
"minimum": 1
},
"check-maximum-elapsed-time": {
"title": "Default Maximum Elapsed Check (seconds)",
"description": "Default maximum elapsed time for a site to reply to be considered good.",
"type": "number",
"default": 2,
"minimum": 1
},
"failure-amount": {
"title": "Default Failure Amount",
"description": "Default amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages.",
"type": "number",
"default": 2,
"minimum": 1
}
}
}
...@@ -28,7 +28,9 @@ ...@@ -28,7 +28,9 @@
{%- set part_list = [] %} {%- set part_list = [] %}
{%- for class, slave_instance_list in slave_instance_dict.items() %} {%- for class, slave_instance_list in slave_instance_dict.items() %}
{#- class is used to separate surykatka with different timeouts #} {#- class is used to separate surykatka with different timeouts #}
{%- set URL_LIST = [] %}
{%- for slave in slave_instance_list | sort(attribute='-slave-title') %} {%- for slave in slave_instance_list | sort(attribute='-slave-title') %}
{%- do URL_LIST.append(slave['url']) %}
{%- set part_id = 'http-query-' ~ hashlib_module.md5(slave['-slave-reference'].encode('utf-8')).hexdigest() ~ '-promise' %} {%- set part_id = 'http-query-' ~ hashlib_module.md5(slave['-slave-reference'].encode('utf-8')).hexdigest() ~ '-promise' %}
{%- do part_list.append(part_id) %} {%- do part_list.append(part_id) %}
{%- set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %} {%- set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}
...@@ -59,7 +61,7 @@ recipe = slapos.recipe.template:jinja2 ...@@ -59,7 +61,7 @@ recipe = slapos.recipe.template:jinja2
db = ${directory:srv}/surykatka-{{ class }}.db db = ${directory:srv}/surykatka-{{ class }}.db
rendered = ${directory:etc}/surykatka-{{ class }}.ini rendered = ${directory:etc}/surykatka-{{ class }}.ini
template = {{ template_surykatka_ini }} template = {{ template_surykatka_ini }}
slave_instance_list = {{ dumps(slave_instance_list) }} url_list = {{ dumps(URL_LIST) }}
nameserver_list = {{ dumps(CONFIGURATION['nameserver-list']) }} nameserver_list = {{ dumps(CONFIGURATION['nameserver-list']) }}
json = ${directory:srv}/surykatka-{{ class }}.json json = ${directory:srv}/surykatka-{{ class }}.json
{#- timeout is just a bit bigger than class time #} {#- timeout is just a bit bigger than class time #}
...@@ -69,7 +71,7 @@ context = ...@@ -69,7 +71,7 @@ context =
import json_module json import json_module json
key db :db key db :db
key nameserver_list :nameserver_list key nameserver_list :nameserver_list
key slave_instance_list :slave_instance_list key url_list :url_list
key timeout :timeout key timeout :timeout
[surykatka-{{ class }}] [surykatka-{{ class }}]
...@@ -84,11 +86,7 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg ...@@ -84,11 +86,7 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
json = ${surykatka-config-{{ class }}:json} json = ${surykatka-config-{{ class }}:json}
template = inline:#!/bin/sh template = inline:#!/bin/sh
if {{ surykatka_binary }} --run status --configuration ${surykatka-{{ class }}:config} --output json > ${:json}.tmp ; then {{ surykatka_binary }} --run status --configuration ${surykatka-{{ class }}:config} --output json --stdout ${:json}
mv -f ${:json}.tmp ${:json}
else
rm -f ${:json}.tmp
fi
rendered = ${monitor-directory:bin}/${:_buildout_section_name_} rendered = ${monitor-directory:bin}/${:_buildout_section_name_}
mode = 0755 mode = 0755
......
{#- PREPARE #}
{%- set DEFAULT_DICT = {
'check-status-code': 200,
'check-http-header-dict': {},
'check-certificate-expiration-days': 15,
'failure-amount': 2,
'check-maximum-elapsed-time': 2,
'check-frontend-ip-list': [],
'nameserver-list': []
} %}
{%- for default_key in ['check-frontend-ip-list', 'nameserver-list', 'check-status-code', 'check-certificate-expiration-days', 'check-maximum-elapsed-time', 'failure-amount'] %}
{%- if default_key in slapparameter_dict %}
{%- do DEFAULT_DICT.__setitem__(default_key, slapparameter_dict[default_key]) %}
{%- endif %}
{%- endfor %}
{%- set CHECK_DICT = {} %}
{%- for check_name, check_definition in slapparameter_dict['check-dict'].items() %}
{%- if 'url-list' in check_definition %}
{%- set check = DEFAULT_DICT.copy() %}
{%- set url_list = [] %}
{%- for url in check_definition.pop('url-list') %}
{%- if len(url.strip()) > 0 %}
{#- Support various URL entries #}
{%- if url.startswith('http:') or url.startswith('https:') %}
{%- if url not in url_list %}
{%- do url_list.append(url) %}
{%- endif %}
{%- else %}
{%- for schema in ['http://', 'https://'] %}
{%- set add_url = '%s%s' % (schema, url) %}
{%- if add_url not in url_list %}
{%- do url_list.append('%s%s' % (schema, url)) %}
{%- endif %}
{%- endfor %}
{%- endif %}
{%- endif %}
{%- endfor %}
{%- do check.update(check_definition) %}
{%- do check.__setitem__('url-list', url_list) %}
{%- set class = check['check-maximum-elapsed-time'] %}
{%- if class not in CHECK_DICT %}
{%- do CHECK_DICT.__setitem__(class, []) %}
{%- endif %}
{%- do check.__setitem__('-name', check_name) %}
{%- do CHECK_DICT[class].append(check) %}
{%- endif %}
{%- endfor %}
{%- set PART_LIST = [] %}
{%- for class, class_check_list in CHECK_DICT.items() %}
{#- class is used to separate surykatka with different timeouts #}
{%- set CLASS_URL_LIST = [] %}
{%- for check in class_check_list | sort(attribute='-name') %}
{%- do CLASS_URL_LIST.extend(check['url-list']) %}
{%- for url in check['url-list'] %}
{%- set promise_name = 'http-query-' ~ hashlib_module.md5(check['-name'].encode('utf-8')).hexdigest() ~ '-' ~ hashlib_module.md5(url.encode('utf-8')).hexdigest() %}
{%- set part_name = promise_name ~ '-promise' %}
{%- do PART_LIST.append(part_name) %}
[{{part_name}}]
<= monitor-promise-base
module = check_surykatka_json
name = {{ promise_name }}.py
config-report = http_query
config-url = {{ url }}
config-status-code = {{ check['check-status-code'] }}
config-http-header-dict = {{ json_module.dumps(check['check-http-header-dict']) }}
config-certificate-expiration-days = {{ check['check-certificate-expiration-days'] }}
config-failure-amount = {{ check['failure-amount'] }}
config-maximum-elapsed-time = {{ check['check-maximum-elapsed-time'] }}
config-ip-list = {{ ' '.join(check['check-frontend-ip-list']) }}
config-json-file = ${surykatka-config-{{ class }}:json}
{%- endfor %} {#- for url in check['url-list'] #}
{%- endfor %} {#- for check in class_check_list | sort(attribute='-name') #}
{%- do PART_LIST.append('surykatka-bot-%i-promise' % (class,)) %}
[surykatka-bot-{{ class }}-promise]
<= monitor-promise-base
module = check_surykatka_json
name = surykatka-bot-{{ class }}.py
config-report = bot_status
config-json-file = ${surykatka-config-{{ class }}:json}
[surykatka-config-{{ class }}]
recipe = slapos.recipe.template:jinja2
db = ${directory:srv}/surykatka-{{ class }}.db
rendered = ${directory:etc}/surykatka-{{ class }}.ini
template = {{ template_surykatka_ini }}
url_list = {{ dumps(CLASS_URL_LIST) }}
nameserver_list = {{ dumps(DEFAULT_DICT['nameserver-list']) }}
json = ${directory:srv}/surykatka-{{ class }}.json
{#- timeout is just a bit bigger than class time #}
timeout = {{ int(class) + 2 }}
context =
import json_module json
key db :db
key nameserver_list :nameserver_list
key url_list :url_list
key timeout :timeout
{%- do PART_LIST.append('surykatka-%i'% (class,)) %}
[surykatka-{{ class }}]
recipe = slapos.cookbook:wrapper
config = ${surykatka-config-{{ class }}:rendered}
command-line =
{{ surykatka_binary }} --run crawl --reload --configuration ${:config}
wrapper-path = ${monitor-directory:service}/${:_buildout_section_name_}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
{%- do PART_LIST.append('surykatka-json-%i-promise'% (class,)) %}
[surykatka-json-{{ class }}-promise]
<= monitor-promise-base
module = check_file_state
name = surykatka-json-{{ class }}.py
config-filename = ${surykatka-config-{{ class }}:json}
config-state = not-empty
# workaround for bug in check_file_state
config-url =
[surykatka-status-json-{{ class }}]
recipe = slapos.recipe.template:jinja2
json = ${surykatka-config-{{ class }}:json}
template = inline:#!/bin/sh
{{ surykatka_binary }} --run status --configuration ${surykatka-{{ class }}:config} --output json --stdout ${:json}
rendered = ${monitor-directory:bin}/${:_buildout_section_name_}
mode = 0755
{%- do PART_LIST.append('cron-entry-surykatka-status-%i' % (class,)) %}
[cron-entry-surykatka-status-{{ class }}]
recipe = slapos.cookbook:cron.d
cron-entries = ${directory:etc}/cron.d
name = surykatka-status-{{ class }}
frequency = */2 * * * *
command = ${surykatka-status-json-{{ class }}:rendered}
{%- endfor %} {#- for class, class_check_list in CHECK_DICT.items() #}
[buildout]
extends = {{ instance_base_monitor }}
parts +=
{% for part_id in sorted(PART_LIST) %}
{{ part_id }}
{% endfor %}
...@@ -10,6 +10,7 @@ recipe = slapos.cookbook:switch-softwaretype ...@@ -10,6 +10,7 @@ recipe = slapos.cookbook:switch-softwaretype
default = instance-base-monitor:rendered default = instance-base-monitor:rendered
edgetest = instance-base-edgetest:rendered edgetest = instance-base-edgetest:rendered
edgebot = instance-base-edgebot:rendered edgebot = instance-base-edgebot:rendered
edgetest-basic = instance-edgetest-basic:rendered
RootSoftwareInstance = $${:default} RootSoftwareInstance = $${:default}
[instance-base-monitor] [instance-base-monitor]
...@@ -27,6 +28,34 @@ context = key develop_eggs_directory buildout:develop-eggs-directory ...@@ -27,6 +28,34 @@ context = key develop_eggs_directory buildout:develop-eggs-directory
raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output} raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
mode = 0644 mode = 0644
[instance-template]
recipe = slapos.recipe.template:jinja2
extensions = jinja2.ext.do
rendered = $${buildout:directory}/$${:_buildout_section_name_}.cfg
context =
import json_module json
import hashlib_module hashlib
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
key slap_software_type slap-configuration:slap-software-type
key instance_base_monitor instance-base-monitor:rendered
raw buildout_bin ${buildout:bin-directory}
$${:extra-context}
mode = 0644
[surykatka]
binary = ${buildout:bin-directory}/${surykatka:script-name}
ini = ${template-surykatka-ini:target}
[instance-edgetest-basic]
<= instance-template
template = ${template-monitor-edgetest-basic:target}
extra-context =
raw software_type edgetest-basic
key template_surykatka_ini surykatka:ini
key surykatka_binary surykatka:binary
[instance-base-edgetest] [instance-base-edgetest]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = ${template-monitor-edgetest:target} template = ${template-monitor-edgetest:target}
...@@ -51,8 +80,6 @@ recipe = slapos.recipe.template:jinja2 ...@@ -51,8 +80,6 @@ recipe = slapos.recipe.template:jinja2
template = ${template-monitor-edgebot:target} template = ${template-monitor-edgebot:target}
rendered = $${buildout:directory}/template-monitor-edgebot.cfg rendered = $${buildout:directory}/template-monitor-edgebot.cfg
extensions = jinja2.ext.do extensions = jinja2.ext.do
surykatka-binary = ${buildout:bin-directory}/${surykatka:script-name}
template-surykatka-ini = ${template-surykatka-ini:target}
context = import json_module json context = import json_module json
import hashlib_module hashlib import hashlib_module hashlib
...@@ -62,8 +89,8 @@ context = import json_module json ...@@ -62,8 +89,8 @@ context = import json_module json
key slapparameter_dict slap-configuration:configuration key slapparameter_dict slap-configuration:configuration
key slap_software_type slap-configuration:slap-software-type key slap_software_type slap-configuration:slap-software-type
raw software_type edgebot raw software_type edgebot
key surykatka_binary :surykatka-binary key surykatka_binary surykatka:binary
key template_surykatka_ini :template-surykatka-ini key template_surykatka_ini surykatka:ini
raw buildout_bin ${buildout:bin-directory} raw buildout_bin ${buildout:bin-directory}
raw monitor_template_output ${monitor-template:output} raw monitor_template_output ${monitor-template:output}
raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output} raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
......
...@@ -31,6 +31,11 @@ recipe = slapos.recipe.build:download ...@@ -31,6 +31,11 @@ recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_} url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644 mode = 0644
[template-monitor-edgetest-basic]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
[template-monitor-edgetest] [template-monitor-edgetest]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_} url = ${:_profile_base_location_}/${:_update_hash_filename_}
...@@ -77,11 +82,13 @@ scripts = ...@@ -77,11 +82,13 @@ scripts =
[versions] [versions]
surykatka = 0.5.0 surykatka = 0.7.0
# For surykatka 0.5.0 # For surykatka 0.7.0
click = 7.0 click = 8.0.1
dnspython = 1.16.0 dnspython = 2.1.0
forcediphttpsadapter = 1.0.1 forcediphttpsadapter = 1.0.1
miniupnpc = 2.0.2 miniupnpc = 2.0.2
peewee = 3.13.1 peewee = 3.14.4
python-whois = 0.7.3
future = 0.18.2
...@@ -10,23 +10,31 @@ ...@@ -10,23 +10,31 @@
"response": "instance-default-output-schema.json", "response": "instance-default-output-schema.json",
"index": 0 "index": 0
}, },
"edgetest-basic": {
"title": "Edge Test Basic",
"description": "Basic URL monitoring configuration",
"request": "instance-edgetest-basic-input-schema.json",
"response": "instance-default-output-schema.json",
"serialisation": "json-in-xml",
"index": 1
},
"edgetest": { "edgetest": {
"title": "Edge Test", "title": "Edge Test [OBSOLETE]",
"description": "Cluster of bots to perform a distributed monitoring ", "description": "Cluster of bots to perform a distributed monitoring. OBSOLETE: Use Edge Test Basic.",
"request": "instance-edgetest-input-schema.json", "request": "instance-edgetest-input-schema.json",
"response": "instance-default-output-schema.json", "response": "instance-default-output-schema.json",
"serialisation": "json-in-xml", "serialisation": "json-in-xml",
"index": 1 "index": 2
}, },
"edgetest-slave": { "edgetest-slave": {
"title": "Edge Test Slave", "title": "Edge Test Slave [OBSOLETE]",
"shared": true, "shared": true,
"software-type": "edgetest", "software-type": "edgetest",
"description": "Cluster of bots to perform a distributed monitoring ", "description": "Cluster of bots to perform a distributed monitoring. OBSOLETE: Use Edge Test Basic.",
"request": "instance-edgetest-slave-input-schema.json", "request": "instance-edgetest-slave-input-schema.json",
"response": "instance-default-output-schema.json", "response": "instance-default-output-schema.json",
"serialisation": "json-in-xml", "serialisation": "json-in-xml",
"index": 2 "index": 3
} }
} }
} }
...@@ -9,9 +9,6 @@ NAMESERVER = ...@@ -9,9 +9,6 @@ NAMESERVER =
{%- endfor %} {%- endfor %}
{% endif %} {% endif %}
URL = URL =
{%- for slave in slave_instance_list | sort(attribute='-slave-title') %} {%- for url in sorted(url_list) %}
{%- if 'url' in slave %} {{ url -}}
{{ slave['url'] }}
{%- endif -%}
{% endfor %} {% endfor %}
This diff is collapsed.
...@@ -139,7 +139,7 @@ zc.recipe.egg = 2.0.3+slapos003 ...@@ -139,7 +139,7 @@ zc.recipe.egg = 2.0.3+slapos003
hexagonit.recipe.download = 1.7.post4 hexagonit.recipe.download = 1.7.post4
traitlets = 4.3.3 traitlets = 4.3.3
Jinja2 = 2.11.2 Jinja2 = 2.11.3
Importing = 1.10 Importing = 1.10
MarkupSafe = 1.0 MarkupSafe = 1.0
PyYAML = 5.4.1 PyYAML = 5.4.1
...@@ -157,7 +157,7 @@ collective.recipe.shelloutput = 0.1 ...@@ -157,7 +157,7 @@ collective.recipe.shelloutput = 0.1
collective.recipe.template = 2.0 collective.recipe.template = 2.0
configparser = 4.0.2:whl configparser = 4.0.2:whl
contextlib2 = 0.6.0.post1 contextlib2 = 0.6.0.post1
cryptography = 3.3.1 cryptography = 3.3.2
dateparser = 0.7.6 dateparser = 0.7.6
decorator = 4.3.0 decorator = 4.3.0
funcsigs = 1.0.2 funcsigs = 1.0.2
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment