Commit 11e0ba60 authored by Alain Takoudjou's avatar Alain Takoudjou

Update Release Candidate

parents 2250b89f 886d9134
...@@ -37,18 +37,14 @@ pyyaml = ${pyyaml-download:target} ...@@ -37,18 +37,14 @@ pyyaml = ${pyyaml-download:target}
pyaml = ${pyaml-download:target} pyaml = ${pyaml-download:target}
install = install =
import os import os, sys
call([options['python'], '-m', 'venv', '--clear', location]) call([sys.executable, '-m', 'venv', '--clear', location])
pip = os.path.join(location, 'bin', 'pip') pip = os.path.join(location, 'bin', 'pip')
call([pip, 'install', '--no-index', options['pyyaml'], options['pyaml']]) call([pip, 'install', '--no-index', options['pyyaml'], options['pyaml']])
call([pip, 'uninstall', '-y', 'pip', 'setuptools']) call([pip, 'uninstall', '-y', 'pip', 'setuptools'])
# selftest # selftest
python = os.path.join(location, 'bin', 'python3') python = os.path.join(location, 'bin', 'python')
call([python, '-c', 'import yaml']) call([python, '-c', 'import yaml'])
python = ${python3:executable}
[librbd-python:python3]
python = ${buildout:executable}
[librbd] [librbd]
# CEPH only for librbd # CEPH only for librbd
......
...@@ -67,7 +67,7 @@ environment = ...@@ -67,7 +67,7 @@ environment =
PATH=${pkgconfig:location}/bin:${bzip2:location}/bin:%(PATH)s PATH=${pkgconfig:location}/bin:${bzip2:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${gnutls:location}/lib/pkgconfig:${gnutls:pkg-config-path}:${libpng:location}/lib/pkgconfig:${liburing:location}/lib/pkgconfig:${ncurses:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig${:PKG_CONFIG_PATH-rbd} PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${gnutls:location}/lib/pkgconfig:${gnutls:pkg-config-path}:${libpng:location}/lib/pkgconfig:${liburing:location}/lib/pkgconfig:${ncurses:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig${:PKG_CONFIG_PATH-rbd}
[qemu:[int(q) for q in platform.libc_ver()[1].split(".")] < [2,25]] [qemu:python2 or [int(q) for q in platform.libc_ver()[1].split(".")] < [2,25]]
configure-rbd = configure-rbd =
CFLAGS-rbd = CFLAGS-rbd =
LDFLAGS-rbd = LDFLAGS-rbd =
......
...@@ -38,18 +38,14 @@ markupsafe = ${systemd-markupsafe-download:target} ...@@ -38,18 +38,14 @@ markupsafe = ${systemd-markupsafe-download:target}
jinja2 = ${systemd-jinja2-download:target} jinja2 = ${systemd-jinja2-download:target}
install = install =
import os import os, sys
call([options['python'], '-m', 'venv', '--clear', location]) call([sys.executable, '-m', 'venv', '--clear', location])
pip = os.path.join(location, 'bin', 'pip') pip = os.path.join(location, 'bin', 'pip')
call([pip, 'install', '--no-index', options['markupsafe'], options['jinja2']]) call([pip, 'install', '--no-index', options['markupsafe'], options['jinja2']])
call([pip, 'uninstall', '-y', 'pip', 'setuptools']) call([pip, 'uninstall', '-y', 'pip', 'setuptools'])
# selftest # selftest
python = os.path.join(location, 'bin', 'python3') python = os.path.join(location, 'bin', 'python')
call([python, '-c', 'import jinja2']) call([python, '-c', 'import jinja2'])
python = ${python3:executable}
[systemd-python:python3]
python = ${buildout:executable}
[systemd] [systemd]
recipe = slapos.recipe.build recipe = slapos.recipe.build
......
[instance-profile] [instance-profile]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = f0714597e23ebcb0f2a1368cd065a84e md5sum = 17004b2adb98b545b16c6be60e8165e8
...@@ -7,7 +7,6 @@ ...@@ -7,7 +7,6 @@
parts = parts =
publish-connection-parameter publish-connection-parameter
download-plc download-plc
compile-plc
beremiz-runtime beremiz-runtime
#beremiz-runtime-promise #beremiz-runtime-promise
...@@ -21,13 +20,6 @@ recipe = slapos.recipe.build:download-unpacked ...@@ -21,13 +20,6 @@ recipe = slapos.recipe.build:download-unpacked
offline = false offline = false
url = ${instance-parameter:configuration.runtime_plc_url} url = ${instance-parameter:configuration.runtime_plc_url}
[compile-plc]
recipe = plone.recipe.command
stop-on-error = true
command =
{{ buildout['bin-directory'] }}/pythonwitheggs {{ buildout['directory'] }}/parts/beremiz-source/Beremiz_cli.py --project-home ${directory:home}/parts/download-plc/ build
update-command = ${:command}
[instance-parameter] [instance-parameter]
recipe = slapos.cookbook:slapconfiguration recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id} computer = ${slap-connection:computer-id}
...@@ -54,8 +46,19 @@ log = ${:var}/log ...@@ -54,8 +46,19 @@ log = ${:var}/log
[beremiz-runtime] [beremiz-runtime]
logfile = ${directory:log}/beremiz-runtime.log logfile = ${directory:log}/beremiz-runtime.log
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
# adding BEREMIZPYTHONPATH is needed so we can override the Beremiz'
# internal code which tries to use sys.executable to spawn processes
# and in the context of SlapOS it's a plain Python without needed modules
# adding LIBRARY_PATH / COMPILER_PATH is needed so that gcc can find
# needed libraries and tools inside SlapOS context
environment =
BEREMIZPYTHONPATH = {{ buildout['bin-directory'] }}/pythonwitheggs
PATH=$PATH:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin
LIBRARY_PATH=/usr/lib/gcc/x86_64-linux-gnu/9/:/usr/lib/x86_64-linux-gnu:/usr/lib:/lib/x86_64-linux-gnu/:/lib:/usr/lib/x86_64-linux-gnu/
COMPILER_PATH=/usr/lib/gcc/x86_64-linux-gnu/9/:/usr/lib/gcc/x86_64-linux-gnu/
command-line = command-line =
{{ buildout['bin-directory'] }}/pythonwitheggs {{ buildout['directory'] }}/parts/beremiz-source/Beremiz_service.py -a ${instance-parameter:configuration.autostart} -p ${instance-parameter:configuration.port} -i ${instance-parameter:configuration.interface} -x 1 ${directory:home}/parts/download-plc/build {{ buildout['bin-directory'] }}/pythonwitheggs {{ buildout['directory'] }}/parts/beremiz-source/Beremiz_cli.py -k --project-home ${directory:home}/parts/download-plc/ build transfer run
wrapper-path = ${directory:service}/beremiz-runtime wrapper-path = ${directory:service}/beremiz-runtime
[beremiz-runtime-promise] [beremiz-runtime-promise]
......
...@@ -17,7 +17,7 @@ environment += ...@@ -17,7 +17,7 @@ environment +=
[template] [template]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
url = ${:_profile_base_location_}/instance.cfg.in url = ${:_profile_base_location_}/instance.cfg.in
md5sum = d7506e861ef87977eaa554b8928d2c99 md5sum = 2e90443314d2036f7077898ab49bfec1
# XXX: "template.cfg" is hardcoded in instanciation recipe # XXX: "template.cfg" is hardcoded in instanciation recipe
output = ${buildout:directory}/template.cfg output = ${buildout:directory}/template.cfg
context = context =
......
...@@ -182,8 +182,9 @@ And now it will be safe to use the paths in ``external-disk`` parameter:: ...@@ -182,8 +182,9 @@ And now it will be safe to use the paths in ``external-disk`` parameter::
} }
} }
Of course ``external-disk-number`` and ``external-disk-size`` HAVE TO be removed Of course ``external-disk-number``, ``external-disk-size`` and
from instance parameters before continuing. ``external-disk-format`` HAVE TO be removed from instance parameters before
continuing.
For now such configuration will lead to no starting kvm process, so after For now such configuration will lead to no starting kvm process, so after
parameters are updated in SlapOS Master **and** are processed on the Compute parameters are updated in SlapOS Master **and** are processed on the Compute
......
============
Edge testing
============
``edgetest`` is a special software type of monitor software release used for website monitoring by using bots.
It uses `surykatka <https://lab.nexedi.com/nexedi/surykatka>`_ and `check_surykatka_json <https://lab.nexedi.com/nexedi/slapos.toolbox/blob/master/slapos/promise/plugin/check_surykatka_json.py>`_ to monitor websites.
``surykatka`` provides a bot to query list of hosts and a JSON reporting system.
``check_surykatka_json`` is used in promises to provide monitoring information about the websites.
In order to monitor an url one need to:
* request a monitor software release with ``edgetest`` software type, configured as described in ``instance-edgetest-input-schema.json``,
* request a slave to monitor with ``edgetest`` software type, configured as described in ``instance-edgetest-slave-input-schema.json``.
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[template] [template]
filename = instance.cfg filename = instance.cfg
md5sum = 0d26da3f83b293efbf330fd2b0350bf6 md5sum = 072b276e0b0dd4b4a96a50348a04c7a7
[template-monitor] [template-monitor]
_update_hash_filename_ = instance-monitor.cfg.jinja2 _update_hash_filename_ = instance-monitor.cfg.jinja2
...@@ -28,14 +28,6 @@ md5sum = 2eb5596544d9c341acf653d4f7ce2680 ...@@ -28,14 +28,6 @@ md5sum = 2eb5596544d9c341acf653d4f7ce2680
_update_hash_filename_ = instance-monitor-edgetest-basic.cfg.jinja2 _update_hash_filename_ = instance-monitor-edgetest-basic.cfg.jinja2
md5sum = 8e0c4041f680312ff054687d7f28275a md5sum = 8e0c4041f680312ff054687d7f28275a
[template-monitor-edgetest]
_update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2
md5sum = a16fb139d9f3d01a88f499450fac50b4
[template-monitor-edgebot]
_update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
md5sum = 7b927af12934c228d2763c5a40c634b5
[network-bench-cfg] [network-bench-cfg]
filename = network_bench.cfg.in filename = network_bench.cfg.in
md5sum = 4f4d1a0afc225656278154e453587aea md5sum = 4f4d1a0afc225656278154e453587aea
......
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {
"region-dict": {
"title": "Regions",
"description": "Defines regions of the cluster",
"patternProperties": {
".*": {
"properties": {
"state": {
"title": "State",
"description": "State of the node of the region. Can be used to destroy not needed regions.",
"type": "string",
"default": "started",
"enum": [
"started",
"stopped",
"destroyed"
]
},
"sla-computer_guid": {
"title": "GUID of the computer on which this region shall be deployed",
"description": "Unique identifier of the computer, like \"COMP-1234\". By default, let Master choose a computer.",
"type": "string",
"default": ""
},
"nameserver-list": {
"default": [],
"title": "Nameservers",
"description": "List of nameservers to use.",
"type": "array"
},
"check-frontend-ip-list": {
"default": [],
"title": "Default Frontend IPs to check",
"description": "List of default frontend IPs to check, if empty no constraint is used.",
"type": "array"
}
},
"type": "object"
}
},
"type": "object"
},
"nameserver-list": {
"default": [],
"title": "Nameservers (backward compatibility)",
"description": "List of nameservers to use. Note: This is backward compatibility, use region-dict for full configuration control.",
"type": "array"
},
"check-frontend-ip-list": {
"default": [],
"title": "Default Frontend IPs to check (backward compatibility)",
"description": "List of default frontend IPs to check, if empty no constraint is used. Note: This is backward compatibility, use region-dict for full configuration control.",
"type": "array"
}
}
}
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {
"url": {
"title": "URL to check",
"description": "URL to check, like https://example.com/",
"type": "string"
},
"region-dict": {
"title": "Applicable Regions",
"description": "Puts the check on the defined regions. No definition will result with presence in all regions.",
"patternProperties": {
".*": {
"properties": {
"state": {
"title": "State",
"description": "State of the check of the region. Used only to make it correctly visible in the SlapOS Master UI if no other parameters are defined.",
"type": "string",
"default": "present",
"enum": [
"present"
]
},
"check-frontend-ip-list": {
"default": [],
"title": "Frontend IPs to check",
"description": "List of default frontend IPs to check, if empty no constraint is used. Defaults to region configuration.",
"type": "array"
}
},
"type": "object"
}
},
"type": "object",
"default": {}
},
"check-status-code": {
"title": "HTTP Code Check",
"description": "Expected response HTTP Code.",
"type": "number",
"default": 200,
"minimum": 100,
"maximum": 599
},
"check-certificate-expiration-days": {
"title": "Certificate Expiration Check (days)",
"description": "Amount of days to consider certificate as being to-be-expired.",
"type": "number",
"default": 15,
"minimum": 1
},
"check-maximum-elapsed-time": {
"title": "Maximum Elapsed Check (seconds)",
"description": "Maximum elapsed time for a site to reply to be considered good.",
"type": "number",
"default": 2,
"minimum": 1
},
"check-http-header-dict": {
"title": "HTTP Header Check",
"description": "JSON object of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}. Note: Shall be expressed directly as object, without any additional qouting.",
"type": "object",
"default": {}
},
"failure-amount": {
"title": "Failure Amount",
"description": "Amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages.",
"type": "number",
"default": 2,
"minimum": 1
},
"check-frontend-ip-list": {
"title": "Frontend IPs to check (backward compatibility)",
"description": "List of Frontend IPs to check, if empty no constraint is used. Defaults to region configuration. Note: Use region-dict's check-frontend-ip-list to ensure specific check on each region.",
"type": "array"
}
}
}
{%- if slap_software_type == software_type %}
{%- set CONFIGURATION = {} %}
{%- for k, v in sorted(slap_configuration.items()) %}
{%- if k.startswith('configuration.') %}
{%- do CONFIGURATION.__setitem__(k[14:], v) %}
{%- endif %}
{%- endfor %}
{%- set slave_instance_dict = {} %}
{%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %}
{%- if extra_slave_instance_list %}
{#- Create slaves to process with setting up defaults #}
{%- for slave in extra_slave_instance_list | sort(attribute='-slave-title') %}
{%- do slave.setdefault('check-status-code', 200) %}
{%- do slave.setdefault('check-http-header-dict', {}) %}
{%- do slave.setdefault('check-certificate-expiration-days', 15) %}
{%- do slave.setdefault('failure-amount', 2) %}
{%- do slave.setdefault('check-maximum-elapsed-time', 2) %}
{%- do slave.setdefault('check-frontend-ip-list', CONFIGURATION['check-frontend-ip-list']) %}
{%- if 'url' in slave %}
{%- set class = slave['check-maximum-elapsed-time'] %}
{%- if class not in slave_instance_dict %}
{%- do slave_instance_dict.__setitem__(class, []) %}
{%- endif %}
{%- do slave_instance_dict[class].append(slave) %}
{%- endif %}
{%- endfor %}
{%- endif %}
{%- set part_list = [] %}
{%- for class, slave_instance_list in slave_instance_dict.items() %}
{#- class is used to separate surykatka with different timeouts #}
{%- set URL_LIST = [] %}
{%- for slave in slave_instance_list | sort(attribute='-slave-title') %}
{%- do URL_LIST.append(slave['url']) %}
{%- set part_id = 'http-query-' ~ hashlib_module.md5(slave['-slave-reference'].encode('utf-8')).hexdigest() ~ '-promise' %}
{%- do part_list.append(part_id) %}
{%- set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}
[{{part_id}}]
<= monitor-promise-base
promise = check_surykatka_json
name = {{ safe_name }}.py
config-report = http_query
config-url = {{ slave['url'] }}
config-status-code = {{ slave['check-status-code'] }}
config-http-header-dict = {{ json_module.dumps(slave['check-http-header-dict']) }}
config-certificate-expiration-days = {{ slave['check-certificate-expiration-days'] }}
config-failure-amount = {{ slave['failure-amount'] }}
config-maximum-elapsed-time = {{ slave['check-maximum-elapsed-time'] }}
config-ip-list = {{ ' '.join(slave['check-frontend-ip-list']) }}
config-json-file = ${surykatka-config-{{ class }}:json}
{%- endfor %}
[surykatka-bot-promise-{{ class }}]
<= monitor-promise-base
promise = check_surykatka_json
name = surykatka-bot-promise-{{ class }}.py
config-report = bot_status
config-json-file = ${surykatka-config-{{ class }}:json}
[surykatka-config-{{ class }}]
recipe = slapos.recipe.template:jinja2
db = ${directory:srv}/surykatka-{{ class }}.db
output = ${directory:etc}/surykatka-{{ class }}.ini
url = {{ template_surykatka_ini }}
url_list = {{ dumps(URL_LIST) }}
nameserver_list = {{ dumps(CONFIGURATION['nameserver-list']) }}
json = ${directory:srv}/surykatka-{{ class }}.json
{#- timeout is just a bit bigger than class time #}
timeout = {{ int(class) + 2 }}
context =
import json_module json
key db :db
key nameserver_list :nameserver_list
key url_list :url_list
key timeout :timeout
[surykatka-{{ class }}]
recipe = slapos.cookbook:wrapper
config = ${surykatka-config-{{ class }}:output}
command-line =
{{ surykatka_binary }} --run crawl --reload --configuration ${:config}
wrapper-path = ${monitor-directory:service}/${:_buildout_section_name_}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[surykatka-status-json-{{ class }}]
recipe = slapos.recipe.template
json = ${surykatka-config-{{ class }}:json}
inline =
#!/bin/sh
{{ surykatka_binary }} --run status --configuration ${surykatka-{{ class }}:config} --output json --stdout ${:json}
output = ${monitor-directory:bin}/${:_buildout_section_name_}
[cron-entry-surykatka-status-{{ class }}]
recipe = slapos.cookbook:cron.d
cron-entries = ${directory:etc}/cron.d
name = surykatka-status-{{ class }}
frequency = */2 * * * *
command = ${surykatka-status-json-{{ class }}:output}
{%- do part_list.append('surykatka-%i'% (class,)) %}
{%- do part_list.append('surykatka-bot-promise-%i' % (class,)) %}
{%- do part_list.append('cron-entry-surykatka-status-%i' % (class,)) %}
{%- endfor %}
[buildout]
extends = {{ monitor_template_output }}
parts =
cron
monitor-base
publish-connection-information
{% for part_id in sorted(part_list) %}
{{ part_id }}
{% endfor %}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[publish-connection-information]
recipe = slapos.cookbook:publish.serialised
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
monitor-url = ${monitor-publish-parameters:monitor-url}
monitor-user = ${monitor-publish-parameters:monitor-user}
monitor-password = ${monitor-publish-parameters:monitor-password}
[monitor-instance-parameter]
monitor-httpd-port = {{ slapparameter_dict['monitor-httpd-port'] }}
cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
{% if slapparameter_dict.get('monitor-username', '') -%}
username = {{ slapparameter_dict['monitor-username'] }}
{% endif -%}
{% if slapparameter_dict.get('monitor-password', '') -%}
password = {{ slapparameter_dict['monitor-password'] }}
{% endif -%}
interface-url = {{ slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') }}
[monitor-directory]
service = ${buildout:directory}/etc/service
var = ${buildout:directory}/var
srv = ${buildout:directory}/srv
server-log = ${:private}/server-log
monitor-log = ${:private}/monitor-log
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[slap-parameter]
{%- endif %} {#- if slap_software_type == software_type #}
{%- if slap_software_type == software_type %}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
output = ${buildout:directory}/${:filename}
extra-context =
context =
import json_module json
${:extra-context}
[slave-test-configuration]
<=jinja2-template-base
url = {{ template_json_edgetest_test }}
filename = srv/monitor/private/test.json
extensions = jinja2.ext.do
extra-context =
section slave_information slap-configuration
{% set part_list = [] -%}
{%- set edgebot_software_type = 'edgebot' %}
{%- set slave_list_name = 'extra_slave_instance_list' %}
{%- set request_dict = {} %}
{%- set namebase = "edgebot" %}
{%- if 'region-dict' not in slapparameter_dict %}
{#- Be nice and allow to work with default configuration #}
{%- do slapparameter_dict.__setitem__('region-dict', {
'1': {
'sla-computer_guid': slap_configuration['computer'],
'state': slap_configuration['instance-state'],
'nameserver-list': slapparameter_dict.get('nameserver-list', []),
'check-frontend-ip-list': slapparameter_dict.get('check-frontend-ip-list', []),
}
}) %}
{%- endif %}
{%- set active_region_list = [] %}
{%- for region_name in sorted(slapparameter_dict['region-dict']) %}
{%- set region_parameter_dict = slapparameter_dict['region-dict'][region_name] %}
{%- if region_parameter_dict.get('state', 'started') == 'started' %}
{%- do active_region_list.append(region_name) %}
{%- endif %}
{%- endfor %}
{%- set authorized_slave_dict = {} %}
{%- set publish_slave_dict_dict = {} %}
{%- for slave in slave_instance_list | sort(attribute='slave_title') %}
{%- set slave_reference = slave.pop('slave_reference') %}
{%- set publish_dict = {'assigned-region-dict': {}} %}
{%- if '_' in slave %}
{%- set base_slave_dict = json_module.loads(slave.pop('_')) %} {#- XXX: Unsafe! #}
{%- do base_slave_dict.__setitem__('-slave-title', slave['slave_title']) %}
{%- do base_slave_dict.__setitem__('-slave-reference', slave_reference) %}
{%- set slave_region_dict = base_slave_dict.pop('region-dict', {}) %}
{%- if slave_region_dict == {} %}
{%- for region in active_region_list %}
{%- do slave_region_dict.__setitem__(region, {}) %}
{%- endfor %}
{%- endif %}
{%- for region in slave_region_dict %}
{%- if region in active_region_list %}
{%- set region_info = {
'nameserver-list': slapparameter_dict['region-dict'][region].get('nameserver-list') or slapparameter_dict.get('slapparameter_dict') or [],
'check-frontend-ip-list': slave_region_dict[region].get('check-frontend-ip-list') or base_slave_dict.get('check-frontend-ip-list') or slapparameter_dict['region-dict'][region].get('check-frontend-ip-list') or slapparameter_dict.get('check-frontend-ip-list') or [],
} %}
{%- do publish_dict['assigned-region-dict'].__setitem__(region, region_info) %}
{%- set slave_dict = base_slave_dict.copy() %}
{%- do slave_dict.update(region_info) %}
{%- if region not in authorized_slave_dict %}
{%- do authorized_slave_dict.__setitem__(region, [slave_dict]) %}
{%- else %}
{%- do authorized_slave_dict[region].append(slave_dict) %}
{%- endif %}
{%- endif %}
{%- endfor %}
{%- endif %}
{%- do publish_slave_dict_dict.__setitem__(slave_reference, publish_dict) %}
{%- endfor %}
{%- set monitor_base_port = int(slap_configuration['configuration.monitor-base-port']) %}
{%- set number = {'i': 1} %}
{%- for region_name in sorted(slapparameter_dict['region-dict']) %}
{%- set region_parameter_dict = slapparameter_dict['region-dict'][region_name] %}
{%- set edgebot_name = "%s-%s" % (namebase, region_name) %}
{%- set request_section_title = 'request-%s' % (hashlib_module.md5(edgebot_name.encode('utf-8')).hexdigest(),) %}
{%- do part_list.append(request_section_title) %}
{#- Note: monitor-httpd-port will vary on regions being added and removed,
but this is accepted, as it's only internal trick #}
{%- do request_dict.__setitem__(
request_section_title,
{
'config': {
'monitor-httpd-port': monitor_base_port + number['i'],
'check-frontend-ip-list': region_parameter_dict.get('check-frontend-ip-list', []),
'nameserver-list': region_parameter_dict.get('nameserver-list', []),
'extra_slave_instance_list': authorized_slave_dict.get(region_name, [])
},
'name': edgebot_name,
'sla': {'computer_guid': region_parameter_dict['sla-computer_guid']},
'state': region_parameter_dict.get('state', 'started'),
}) %}
{%- do number.__setitem__('i', number['i'] + 1) %}
{%- endfor %}
[replicate]
<= slap-connection
recipe = slapos.cookbook:request.serialised
config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-password = ${monitor-htpasswd:passwd}
software-url = ${slap-connection:software-release-url}
software-type = {{edgebot_software_type}}
{%- set monitor_base_url_dict = {} -%}
{% for section, edgebot_request in request_dict.items() %}
[{{section}}]
<= replicate
name = {{ edgebot_request['name'] }}
state = {{ edgebot_request['state'] }}
{%- if edgebot_request['state'] != 'destroyed' %}
{%- do monitor_base_url_dict.__setitem__(section, '${' ~ section ~ ':connection-monitor-base-url}') %}
return = monitor-base-url
{%- endif %}
{%- set edgebot_configuration_dict = edgebot_request['config'] %}
{%- for config_key, config_value in edgebot_configuration_dict.items() %}
config-{{ config_key }} = {{ dumps(config_value) }}
{% endfor -%}
{%- for parameter, value in edgebot_request['sla'].items() %}
sla-{{ parameter }} = {{ value }}
{%- endfor %}
{%- endfor %}
# Publish information for each slave
{%- for slave_reference, publish_dict in publish_slave_dict_dict.items() -%}
{%- set publish_section_title = 'publish-%s' % (hashlib_module.md5(slave_reference.encode('utf-8')).hexdigest(),) -%}
{%- do part_list.append(publish_section_title) %}
[{{ publish_section_title }}]
recipe = slapos.cookbook:publish.serialised
available-region-list = {{ dumps(list(active_region_list)) }}
-slave-reference = {{ slave_reference }}
{%- for key, value in publish_dict.items() %}
{{ key }} = {{ dumps(value) }}
{%- endfor %}
{% endfor %}
[monitor-conf-parameters]
monitor-title = Monitor
password = ${monitor-htpasswd:passwd}
[monitor-base-url-dict]
{% for key, value in monitor_base_url_dict.items() -%}
{{ key }} = {{ value }}
{% endfor %}
[buildout]
extends = {{ instance_base_monitor }}
parts +=
slave-test-configuration
{% for part in part_list %}
{{ ' %s' % part }}
{%- endfor %}
[publish-connection-information]
recipe = slapos.cookbook:publish.serialised
active-region-list = {{ dumps(list(active_region_list)) }}
sla-computer_guid = {{ dumps(slap_configuration['computer']) }}
sla-instance_guid = {{ dumps(slap_configuration['instance-guid']) }}
{%- endif %} {#- if slap_software_type == software_type #}
...@@ -8,8 +8,6 @@ develop-eggs-directory = ${buildout:develop-eggs-directory} ...@@ -8,8 +8,6 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
[switch_softwaretype] [switch_softwaretype]
recipe = slapos.cookbook:switch-softwaretype recipe = slapos.cookbook:switch-softwaretype
default = instance-base-monitor:output default = instance-base-monitor:output
edgetest = instance-base-edgetest:output
edgebot = instance-base-edgebot:output
edgetest-basic = instance-edgetest-basic:output edgetest-basic = instance-edgetest-basic:output
RootSoftwareInstance = $${:default} RootSoftwareInstance = $${:default}
...@@ -54,45 +52,6 @@ extra-context = ...@@ -54,45 +52,6 @@ extra-context =
key template_surykatka_ini surykatka:ini key template_surykatka_ini surykatka:ini
key surykatka_binary surykatka:binary key surykatka_binary surykatka:binary
[instance-base-edgetest]
recipe = slapos.recipe.template:jinja2
url = ${template-monitor-edgetest:target}
output = $${buildout:directory}/template-monitor-base-edgetest.cfg
extensions = jinja2.ext.do
context = import json_module json
import hashlib_module hashlib
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
key slap_software_type slap-configuration:slap-software-type
section slap_configuration slap-configuration
raw software_type edgetest
key instance_base_monitor instance-base-monitor:output
key slave_instance_list slap-configuration:slave-instance-list
raw buildout_bin ${buildout:bin-directory}
raw template_json_edgetest_test ${json-test-template:target}
[instance-base-edgebot]
recipe = slapos.recipe.template:jinja2
url = ${template-monitor-edgebot:target}
output = $${buildout:directory}/template-monitor-edgebot.cfg
extensions = jinja2.ext.do
context = import json_module json
import hashlib_module hashlib
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
section slap_configuration slap-configuration
key slapparameter_dict slap-configuration:configuration
key slap_software_type slap-configuration:slap-software-type
raw software_type edgebot
key surykatka_binary surykatka:binary
key template_surykatka_ini surykatka:ini
raw buildout_bin ${buildout:bin-directory}
raw monitor_template_output ${monitor-template:output}
raw monitor_collect_csv_dump ${monitor-collect-csv-dump:target}
[slap-configuration] [slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap-connection:computer-id} computer = $${slap-connection:computer-id}
......
...@@ -13,7 +13,6 @@ parts = ...@@ -13,7 +13,6 @@ parts =
network-bench-cfg network-bench-cfg
json-test-template json-test-template
template template
template-monitor-edgetest
template-monitor template-monitor
[python] [python]
...@@ -32,14 +31,6 @@ url = ${:_profile_base_location_}/${:_update_hash_filename_} ...@@ -32,14 +31,6 @@ url = ${:_profile_base_location_}/${:_update_hash_filename_}
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_} url = ${:_profile_base_location_}/${:_update_hash_filename_}
[template-monitor-edgetest]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
[template-monitor-edgebot]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
[template-surykatka-ini] [template-surykatka-ini]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_} url = ${:_profile_base_location_}/${:_update_hash_filename_}
......
...@@ -17,24 +17,6 @@ ...@@ -17,24 +17,6 @@
"response": "instance-default-output-schema.json", "response": "instance-default-output-schema.json",
"serialisation": "json-in-xml", "serialisation": "json-in-xml",
"index": 1 "index": 1
},
"edgetest": {
"title": "Edge Test [OBSOLETE]",
"description": "Cluster of bots to perform a distributed monitoring. OBSOLETE: Use Edge Test Basic.",
"request": "instance-edgetest-input-schema.json",
"response": "instance-default-output-schema.json",
"serialisation": "json-in-xml",
"index": 2
},
"edgetest-slave": {
"title": "Edge Test Slave [OBSOLETE]",
"shared": true,
"software-type": "edgetest",
"description": "Cluster of bots to perform a distributed monitoring. OBSOLETE: Use Edge Test Basic.",
"request": "instance-edgetest-slave-input-schema.json",
"response": "instance-default-output-schema.json",
"serialisation": "json-in-xml",
"index": 3
} }
} }
} }
This diff is collapsed.
...@@ -24,7 +24,7 @@ md5sum = b10c10a9408a934052fd0960f31a75fc ...@@ -24,7 +24,7 @@ md5sum = b10c10a9408a934052fd0960f31a75fc
[template-lte-enb] [template-lte-enb]
_update_hash_filename_ = instance-enb.jinja2.cfg _update_hash_filename_ = instance-enb.jinja2.cfg
md5sum = b5a0f5e258fd91573425b00027ca11b4 md5sum = c5a82279b6ada358c74a90c3b660bf55
[template-lte-gnb-epc] [template-lte-gnb-epc]
_update_hash_filename_ = instance-gnb-epc.jinja2.cfg _update_hash_filename_ = instance-gnb-epc.jinja2.cfg
...@@ -36,7 +36,7 @@ md5sum = 2cfe5679374559ab94478b987d646d83 ...@@ -36,7 +36,7 @@ md5sum = 2cfe5679374559ab94478b987d646d83
[template-lte-gnb] [template-lte-gnb]
_update_hash_filename_ = instance-gnb.jinja2.cfg _update_hash_filename_ = instance-gnb.jinja2.cfg
md5sum = 771c6b550ea558c708f207c75ff00f49 md5sum = eabbdb8ced563180f0bba8ca06e4bac4
[template-lte-mme] [template-lte-mme]
_update_hash_filename_ = instance-mme.jinja2.cfg _update_hash_filename_ = instance-mme.jinja2.cfg
...@@ -48,7 +48,7 @@ md5sum = dcaac06553a3222b14c0013a13f4a149 ...@@ -48,7 +48,7 @@ md5sum = dcaac06553a3222b14c0013a13f4a149
[enb.jinja2.cfg] [enb.jinja2.cfg]
filename = config/enb.jinja2.cfg filename = config/enb.jinja2.cfg
md5sum = cd93c9c7bff2bcdd1cc01ad1cdf0c88c md5sum = f5848c48f573c79615f595b07c358ce8
[sib23.asn] [sib23.asn]
filename = config/sib23.asn filename = config/sib23.asn
...@@ -56,7 +56,7 @@ md5sum = b377dac7f1fcf94fb9ce9ebed617f36a ...@@ -56,7 +56,7 @@ md5sum = b377dac7f1fcf94fb9ce9ebed617f36a
[gnb.jinja2.cfg] [gnb.jinja2.cfg]
filename = config/gnb.jinja2.cfg filename = config/gnb.jinja2.cfg
md5sum = 91005da479a848798d128dbfd407d2c7 md5sum = b3fb1b32aa1b151892eeb87a03e1d152
[ltelogs.jinja2.sh] [ltelogs.jinja2.sh]
filename = ltelogs.jinja2.sh filename = ltelogs.jinja2.sh
...@@ -64,7 +64,7 @@ md5sum = 1ba2e065bdf14a6411e95e80db17dcfd ...@@ -64,7 +64,7 @@ md5sum = 1ba2e065bdf14a6411e95e80db17dcfd
[mme.jinja2.cfg] [mme.jinja2.cfg]
filename = config/mme.jinja2.cfg filename = config/mme.jinja2.cfg
md5sum = da10ef4c61532364c758c7845614c4d6 md5sum = a96c33a0a2a301f204cac8642267ad97
[ims.jinja2.cfg] [ims.jinja2.cfg]
filename = config/ims.jinja2.cfg filename = config/ims.jinja2.cfg
......
...@@ -4,7 +4,7 @@ ...@@ -4,7 +4,7 @@
#define N_ANTENNA_UL 2 #define N_ANTENNA_UL 2
{ {
log_options: "all.level=debug,all.max_size=32,file.rotate=1G,file.path={{ directory['tmp'] }}", log_options: "all.level=debug,all.max_size=32,file.rotate=1G,file.path=/dev/null",
log_filename: "{{ directory['log'] }}/enb.log", log_filename: "{{ directory['log'] }}/enb.log",
rf_driver: { rf_driver: {
......
...@@ -6,7 +6,7 @@ ...@@ -6,7 +6,7 @@
#define NR_LONG_PUCCH_FORMAT 2 // Values: 2, 3, 4 #define NR_LONG_PUCCH_FORMAT 2 // Values: 2, 3, 4
{ {
log_options: "all.level=debug,all.max_size=32,file.rotate=1G,file.path={{ directory['tmp'] }}", log_options: "all.level=debug,all.max_size=32,file.rotate=1G,file.path=/dev/null",
log_filename: "{{ directory['log'] }}/gnb.log", log_filename: "{{ directory['log'] }}/gnb.log",
rf_driver: { rf_driver: {
......
{ {
log_options: "all.level=debug,all.max_size=32,file.rotate=1G,file.path={{ directory['tmp'] }}", log_options: "all.level=debug,all.max_size=32,file.rotate=1G,file.path=/dev/null",
log_filename: "{{ directory['log'] }}/mme.log", log_filename: "{{ directory['log'] }}/mme.log",
{% if slapparameter_dict.get('external_enb_gnb', '') %} {% if slapparameter_dict.get('external_enb_gnb', '') %}
......
...@@ -5,7 +5,6 @@ parts = ...@@ -5,7 +5,6 @@ parts =
lte-enb-config lte-enb-config
lte-enb-service lte-enb-service
sdr-busy-promise sdr-busy-promise
remove-tmp
monitor-base monitor-base
publish-connection-information publish-connection-information
...@@ -15,11 +14,6 @@ eggs-directory = {{ eggs_directory }} ...@@ -15,11 +14,6 @@ eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }} develop-eggs-directory = {{ develop_eggs_directory }}
offline = true offline = true
[remove-tmp]
# Remove old logs stored in tmp directory to prevent disk from becoming full
recipe = plone.recipe.command
command = rm -rf ${directory:tmp}/*
[slap-configuration] [slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised recipe = slapos.cookbook:slapconfiguration.serialised
computer = {{ slap_connection['computer-id'] }} computer = {{ slap_connection['computer-id'] }}
......
...@@ -5,7 +5,6 @@ parts = ...@@ -5,7 +5,6 @@ parts =
lte-gnb-config lte-gnb-config
lte-enb-service lte-enb-service
sdr-busy-promise sdr-busy-promise
remove-tmp
monitor-base monitor-base
publish-connection-information publish-connection-information
...@@ -15,11 +14,6 @@ eggs-directory = {{ eggs_directory }} ...@@ -15,11 +14,6 @@ eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }} develop-eggs-directory = {{ develop_eggs_directory }}
offline = true offline = true
[remove-tmp]
# Remove old logs stored in tmp directory to prevent disk from becoming full
recipe = plone.recipe.command
command = rm -rf ${directory:tmp}/*
[slap-configuration] [slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised recipe = slapos.cookbook:slapconfiguration.serialised
computer = {{ slap_connection['computer-id'] }} computer = {{ slap_connection['computer-id'] }}
......
...@@ -18,7 +18,7 @@ md5sum = 55c7fd4dd6a39b31878889fbfb00f995 ...@@ -18,7 +18,7 @@ md5sum = 55c7fd4dd6a39b31878889fbfb00f995
[instance-repman.cfg] [instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in _update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum = 6fc82c4b645bceb97ba7b6f5b56d161b md5sum = dc01433ffed82386a3feee0b34767bed
[config-toml.in] [config-toml.in]
_update_hash_filename_ = templates/config.toml.in _update_hash_filename_ = templates/config.toml.in
...@@ -74,4 +74,4 @@ md5sum = 455aaf369bf5141758dc57f2c0e67b08 ...@@ -74,4 +74,4 @@ md5sum = 455aaf369bf5141758dc57f2c0e67b08
[slave-db-manage.in] [slave-db-manage.in]
_update_hash_filename_ = templates/slave-db-manage.in _update_hash_filename_ = templates/slave-db-manage.in
md5sum = cefcb8c7d17367b14414314ffd325c26 md5sum = a995bcd2751739710ba01a269d3d5b46
...@@ -110,6 +110,9 @@ output = ${directory:bin}/update-proxysql-config ...@@ -110,6 +110,9 @@ output = ${directory:bin}/update-proxysql-config
'cluster_id': instance_dict.get('cluster_id', default_slave_cluster_id), 'cluster_id': instance_dict.get('cluster_id', default_slave_cluster_id),
'require_ssl': True 'require_ssl': True
} -%} } -%}
{% if not slave_dict['cluster_id'] in cluster_name_list -%}
{% do slave_dict.__setitem__('cluster_id', default_slave_cluster_id) -%}
{% endif -%}
{% do slave_information_list.append(slave_dict) -%} {% do slave_information_list.append(slave_dict) -%}
{% do database_slave_dict[slave_dict['cluster_id']].append(slave_dict) -%} {% do database_slave_dict[slave_dict['cluster_id']].append(slave_dict) -%}
{% do db_name_dict[slave_dict['cluster_id']].append([slave_dict['name'], slave_dict['user']]) -%} {% do db_name_dict[slave_dict['cluster_id']].append([slave_dict['name'], slave_dict['user']]) -%}
......
...@@ -2,27 +2,17 @@ ...@@ -2,27 +2,17 @@
set -e set -e
curl () {
{{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
}
get_token () {
curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login
}
run_mysql () { run_mysql () {
{{ mysql_bin }} --defaults-file="{{ mysql_conf }}" --protocol=socket "$@" {{ mysql_bin }} --defaults-file="{{ mysql_conf }}" --protocol=socket "$@"
} }
TOKEN=$(get_token | {{ jq_bin }} -r '.token')
DATADIR=$(curl -H "Authorization: Bearer ${TOKEN}" \ {{ secure_url }}/api/clusters/{{ cluster_name }}/topology/master | {{ jq_bin }} -r '.slaposDatadir')
revoke_user () { revoke_user () {
DB=$1 DB=$1
UNAME=$2 UNAME=$2
if [ ! -z "$UNAME" ]; then if [ ! -z "$UNAME" ]; then
echo "Revoking all grants for user '$USER'"; echo "Revoking all grants for user '$USER'";
run_mysql -Be " run_mysql -Be "
set sql_log_bin=0;
REVOKE ALL PRIVILEGES, GRANT OPTION FROM '$UNAME'; REVOKE ALL PRIVILEGES, GRANT OPTION FROM '$UNAME';
DROP USER IF EXISTS '$UNAME'@'%'; DROP USER IF EXISTS '$UNAME'@'%';
DROP USER IF EXISTS '$UNAME'@'localhost'; DROP USER IF EXISTS '$UNAME'@'localhost';
...@@ -32,10 +22,10 @@ revoke_user () { ...@@ -32,10 +22,10 @@ revoke_user () {
fi fi
} }
# Only write or delete on master database else, we break replication. # Disable SQL_LOG_BIN, repman_slave_definition database doesn't need to be replicated
if [ "$DATADIR" = "{{ partition_dir }}" ]; then
cat << EOF > {{ tmp_dir }}/.script.sql cat << EOF > {{ tmp_dir }}/.script.sql
SET @@SESSION.SQL_LOG_BIN=0;
use repman_slave_definition; use repman_slave_definition;
CREATE TABLE IF NOT EXISTS \`slave\` ( CREATE TABLE IF NOT EXISTS \`slave\` (
...@@ -54,34 +44,33 @@ REPLACE INTO \`slave\` VALUES ('{{ name }}', true, '{{ user }}'); ...@@ -54,34 +44,33 @@ REPLACE INTO \`slave\` VALUES ('{{ name }}', true, '{{ user }}');
EOF EOF
# Update requested slaves database # Update requested slaves database
OUTPUT="{{ tmp_dir }}/removed_db.txt" OUTPUT="{{ tmp_dir }}/removed_db.txt"
run_mysql < {{ tmp_dir }}/.script.sql run_mysql < {{ tmp_dir }}/.script.sql
rm -f {{ tmp_dir }}/.script.sql rm -f {{ tmp_dir }}/.script.sql
run_mysql -NBe "SELECT name, user FROM repman_slave_definition.slave WHERE state=false" > $OUTPUT run_mysql -NBe "set sql_log_bin=0;SELECT name, user FROM repman_slave_definition.slave WHERE state=false" > $OUTPUT
DBNAME=$(run_mysql --skip-column-names -Be "SELECT name FROM repman_slave_definition.slave WHERE state=false"); DBNAME=$(run_mysql --skip-column-names -Be "set sql_log_bin=0;SELECT name FROM repman_slave_definition.slave WHERE state=false");
RET=$? RET=$?
if [ ! "$RET" = "0" ]; then if [ ! "$RET" = "0" ]; then
echo "Mysql command failed: $DBNAME" echo "Mysql command failed: $DBNAME"
exit $RET exit $RET
fi fi
if [ -z "$DBNAME" ]; then if [ -z "$DBNAME" ]; then
echo "No database for slave to remove."; echo "No database for slave to remove.";
fi fi
for NAME in $DBNAME; do for NAME in $DBNAME; do
if [ ! -z "$NAME" ]; then if [ ! -z "$NAME" ]; then
USER=$(grep -oP "$NAME\s*\K\w+" $OUTPUT); USER=$(grep -oP "$NAME\s*\K\w+" $OUTPUT);
if [ ! -z "$USER" ]; then if [ ! -z "$USER" ]; then
revoke_user $NAME $USER; revoke_user $NAME $USER;
fi fi
echo "Deleting database $NAME..." echo "Deleting database $NAME..."
run_mysql -e "DROP DATABASE IF EXISTS $NAME"; run_mysql -e "set sql_log_bin=0;DROP DATABASE IF EXISTS $NAME";
run_mysql -e "DELETE FROM repman_slave_definition.slave WHERE name='$NAME'"; run_mysql -e "set sql_log_bin=0;DELETE FROM repman_slave_definition.slave WHERE name='$NAME'";
echo "Done." echo "Done."
fi fi
done done
fi
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment