Commit 64f10409 authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼

Update Release Candidate

parents ad5fe700 7ae4d31a
...@@ -14,7 +14,7 @@ recipe = slapos.recipe.build:gitclone ...@@ -14,7 +14,7 @@ recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/cloudooo.git repository = https://lab.nexedi.com/nexedi/cloudooo.git
branch = master branch = master
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
revision = be6c35c0156e028f31da3ccb205afc6f95728d97 revision = 0b5ff71a2ede76499e81659aed392057ae910917
[cloudooo] [cloudooo]
recipe = zc.recipe.egg recipe = zc.recipe.egg
......
...@@ -62,7 +62,7 @@ md5sum = 975177dedf677d24e14cede5d13187ce ...@@ -62,7 +62,7 @@ md5sum = 975177dedf677d24e14cede5d13187ce
[template-trafficserver-records-config] [template-trafficserver-records-config]
_update_hash_filename_ = templates/trafficserver/records.config.jinja2 _update_hash_filename_ = templates/trafficserver/records.config.jinja2
md5sum = b99403e02d1aff471a7d5ebd0afbdb6c md5sum = 1696321871bb71083550c95d77487602
[template-trafficserver-storage-config] [template-trafficserver-storage-config]
_update_hash_filename_ = templates/trafficserver/storage.config.jinja2 _update_hash_filename_ = templates/trafficserver/storage.config.jinja2
......
...@@ -80,7 +80,7 @@ CONFIG proxy.config.http.uncacheable_requests_bypass_parent INT 1 ...@@ -80,7 +80,7 @@ CONFIG proxy.config.http.uncacheable_requests_bypass_parent INT 1
CONFIG proxy.config.http.keep_alive_no_activity_timeout_in INT 120 CONFIG proxy.config.http.keep_alive_no_activity_timeout_in INT 120
CONFIG proxy.config.http.keep_alive_no_activity_timeout_out INT 120 CONFIG proxy.config.http.keep_alive_no_activity_timeout_out INT 120
CONFIG proxy.config.http.transaction_no_activity_timeout_in INT {{ ats_configuration['request-timeout'] }} CONFIG proxy.config.http.transaction_no_activity_timeout_in INT {{ ats_configuration['request-timeout'] }}
CONFIG proxy.config.http.transaction_no_activity_timeout_out INT 30 {{ ats_configuration['request-timeout'] }} CONFIG proxy.config.http.transaction_no_activity_timeout_out INT {{ ats_configuration['request-timeout'] }}
CONFIG proxy.config.http.transaction_active_timeout_in INT 900 CONFIG proxy.config.http.transaction_active_timeout_in INT 900
CONFIG proxy.config.http.transaction_active_timeout_out INT 0 CONFIG proxy.config.http.transaction_active_timeout_out INT 0
CONFIG proxy.config.http.accept_no_activity_timeout INT 120 CONFIG proxy.config.http.accept_no_activity_timeout INT 120
......
...@@ -18,4 +18,4 @@ md5sum = e986de01a57161b32425f1cd3ccac924 ...@@ -18,4 +18,4 @@ md5sum = e986de01a57161b32425f1cd3ccac924
[template-cloudooo-instance] [template-cloudooo-instance]
filename = instance-cloudooo.cfg.in filename = instance-cloudooo.cfg.in
md5sum = 9e1a66cf18d7c30c14afeb66c20afb46 md5sum = 440f2b82b119cbfa6f8c7d27652c3170
...@@ -162,6 +162,7 @@ environment = ...@@ -162,6 +162,7 @@ environment =
LD_LIBRARY_PATH = {{ parameter_dict['cairo'] }}/lib:{{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['gcc'] }}/lib:{{ parameter_dict['gcc'] }}/lib64:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['glu'] }}/lib:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['libexpat'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['mesa'] }}/lib:{{ parameter_dict['pixman'] }}/lib:{{ parameter_dict['xdamage'] }}/lib:{{ parameter_dict['xfixes'] }}/lib:{{ parameter_dict['zlib'] }}/lib LD_LIBRARY_PATH = {{ parameter_dict['cairo'] }}/lib:{{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['gcc'] }}/lib:{{ parameter_dict['gcc'] }}/lib64:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['glu'] }}/lib:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['libexpat'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['mesa'] }}/lib:{{ parameter_dict['pixman'] }}/lib:{{ parameter_dict['xdamage'] }}/lib:{{ parameter_dict['xfixes'] }}/lib:{{ parameter_dict['zlib'] }}/lib
FONTCONFIG_FILE = ${fontconfig-conf:rendered} FONTCONFIG_FILE = ${fontconfig-conf:rendered}
PATH = ${binary-link:target-directory} PATH = ${binary-link:target-directory}
LANG = C.UTF-8
mimetype_entry_addition = mimetype_entry_addition =
{% for entry in mimetype_entry_addition.splitlines() -%} {% for entry in mimetype_entry_addition.splitlines() -%}
{{ " " ~ entry.strip() }} {{ " " ~ entry.strip() }}
...@@ -172,7 +173,7 @@ ooo-binary-path = {{ parameter_dict['libreoffice-bin'] }}/program ...@@ -172,7 +173,7 @@ ooo-binary-path = {{ parameter_dict['libreoffice-bin'] }}/program
ooo-paster = {{ bin_directory }}/cloudooo_paster ooo-paster = {{ bin_directory }}/cloudooo_paster
ooo-uno-path = {{ parameter_dict['libreoffice-bin'] }}/basis-link/program ooo-uno-path = {{ parameter_dict['libreoffice-bin'] }}/basis-link/program
{% for index in range(backend_count) -%} {% for index in range(1, backend_count + 1) -%}
{% set name = 'cloudooo-' ~ index -%} {% set name = 'cloudooo-' ~ index -%}
[{{ cloudooo(name) }}] [{{ cloudooo(name) }}]
< = cloudooo-base < = cloudooo-base
...@@ -207,8 +208,8 @@ prepend-path = ${buildout:bin-directory} ...@@ -207,8 +208,8 @@ prepend-path = ${buildout:bin-directory}
run-unit-test = ${buildout:bin-directory}/runUnitTest run-unit-test = ${buildout:bin-directory}/runUnitTest
run-test-suite = ${buildout:bin-directory}/runTestSuite run-test-suite = ${buildout:bin-directory}/runTestSuite
ooo-paster = ${cloudooo-0:ooo-paster} ooo-paster = ${cloudooo-1:ooo-paster}
configuration-file = ${cloudooo-0:configuration-file} configuration-file = ${cloudooo-1:configuration-file}
run-unit-test-binary = {{ bin_directory }}/runCloudoooUnitTest run-unit-test-binary = {{ bin_directory }}/runCloudoooUnitTest
run-test-suite-binary = {{ bin_directory }}/runCloudoooTestSuite run-test-suite-binary = {{ bin_directory }}/runCloudoooTestSuite
......
...@@ -44,6 +44,7 @@ setup(name=name, ...@@ -44,6 +44,7 @@ setup(name=name,
'slapos.core', 'slapos.core',
'slapos.cookbook', 'slapos.cookbook',
'slapos.libnetworkcache', 'slapos.libnetworkcache',
'requests',
'six', 'six',
'PyPDF2', 'PyPDF2',
], ],
......
...@@ -26,23 +26,43 @@ ...@@ -26,23 +26,43 @@
# #
############################################################################## ##############################################################################
import codecs
import csv
import multiprocessing
import os import os
import json import json
import six.moves.xmlrpc_client as xmlrpclib import six.moves.xmlrpc_client as xmlrpclib
import six.moves.urllib.parse as urllib_parse
import ssl import ssl
import base64 import base64
import io import io
import requests
import PyPDF2 import PyPDF2
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, CloudOooTestCase = makeModuleSetUpAndTestCaseClass( setUpModule, _CloudOooTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath( os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))) os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
# Cloudooo needs a lot of time before being available.
CloudOooTestCase.instance_max_retry = 30 class CloudOooTestCase(_CloudOooTestCase):
# Cloudooo needs a lot of time before being available.
instance_max_retry = 30
def setUp(self):
self.url = json.loads(
self.computer_partition.getConnectionParameterDict()["_"])['cloudooo']
# XXX ignore certificate errors
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
self.server = xmlrpclib.ServerProxy(
self.url,
context=ssl_context,
allow_none=True,
)
def normalizeFontName(font_name): def normalizeFontName(font_name):
...@@ -88,19 +108,6 @@ class HTMLtoPDFConversionFontTestMixin: ...@@ -88,19 +108,6 @@ class HTMLtoPDFConversionFontTestMixin:
"""Convert the HTML source to pdf bytes. """Convert the HTML source to pdf bytes.
""" """
def setUp(self):
self.url = json.loads(
self.computer_partition.getConnectionParameterDict()["_"])['cloudooo']
# XXX ignore certificate errors
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
self.server = xmlrpclib.ServerProxy(
self.url,
context=ssl_context,
allow_none=True,
)
def test(self): def test(self):
actual_font_mapping_mapping = {} actual_font_mapping_mapping = {}
for font in self.expected_font_mapping: for font in self.expected_font_mapping:
...@@ -237,3 +244,84 @@ class TestLibreoffice(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase): ...@@ -237,3 +244,84 @@ class TestLibreoffice(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
'html', 'html',
'pdf', 'pdf',
).encode()) ).encode())
class TestLibreOfficeTextConversion(CloudOooTestCase):
__partition_reference__ = 'txt'
def test_html_to_text(self):
self.assertEqual(
base64.decodestring(
self.server.convertFile(
base64.encodestring(
u'<html>héhé</html>'.encode('utf-8')).decode(),
'html',
'txt',
).encode()),
codecs.BOM_UTF8 + b'h\xc3\xa9h\xc3\xa9\n',
)
class TestLibreOfficeCluster(CloudOooTestCase):
__partition_reference__ = 'lc'
@classmethod
def getInstanceParameterDict(cls):
return {'backend-count': 4}
def test_multiple_conversions(self):
# make this function global so that it can be picked and used by multiprocessing
global _convert_html_to_text
def _convert_html_to_text(src_html):
return base64.decodestring(
self.server.convertFile(
base64.encodestring(src_html.encode()).decode(),
'html',
'txt',
).encode())
pool = multiprocessing.Pool(5)
# TODO py3: use with pool
converted = pool.map(_convert_html_to_text,
['<html><body>hello</body></html>'] * 100)
pool.terminate()
pool.join()
self.assertEqual(converted, [codecs.BOM_UTF8 + b'hello\n'] * 100)
# haproxy stats are exposed
res = requests.get(
urllib_parse.urljoin(self.url, '/haproxy;csv'),
verify=False,
stream=True,
)
reader = csv.DictReader(res.raw)
line_list = list(reader)
# requests have been balanced
total_hrsp_2xx = {
line['svname']: int(line['hrsp_2xx'])
for line in line_list
}
self.assertEqual(total_hrsp_2xx['FRONTEND'], 100)
self.assertEqual(total_hrsp_2xx['BACKEND'], 100)
for backend in 'cloudooo_1', 'cloudooo_2', 'cloudooo_3', 'cloudooo_4':
# ideally there should be 25% of requests on each backend, because we use
# round robin scheduling, but it can happen that some backend take longer
# to start, so we are tolerant here and just check that each backend
# process at least 15% of requests.
self.assertGreater(total_hrsp_2xx[backend], 15)
# no errors
total_eresp = {
line['svname']: int(line['eresp'] or 0)
for line in line_list
}
self.assertEqual(
total_eresp, {
'FRONTEND': 0,
'cloudooo_1': 0,
'cloudooo_2': 0,
'cloudooo_3': 0,
'cloudooo_4': 0,
'BACKEND': 0,
})
...@@ -19,7 +19,7 @@ md5sum = 0d34ff81779115bf899f7bc752877b70 ...@@ -19,7 +19,7 @@ md5sum = 0d34ff81779115bf899f7bc752877b70
[template-kvm] [template-kvm]
filename = instance-kvm.cfg.jinja2 filename = instance-kvm.cfg.jinja2
md5sum = 854ec0f379a05246f5f3761fdb634c36 md5sum = bf0c01ac7493693bb57ebef00bb20fa0
[template-kvm-cluster] [template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in filename = instance-kvm-cluster.cfg.jinja2.in
...@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257 ...@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
[template-kvm-run] [template-kvm-run]
filename = template/template-kvm-run.in filename = template/template-kvm-run.in
md5sum = a8afeb2f80199368e291621576a80ca1 md5sum = b8cc7c76438212e0522ebede88649393
[template-kvm-controller] [template-kvm-controller]
filename = template/kvm-controller-run.in filename = template/kvm-controller-run.in
......
...@@ -359,7 +359,7 @@ ...@@ -359,7 +359,7 @@
}, },
"virtual-hard-drive-md5sum": { "virtual-hard-drive-md5sum": {
"title": "Checksum of virtual hard drive", "title": "Checksum of virtual hard drive",
"description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.", "description": "MD5 checksum of virtual hard drive, required if virtual-hard-drive-url is specified.",
"type": "string" "type": "string"
}, },
"virtual-hard-drive-gzipped": { "virtual-hard-drive-gzipped": {
......
...@@ -160,7 +160,7 @@ ...@@ -160,7 +160,7 @@
}, },
"virtual-hard-drive-md5sum": { "virtual-hard-drive-md5sum": {
"title": "Checksum of virtual hard drive", "title": "Checksum of virtual hard drive",
"description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.", "description": "MD5 checksum of virtual hard drive, required if virtual-hard-drive-url is specified.",
"type": "string" "type": "string"
}, },
"virtual-hard-drive-gzipped": { "virtual-hard-drive-gzipped": {
......
...@@ -17,6 +17,8 @@ ...@@ -17,6 +17,8 @@
{% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%} {% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
{% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%} {% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%}
{% set whitelist_domains = slapparameter_dict.get('whitelist-domains', '') -%} {% set whitelist_domains = slapparameter_dict.get('whitelist-domains', '') -%}
{% set virtual_hard_drive_url_enabled = 'virtual-hard-drive-url' in slapparameter_dict %}
{% set virtual_hard_drive_url_gzipped = str(slapparameter_dict.get('virtual-hard-drive-gzipped', False)).lower() == 'true' %}
{% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %} {% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %}
{% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %} {% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %}
{% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url') -%} {% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url') -%}
...@@ -56,6 +58,11 @@ public = ${:srv}/public/ ...@@ -56,6 +58,11 @@ public = ${:srv}/public/
cron-entries = ${:etc}/cron.d cron-entries = ${:etc}/cron.d
crontabs = ${:etc}/crontabs crontabs = ${:etc}/crontabs
cronstamps = ${:etc}/cronstamps cronstamps = ${:etc}/cronstamps
{%- if virtual_hard_drive_url_enabled %}
virtual-hard-drive-url-repository = ${:srv}/virtual-hard-drive-url-repository
virtual-hard-drive-url-var = ${:var}/virtual-hard-drive-url
virtual-hard-drive-url-expose = ${monitor-directory:private}/virtual-hard-drive-url
{%- endif %}
{%- if boot_image_url_list_enabled %} {%- if boot_image_url_list_enabled %}
boot-image-url-list-repository = ${:srv}/boot-image-url-list-repository boot-image-url-list-repository = ${:srv}/boot-image-url-list-repository
boot-image-url-list-var = ${:var}/boot-image-url-list boot-image-url-list-var = ${:var}/boot-image-url-list
...@@ -278,6 +285,106 @@ config-filename = ${boot-image-url-list-download-wrapper:error-state-file} ...@@ -278,6 +285,106 @@ config-filename = ${boot-image-url-list-download-wrapper:error-state-file}
## boot-image-url-list support END ## boot-image-url-list support END
{% endif %} {# if boot_image_url_list_enabled #} {% endif %} {# if boot_image_url_list_enabled #}
{% if virtual_hard_drive_url_enabled %}
## virtual-hard-drive-url support BEGIN
[empty-file-state-base-virtual-promise]
<= monitor-promise-base
module = check_file_state
name = ${:_buildout_section_name_}.py
config-state = empty
# It's very hard to put the username and password correctly, after schema://
# and before the host, as it's not the way how one can use monitor provided
# information, so just show the information in the URL
config-url = ${monitor-base:base-url}/private/virtual-hard-drive-url/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
[virtual-hard-drive-url-source-config]
recipe = slapos.recipe.template:jinja2
template = inline:
{%- raw %}
{{ virtual_hard_drive_url }}
{% endraw -%}
{# Enforce md5sum on virtual-hard-drive-url #}
virtual-hard-drive-url = {{ slapparameter_dict['virtual-hard-drive-url'] }}#{{ slapparameter_dict['virtual-hard-drive-md5sum'] }}
context =
key virtual_hard_drive_url :virtual-hard-drive-url
rendered = ${directory:etc}/virtual-hard-drive-url.conf
[virtual-hard-drive-url-processed-config]
# compares if the current configuration has been used by
# the virtual-hard-drive-url-download, if not, exposes it as not empty file with
# information
recipe = slapos.recipe.build
install =
import os
import hashlib
if not os.path.exists(location):
os.mkdir(location)
with open('${:state-file}', 'w') as state_handler:
try:
with open('${:config-file}', 'rb') as config_handler, open('${:processed-md5sum}') as processed_handler:
config_md5sum = hashlib.md5(config_handler.read()).hexdigest()
processed_md5sum = processed_handler.read()
if config_md5sum == processed_md5sum:
state_handler.write('')
else:
state_handler.write('config %s != processed %s' % (config_md5sum, processed_md5sum))
except Exception as e:
state_handler.write(str(e))
update = ${:install}
config-file = ${virtual-hard-drive-url-source-config:rendered}
state-filename = virtual-hard-drive-url-processed-config.state
state-file = ${directory:virtual-hard-drive-url-expose}/${:state-filename}
processed-md5sum = ${directory:virtual-hard-drive-url-var}/update-image-processed.md5sum
[virtual-hard-drive-url-processed-config-promise]
# promise to check if the configuration provided by the user has been already
# processed by the virtual-hard-drive-url-download script, which runs asynchronously
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-processed-config:state-filename}
config-filename = ${virtual-hard-drive-url-processed-config:state-file}
[virtual-hard-drive-url-json-config]
# generates json configuration from user configuration
recipe = plone.recipe.command
command = {{ python_executable }} {{ image_download_config_creator }} ${virtual-hard-drive-url-source-config:rendered} ${:rendered} ${directory:virtual-hard-drive-url-repository} ${:error-state-file}
update-command = ${:command}
rendered = ${directory:virtual-hard-drive-url-var}/virtual-hard-drive-url.json
error-state-filename = virtual-hard-drive-url-json-config-error.txt
error-state-file = ${directory:virtual-hard-drive-url-expose}/${:error-state-filename}
[virtual-hard-drive-url-config-state-promise]
# promise to check if configuration has been parsed without errors
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-json-config:error-state-filename}
config-filename = ${virtual-hard-drive-url-json-config:error-state-file}
[virtual-hard-drive-url-download-wrapper]
# wrapper to execute virtual-hard-drive-url-download on each run
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:scripts}/virtual-hard-drive-url-updater
command-line = {{ python_executable }} {{ image_download_controller }} ${virtual-hard-drive-url-json-config:rendered} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${virtual-hard-drive-url-processed-config:processed-md5sum}
md5sum-state-filename = virtual-hard-drive-url-download-controller-md5sum-fail.json
md5sum-state-file = ${directory:virtual-hard-drive-url-expose}/${:md5sum-state-filename}
error-state-filename = virtual-hard-drive-url-download-controller-error.text
error-state-file = ${directory:virtual-hard-drive-url-expose}/${:error-state-filename}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[virtual-hard-drive-url-download-md5sum-promise]
# promise to report errors with problems with calculating md5sum of the
# downloaded images
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-download-wrapper:md5sum-state-filename}
config-filename = ${virtual-hard-drive-url-download-wrapper:md5sum-state-file}
[virtual-hard-drive-url-download-state-promise]
# promise to report errors during download
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-download-wrapper:error-state-filename}
config-filename = ${virtual-hard-drive-url-download-wrapper:error-state-file}
## virtual-hard-drive-url support END
{% endif %} {# if virtual_hard_drive_url_enabled #}
[kvm-controller-parameter-dict] [kvm-controller-parameter-dict]
python-path = {{ python_eggs_executable }} python-path = {{ python_eggs_executable }}
vnc-passwd = ${gen-passwd:passwd} vnc-passwd = ${gen-passwd:passwd}
...@@ -298,6 +405,11 @@ vnc-ip = ${:ipv4} ...@@ -298,6 +405,11 @@ vnc-ip = ${:ipv4}
vnc-port = 5901 vnc-port = 5901
default-cdrom-iso = {{ debian_amd64_netinst_location }} default-cdrom-iso = {{ debian_amd64_netinst_location }}
{% if virtual_hard_drive_url_enabled %}
virtual-hard-drive-url-json-config = ${virtual-hard-drive-url-json-config:rendered}
{% else %}
virtual-hard-drive-url-json-config =
{% endif %}
{% if boot_image_url_list_enabled %} {% if boot_image_url_list_enabled %}
boot-image-url-list-json-config = ${boot-image-url-list-json-config:rendered} boot-image-url-list-json-config = ${boot-image-url-list-json-config:rendered}
{% else %} {% else %}
...@@ -425,14 +537,42 @@ ipv6-port = {{ external_port }} ...@@ -425,14 +537,42 @@ ipv6-port = {{ external_port }}
{% endfor -%} {% endfor -%}
{% endif -%} {% endif -%}
{%- set depend_section_list = [] %}
{%- set hash_file_list = ['${kvm-run:rendered}'] %}
{%- macro generate_depend_section(section, key) %}
{%- do depend_section_list.append('${' + section + ':command}' ) %}
{%- do hash_file_list.append('${' + key + '}') %}
[{{ section }}]
recipe = plone.recipe.command
update-command = ${:command}
command = [ ! -f {{ '${' + key + '}' }} ] && touch {{ '${' + key + '}' }}
{%- endmacro %}
{#- Create depending sections, as state files appear late, so it's better to have empty file which will impact the hash anyway #}
{%- if boot_image_url_list_enabled %}
{{ generate_depend_section('boot-image-url-list-depend', 'boot-image-url-list-download-wrapper:md5sum-state-file') }}
{%- endif %}
{%- if boot_image_url_select_enabled %}
{{ generate_depend_section('boot-image-url-select-depend', 'boot-image-url-select-download-wrapper:md5sum-state-file') }}
{%- endif %}
{%- if virtual_hard_drive_url_enabled %}
{{ generate_depend_section('virtual-hard-drive-url-depend', 'virtual-hard-drive-url-download-wrapper:md5sum-state-file') }}
{%- endif %}
[kvm-instance] [kvm-instance]
depends =
{%- for depend_section in depend_section_list %}
{{ depend_section }}
{%- endfor %}
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
socket-path = ${kvm-controller-parameter-dict:socket-path} socket-path = ${kvm-controller-parameter-dict:socket-path}
wrapper-path = ${directory:services}/kvm wrapper-path = ${directory:services}/kvm
command-line = ${kvm-run:rendered} command-line = ${kvm-run:rendered}
kvm-controller = ${kvm-controller-wrapper:wrapper-path} kvm-controller = ${kvm-controller-wrapper:wrapper-path}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
hash-files =
{%- for hash_file in hash_file_list %}
{{ hash_file }}
{%- endfor %}
[kvm-controller-wrapper] [kvm-controller-wrapper]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
...@@ -932,8 +1072,8 @@ disk-format = qcow2 ...@@ -932,8 +1072,8 @@ disk-format = qcow2
disk-device-path = disk-device-path =
cpu-count = 1 cpu-count = 1
disk-cache = writeback disk-cache =
disk-aio = native disk-aio =
auto-ballooning = True auto-ballooning = True
machine-options = machine-options =
cpu-model = host cpu-model = host
...@@ -1108,6 +1248,13 @@ parts = ...@@ -1108,6 +1248,13 @@ parts =
cron-service cron-service
cron-entry-logrotate cron-entry-logrotate
frontend-promise frontend-promise
{% if virtual_hard_drive_url_enabled %}
virtual-hard-drive-url-download-wrapper
virtual-hard-drive-url-config-state-promise
virtual-hard-drive-url-download-md5sum-promise
virtual-hard-drive-url-download-state-promise
virtual-hard-drive-url-processed-config-promise
{% endif %}
{% if boot_image_url_list_enabled %} {% if boot_image_url_list_enabled %}
boot-image-url-list-download-wrapper boot-image-url-list-download-wrapper
boot-image-url-list-config-state-promise boot-image-url-list-config-state-promise
......
...@@ -6,10 +6,6 @@ import hashlib ...@@ -6,10 +6,6 @@ import hashlib
import os import os
import socket import socket
import subprocess import subprocess
try:
from urllib.request import FancyURLopener
except ImportError:
from urllib import FancyURLopener
import gzip import gzip
import shutil import shutil
from random import shuffle from random import shuffle
...@@ -17,16 +13,11 @@ import glob ...@@ -17,16 +13,11 @@ import glob
import re import re
import json import json
import ssl
# XXX: give all of this through parameter, don't use this as template, but as module # XXX: give all of this through parameter, don't use this as template, but as module
qemu_img_path = '{{ parameter_dict.get("qemu-img-path") }}' qemu_img_path = {{ repr(parameter_dict["qemu-img-path"]) }}
qemu_path = '{{ parameter_dict.get("qemu-path") }}' qemu_path = {{ repr(parameter_dict["qemu-path"]) }}
disk_size = '{{ parameter_dict.get("disk-size") }}' disk_size = {{ repr(parameter_dict["disk-size"]) }}
disk_type = '{{ parameter_dict.get("disk-type") }}' disk_type = {{ repr(parameter_dict["disk-type"]) }}
disk_format = '{{ parameter_dict.get("disk-format", "qcow2")}}'
disk_format = disk_format \
if disk_format in ['qcow2', 'raw', 'vdi', 'vmdk', 'cloop', 'qed'] else 'qcow2'
socket_path = '{{ parameter_dict.get("socket-path") }}' socket_path = '{{ parameter_dict.get("socket-path") }}'
nbd_list = (('{{ parameter_dict.get("nbd-host") }}', nbd_list = (('{{ parameter_dict.get("nbd-host") }}',
...@@ -34,11 +25,7 @@ nbd_list = (('{{ parameter_dict.get("nbd-host") }}', ...@@ -34,11 +25,7 @@ nbd_list = (('{{ parameter_dict.get("nbd-host") }}',
('{{ parameter_dict.get("nbd2-host") }}', ('{{ parameter_dict.get("nbd2-host") }}',
{{ parameter_dict.get("nbd2-port") }})) {{ parameter_dict.get("nbd2-port") }}))
default_cdrom_iso = '{{ parameter_dict.get("default-cdrom-iso") }}' default_cdrom_iso = '{{ parameter_dict.get("default-cdrom-iso") }}'
disk_path = '{{ parameter_dict.get("disk-path") }}'
virtual_hard_drive_url = '{{ parameter_dict.get("virtual-hard-drive-url") }}'.strip()
virtual_hard_drive_md5sum = '{{ parameter_dict.get("virtual-hard-drive-md5sum") }}'.strip()
virtual_hard_drive_gzipped = '{{ parameter_dict.get("virtual-hard-drive-gzipped") }}'.strip().lower()
nat_rules = '{{ parameter_dict.get("nat-rules") }}'.strip() nat_rules = '{{ parameter_dict.get("nat-rules") }}'.strip()
use_tap = '{{ parameter_dict.get("use-tap") }}'.lower() use_tap = '{{ parameter_dict.get("use-tap") }}'.lower()
use_nat = '{{ parameter_dict.get("use-nat") }}'.lower() use_nat = '{{ parameter_dict.get("use-nat") }}'.lower()
...@@ -56,9 +43,7 @@ init_ram_size = {{ parameter_dict.get("init-ram-size") }} ...@@ -56,9 +43,7 @@ init_ram_size = {{ parameter_dict.get("init-ram-size") }}
pid_file_path = '{{ parameter_dict.get("pid-file-path") }}' pid_file_path = '{{ parameter_dict.get("pid-file-path") }}'
external_disk_number = {{ parameter_dict.get("external-disk-number") }} external_disk_number = {{ parameter_dict.get("external-disk-number") }}
external_disk_size = {{ parameter_dict.get("external-disk-size") }} external_disk_size = {{ parameter_dict.get("external-disk-size") }}
external_disk_format = '{{ parameter_dict.get("external-disk-format", "qcow2") }}' external_disk_format = {{ repr(parameter_dict["external-disk-format"]) }}
external_disk_format = external_disk_format \
if external_disk_format in ['qcow2', 'raw', 'vdi', 'vmdk', 'cloop', 'qed'] else 'qcow2'
disk_storage_dict = {} disk_storage_dict = {}
disk_storage_list = """{{ parameter_dict.get("disk-storage-list") }}""".split('\n') disk_storage_list = """{{ parameter_dict.get("disk-storage-list") }}""".split('\n')
map_storage_list = [] map_storage_list = []
...@@ -70,11 +55,6 @@ cluster_doc_port = {{ parameter_dict.get("cluster-doc-port") }} ...@@ -70,11 +55,6 @@ cluster_doc_port = {{ parameter_dict.get("cluster-doc-port") }}
url_check_certificate = '{{ parameter_dict.get("hard-drive-url-check-certificate", "true") }}'.lower() url_check_certificate = '{{ parameter_dict.get("hard-drive-url-check-certificate", "true") }}'.lower()
auto_ballooning = '{{ parameter_dict.get("auto-ballooning") }}' in ('true', 'True', '1') auto_ballooning = '{{ parameter_dict.get("auto-ballooning") }}' in ('true', 'True', '1')
vm_name = '{{ parameter_dict.get("name") }}' vm_name = '{{ parameter_dict.get("name") }}'
disk_cache = '{{ parameter_dict.get("disk-cache", "writeback") }}'.strip()
disk_cache = disk_cache if disk_cache in ["none", "writeback", "unsafe",
"directsync", "writethrough"] else "writeback"
disk_aio = '{{ parameter_dict.get("disk-aio", "threads") }}'.strip()
disk_aio = disk_aio if disk_aio in ["threads", "native"] else "threads"
# If a device (ie.: /dev/sdb) is provided, use it instead # If a device (ie.: /dev/sdb) is provided, use it instead
# the disk_path with disk_format # the disk_path with disk_format
...@@ -84,16 +64,18 @@ for disk_device_path in '{{ parameter_dict.get("disk-device-path", "") }}'.split ...@@ -84,16 +64,18 @@ for disk_device_path in '{{ parameter_dict.get("disk-device-path", "") }}'.split
disk_info_list.append({ disk_info_list.append({
'path': disk_device_path, 'path': disk_device_path,
'format': "raw", 'format': "raw",
'io': "native", 'aio': "native",
'cache': "none" 'cache': "none"
}) })
if len(disk_info_list) == 0: if not disk_info_list:
disk_info_list.append({ disk_info_list.append({
'path': disk_path, {%- for k in 'path', 'format', 'aio', 'cache' %}
'format': disk_format, {%- set v = parameter_dict['disk-' + k] %}
'io': disk_aio, {%- if v %}
'cache': disk_cache, {{ repr(k) }}: {{ repr(v) }},
{%- endif %}
{%- endfor %}
}) })
smp_count = {{ parameter_dict.get("smp-count") }} smp_count = {{ parameter_dict.get("smp-count") }}
...@@ -107,11 +89,8 @@ logfile = '{{ parameter_dict.get("log-file") }}' ...@@ -107,11 +89,8 @@ logfile = '{{ parameter_dict.get("log-file") }}'
boot_image_url_list_json_config = '{{ parameter_dict.get("boot-image-url-list-json-config") }}' boot_image_url_list_json_config = '{{ parameter_dict.get("boot-image-url-list-json-config") }}'
boot_image_url_select_json_config = '{{ parameter_dict.get("boot-image-url-select-json-config") }}' boot_image_url_select_json_config = '{{ parameter_dict.get("boot-image-url-select-json-config") }}'
virtual_hard_drive_url_json_config = '{{ parameter_dict.get("virtual-hard-drive-url-json-config") }}'
if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false': virtual_hard_drive_gzipped = '{{ parameter_dict.get("virtual-hard-drive-gzipped") }}'.strip().lower()
opener = FancyURLopener(context=ssl._create_unverified_context())
else:
opener = FancyURLopener({})
def md5Checksum(file_path): def md5Checksum(file_path):
with open(file_path, 'rb') as fh: with open(file_path, 'rb') as fh:
...@@ -172,39 +151,34 @@ def getMapStorageList(disk_storage_dict, external_disk_number): ...@@ -172,39 +151,34 @@ def getMapStorageList(disk_storage_dict, external_disk_number):
lf.write('%s' % external_disk_number) lf.write('%s' % external_disk_number)
return id_list, external_disk_number return id_list, external_disk_number
# Download existing hard drive if needed at first boot # Use downloaded virtual-hard-drive-url
if len(disk_info_list) == 1 and not os.path.exists(disk_info_list[0]['path']) and virtual_hard_drive_url != '': if len(disk_info_list) == 1 and not os.path.exists(disk_info_list[0]['path']) and virtual_hard_drive_url_json_config != '':
print('Downloading virtual hard drive...') print('Using virtual hard drive...')
try: with open(virtual_hard_drive_url_json_config) as fh:
downloaded_disk = disk_info_list[0]['path'] image_config = json.load(fh)
if image_config['error-amount'] == 0:
image = image_config['image-list'][0]
downloaded_image = os.path.join(image_config['destination-directory'], image['destination'])
# previous version was using disk in place, but here it would result with
# redownload, so copy it
if virtual_hard_drive_gzipped == 'true': if virtual_hard_drive_gzipped == 'true':
downloaded_disk = '%s.gz' % disk_info_list[0]['path'] try:
opener.retrieve(virtual_hard_drive_url, downloaded_disk) with open(disk_info_list[0]['path'], 'wb') as d_fh:
except: with gzip.open(downloaded_image, 'rb') as s_fh:
if os.path.exists(downloaded_disk): shutil.copyfileobj(s_fh, d_fh)
os.remove(downloaded_disk) except Exception:
raise if os.path.exists(disk_info_list[0]['path']):
md5sum = virtual_hard_drive_md5sum.strip() os.unlink(disk_info_list[0]['path'])
if md5sum: raise
print('Checking MD5 checksum...') else:
local_md5sum = md5Checksum(downloaded_disk) try:
if local_md5sum != md5sum: shutil.copyfile(downloaded_image, disk_info_list[0]['path'])
os.remove(downloaded_disk) except Exception:
raise Exception('MD5 mismatch. MD5 of local file is %s, Specified MD5 is %s.' % ( if os.path.exists(disk_info_list[0]['path']):
local_md5sum, md5sum)) os.unlink(disk_info_list[0]['path'])
print('MD5sum check passed.') raise
else: else:
print('Warning: not checksum specified.') raise ValueError('virtual-hard-drive-url not ready yet')
if downloaded_disk.endswith('.gz'):
try:
with open(disk_info_list[0]['path'], 'w') as disk:
with gzip.open(downloaded_disk, 'rb') as disk_gz:
shutil.copyfileobj(disk_gz, disk)
except Exception:
if os.path.exists(disk_info_list[0]['path']):
os.remove(disk_info_list[0]['path'])
raise
os.remove(downloaded_disk)
# Create disk if doesn't exist # Create disk if doesn't exist
# XXX: move to Buildout profile # XXX: move to Buildout profile
...@@ -310,17 +284,12 @@ kvm_argument_list = [qemu_path, ...@@ -310,17 +284,12 @@ kvm_argument_list = [qemu_path,
'-nodefaults', '-nodefaults',
] ]
for disk_info in disk_info_list: for disk_info in disk_info_list:
additional_disk_options = '' kvm_argument_list += (
if disk_info['io'] == 'native':
additional_disk_options += ',cache.direct=on'
if disk_info['format'] == "raw":
additional_disk_options += ',discard=on'
kvm_argument_list.extend([
'-drive', '-drive',
'file=%s,if=%s,cache=%s,aio=%s%s' % ( disk_info['path'], disk_type, disk_info['cache'], disk_info['io'], additional_disk_options) 'file=%s,if=%s,discard=on%s' % (
]) disk_info['path'], disk_type,
''.join(',%s=%s' % x for x in disk_info.items() if x[0] != 'path'))
)
rgx = re.compile('^[\w*\,][\=\d+\-\,\w]*$') rgx = re.compile('^[\w*\,][\=\d+\-\,\w]*$')
for numa in numa_list: for numa in numa_list:
...@@ -392,6 +361,9 @@ else: ...@@ -392,6 +361,9 @@ else:
'-drive', '-drive',
'file=%s,media=cdrom' % (link,) 'file=%s,media=cdrom' % (link,)
]) ])
else:
raise ValueError('boot-image-url-select not ready yet')
if boot_image_url_list_json_config: if boot_image_url_list_json_config:
# Support boot-image-url-list # Support boot-image-url-list
with open(boot_image_url_list_json_config) as fh: with open(boot_image_url_list_json_config) as fh:
...@@ -404,6 +376,8 @@ else: ...@@ -404,6 +376,8 @@ else:
'-drive', '-drive',
'file=%s,media=cdrom' % (link,) 'file=%s,media=cdrom' % (link,)
]) ])
else:
raise ValueError('boot-image-url-list not ready yet')
# Always add by default the default image # Always add by default the default image
kvm_argument_list.extend([ kvm_argument_list.extend([
'-drive', 'file=%s,media=cdrom' % default_cdrom_iso '-drive', 'file=%s,media=cdrom' % default_cdrom_iso
......
This diff is collapsed.
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[template] [template]
filename = instance.cfg filename = instance.cfg
md5sum = 9ddae686379e8d747410c1adf82b47d6 md5sum = c115ed9d4ff0f785d79cdcacbb0bd1ad
[template-monitor] [template-monitor]
_update_hash_filename_ = instance-monitor.cfg.jinja2 _update_hash_filename_ = instance-monitor.cfg.jinja2
...@@ -26,11 +26,11 @@ md5sum = 2eb5596544d9c341acf653d4f7ce2680 ...@@ -26,11 +26,11 @@ md5sum = 2eb5596544d9c341acf653d4f7ce2680
[template-monitor-edgetest] [template-monitor-edgetest]
_update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2 _update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2
md5sum = a57106ee88ff3295b9ffce84105da79b md5sum = 3c8ab4e78f66c974eb95afc595a13514
[template-monitor-edgebot] [template-monitor-edgebot]
_update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2 _update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
md5sum = c1885a42aadd45bab3185a53258d4ff4 md5sum = 365a6cc6831267a73fa5ebd56ad394ee
[network-bench-cfg] [network-bench-cfg]
filename = network_bench.cfg.in filename = network_bench.cfg.in
...@@ -42,4 +42,4 @@ md5sum = d3cfa1f6760e3fa64ccd64acf213bdfb ...@@ -42,4 +42,4 @@ md5sum = d3cfa1f6760e3fa64ccd64acf213bdfb
[template-surykatka-ini] [template-surykatka-ini]
_update_hash_filename_ = surykatka.ini.jinja2 _update_hash_filename_ = surykatka.ini.jinja2
md5sum = 89545501f0e5bf11608978886429da3d md5sum = 609c6cca763b73a80fa05ee56475eb20
{ {
"$schema": "http://json-schema.org/draft-04/schema#", "$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": { "properties": {
"nameserver": { "region-dict": {
"default": "", "title": "Regions",
"title": "Nameserver", "description": "Defines regions of the cluster",
"description": "Space separated list of name servers to use.", "patternProperties": {
"type": "string" ".*": {
}, "properties": {
"check-status-code": { "state": {
"default": "200", "title": "State",
"title": "Default Check HTTP Code", "description": "State of the node of the region. Can be used to destroy not needed regions.",
"description": "Default HTTP code to check against (default: 200).", "type": "string",
"type": "string" "default": "started",
}, "enum": [
"check-http-header-dict": { "started",
"default": "{}", "stopped",
"title": "HTTP header dict to check", "destroyed"
"description": "JSON dict of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}", ]
},
"sla-computer_guid": {
"title": "GUID of the computer on which this region shall be deployed",
"description": "Unique identifier of the computer, like \"COMP-1234\". By default, let Master choose a computer.",
"type": "string",
"default": ""
},
"nameserver-list": {
"default": [],
"title": "Nameservers",
"description": "List of nameservers to use.",
"type": "array"
},
"check-frontend-ip-list": {
"default": [],
"title": "Default Frontend IPs to check",
"description": "List of default frontend IPs to check, if empty no constraint is used.",
"type": "array"
}
},
"type": "object"
}
},
"type": "object" "type": "object"
}, },
"check-frontend-ip": { "nameserver-list": {
"default": "", "default": [],
"title": "Default space separated list of Frontend IPs to check", "title": "Nameservers (backward compatibility)",
"description": "Default list of Frontend IPs to check, if empty no constraint is used.", "description": "List of nameservers to use. Note: This is backward compatibility, use region-dict for full configuration control.",
"type": "string" "type": "array"
},
"check-certificate-expiration-days": {
"default": "15",
"title": "Default certificate expiration days check",
"description": "Default amount of days to consider certificate as being to-be-expired (default: 15).",
"type": "string"
},
"check-maximum-elapsed-time": {
"default": "2",
"title": "Default maximum elapsed time for a site to reply (seconds)",
"description": "Default maximum elapsed time for a site to reply to be considered good (default: 2s).",
"type": "string"
}, },
"failure-amount": { "check-frontend-ip-list": {
"default": "1", "default": [],
"title": "Default amount of failures to consider URL as in bad state", "title": "Default Frontend IPs to check (backward compatibility)",
"description": "Default amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages (default: 1).", "description": "List of default frontend IPs to check, if empty no constraint is used. Note: This is backward compatibility, use region-dict for full configuration control.",
"type": "string" "type": "array"
} }
} }
} }
{ {
"$schema": "http://json-schema.org/draft-04/schema#", "$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": { "properties": {
"url": { "url": {
"title": "URL to check", "title": "URL to check",
"description": "URL to check, like https://example.com", "description": "URL to check, like https://example.com/",
"type": "string" "type": "string"
}, },
"check-status-code": { "region-dict": {
"default": "Master default", "title": "Applicable Regions",
"title": "Default Check HTTP Code.", "description": "Puts the check on the defined regions. No definition will result with presence in all regions.",
"description": "HTTP code to check against (default: comes from master partition).", "patternProperties": {
"type": "string" ".*": {
}, "properties": {
"check-http-header-dict": { "state": {
"default": "Master default", "title": "State",
"title": "HTTP header dict to check", "description": "State of the check of the region. Used only to make it correctly visible in the SlapOS Master UI if no other parameters are defined.",
"description": "JSON dict of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}", "type": "string",
"type": "object" "default": "present",
"enum": [
"present"
]
},
"check-frontend-ip-list": {
"default": [],
"title": "Frontend IPs to check",
"description": "List of default frontend IPs to check, if empty no constraint is used. Defaults to region configuration.",
"type": "array"
}
},
"type": "object"
}
},
"type": "object",
"default": {}
}, },
"check-frontend-ip": { "check-status-code": {
"default": "Master default", "title": "HTTP Code Check",
"title": "Space separated list of Frontend IPs to check", "description": "Expected response HTTP Code.",
"description": "List of Frontend IPs to check, if empty no constraint is used (default: comes from master partition).", "type": "number",
"type": "string" "default": 200,
"minimum": 100,
"maximum": 599
}, },
"check-certificate-expiration-days": { "check-certificate-expiration-days": {
"default": "Master default", "title": "Certificate Expiration Check (days)",
"title": "Certificate expiration days check", "description": "Amount of days to consider certificate as being to-be-expired.",
"description": "Amount of days to consider certificate as being to-be-expired (default: comes from master partition).", "type": "number",
"type": "string" "default": 15,
"minimum": 1
}, },
"check-maximum-elapsed-time": { "check-maximum-elapsed-time": {
"default": "Master default", "title": "Maximum Elapsed Check (seconds)",
"title": "Maximum elapsed time for a site to reply (seconds)", "description": "Maximum elapsed time for a site to reply to be considered good.",
"description": "Maximum elapsed time for a site to reply to be considered good.(default: comes from master partition).", "type": "number",
"type": "string" "default": 2,
"minimum": 1
},
"check-http-header-dict": {
"title": "HTTP Header Check",
"description": "JSON object of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}. Note: Shall be expressed directly as object, without any additional qouting.",
"type": "object",
"default": {}
}, },
"failure-amount": { "failure-amount": {
"default": "Master default", "title": "Failure Amount",
"title": "Amount of failures to consider URL as in bad state", "description": "Amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages.",
"description": "Amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages (default: comes from master partition).", "type": "number",
"type": "string" "default": 2,
"minimum": 1
},
"check-frontend-ip-list": {
"title": "Frontend IPs to check (backward compatibility)",
"description": "List of Frontend IPs to check, if empty no constraint is used. Defaults to region configuration. Note: Use region-dict's check-frontend-ip-list to ensure specific check on each region.",
"type": "array"
} }
} }
} }
...@@ -9,39 +9,27 @@ ...@@ -9,39 +9,27 @@
{%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %} {%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %}
{%- if extra_slave_instance_list %} {%- if extra_slave_instance_list %}
{#- Create slaves to process with setting up defaults #} {#- Create slaves to process with setting up defaults #}
{%- for slave in json_module.loads(extra_slave_instance_list) | sort(attribute='slave_title') %} {%- for slave in extra_slave_instance_list | sort(attribute='-slave-title') %}
{%- if 'check-status-code' not in slave %} {%- do slave.setdefault('check-status-code', 200) %}
{%- do slave.__setitem__('check-status-code', CONFIGURATION['check-status-code']) %} {%- do slave.setdefault('check-http-header-dict', {}) %}
{%- endif %} {%- do slave.setdefault('check-certificate-expiration-days', 15) %}
{%- if 'check-http-header-dict' not in slave %} {%- do slave.setdefault('failure-amount', 2) %}
{%- do slave.__setitem__('check-http-header-dict', CONFIGURATION['check-http-header-dict']) %} {%- do slave.setdefault('check-maximum-elapsed-time', 2) %}
{%- endif %} {%- do slave.setdefault('check-frontend-ip-list', CONFIGURATION['check-frontend-ip-list']) %}
{%- if 'check-certificate-expiration-days' not in slave %} {%- if 'url' in slave %}
{%- do slave.__setitem__('check-certificate-expiration-days', CONFIGURATION['check-certificate-expiration-days']) %} {%- set class = slave['check-maximum-elapsed-time'] %}
{%- endif %} {%- if class not in slave_instance_dict %}
{%- if 'failure-amount' not in slave %} {%- do slave_instance_dict.__setitem__(class, []) %}
{%- do slave.__setitem__('failure-amount', CONFIGURATION['failure-amount']) %} {%- endif %}
{%- endif %} {%- do slave_instance_dict[class].append(slave) %}
{%- if 'check-maximum-elapsed-time' not in slave %}
{%- do slave.__setitem__('check-maximum-elapsed-time', CONFIGURATION['check-maximum-elapsed-time']) %}
{%- endif %}
{%- if 'check-frontend-ip' not in slave %}
{%- do slave.__setitem__('check-frontend-ip', CONFIGURATION['check-frontend-ip']) %}
{%- endif %}
{%- if 'url' in slave %}
{%- set class = slave['check-maximum-elapsed-time'] %}
{%- if class not in slave_instance_dict %}
{%- do slave_instance_dict.__setitem__(class, []) %}
{%- endif %} {%- endif %}
{%- do slave_instance_dict[class].append(slave) %}
{%- endif %}
{%- endfor %} {%- endfor %}
{%- endif %} {%- endif %}
{%- set part_list = [] %} {%- set part_list = [] %}
{%- for class, slave_instance_list in slave_instance_dict.items() %} {%- for class, slave_instance_list in slave_instance_dict.items() %}
{#- class is used to separate surykatka with different timeouts #} {#- class is used to separate surykatka with different timeouts #}
{%- for slave in slave_instance_list | sort(attribute='slave_title') %} {%- for slave in slave_instance_list | sort(attribute='-slave-title') %}
{%- set part_id = 'http-query-' ~ slave['slave_reference'] ~ '-promise' %} {%- set part_id = 'http-query-' ~ hashlib_module.md5(slave['-slave-reference'].encode('utf-8')).hexdigest() ~ '-promise' %}
{%- do part_list.append(part_id) %} {%- do part_list.append(part_id) %}
{%- set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %} {%- set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}
[{{part_id}}] [{{part_id}}]
...@@ -51,11 +39,11 @@ name = {{ safe_name }}.py ...@@ -51,11 +39,11 @@ name = {{ safe_name }}.py
config-report = http_query config-report = http_query
config-url = {{ slave['url'] }} config-url = {{ slave['url'] }}
config-status-code = {{ slave['check-status-code'] }} config-status-code = {{ slave['check-status-code'] }}
config-http-header-dict = {{ slave['check-http-header-dict'] }} config-http-header-dict = {{ json_module.dumps(slave['check-http-header-dict']) }}
config-certificate-expiration-days = {{ slave['check-certificate-expiration-days'] }} config-certificate-expiration-days = {{ slave['check-certificate-expiration-days'] }}
config-failure-amount = {{ slave['failure-amount'] }} config-failure-amount = {{ slave['failure-amount'] }}
config-maximum-elapsed-time = {{ slave['check-maximum-elapsed-time'] }} config-maximum-elapsed-time = {{ slave['check-maximum-elapsed-time'] }}
config-ip-list = {{ slave['check-frontend-ip'] }} config-ip-list = {{ ' '.join(slave['check-frontend-ip-list']) }}
config-json-file = ${surykatka-config-{{ class }}:json} config-json-file = ${surykatka-config-{{ class }}:json}
{%- endfor %} {%- endfor %}
...@@ -72,7 +60,7 @@ db = ${directory:srv}/surykatka-{{ class }}.db ...@@ -72,7 +60,7 @@ db = ${directory:srv}/surykatka-{{ class }}.db
rendered = ${directory:etc}/surykatka-{{ class }}.ini rendered = ${directory:etc}/surykatka-{{ class }}.ini
template = {{ template_surykatka_ini }} template = {{ template_surykatka_ini }}
slave_instance_list = {{ dumps(slave_instance_list) }} slave_instance_list = {{ dumps(slave_instance_list) }}
nameserver = {{ dumps(CONFIGURATION['nameserver']) }} nameserver_list = {{ dumps(CONFIGURATION['nameserver-list']) }}
json = ${directory:srv}/surykatka-{{ class }}.json json = ${directory:srv}/surykatka-{{ class }}.json
{#- timeout is just a bit bigger than class time #} {#- timeout is just a bit bigger than class time #}
timeout = {{ int(class) + 2 }} timeout = {{ int(class) + 2 }}
...@@ -80,7 +68,7 @@ timeout = {{ int(class) + 2 }} ...@@ -80,7 +68,7 @@ timeout = {{ int(class) + 2 }}
context = context =
import json_module json import json_module json
key db :db key db :db
key nameserver :nameserver key nameserver_list :nameserver_list
key slave_instance_list :slave_instance_list key slave_instance_list :slave_instance_list
key timeout :timeout key timeout :timeout
...@@ -110,9 +98,9 @@ cron-entries = ${directory:etc}/cron.d ...@@ -110,9 +98,9 @@ cron-entries = ${directory:etc}/cron.d
name = surykatka-status-{{ class }} name = surykatka-status-{{ class }}
frequency = */2 * * * * frequency = */2 * * * *
command = ${surykatka-status-json-{{ class }}:rendered} command = ${surykatka-status-json-{{ class }}:rendered}
{%- do part_list.append('surykatka-' + class) %} {%- do part_list.append('surykatka-%i'% (class,)) %}
{%- do part_list.append('surykatka-bot-promise-' + class) %} {%- do part_list.append('surykatka-bot-promise-%i' % (class,)) %}
{%- do part_list.append('cron-entry-surykatka-status-' + class) %} {%- do part_list.append('cron-entry-surykatka-status-%i' % (class,)) %}
{%- endfor %} {%- endfor %}
[buildout] [buildout]
...@@ -163,4 +151,4 @@ key = ${slap-connection:key-file} ...@@ -163,4 +151,4 @@ key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file} cert = ${slap-connection:cert-file}
[slap-parameter] [slap-parameter]
{%- endif %} {%- endif %} {#- if slap_software_type == software_type #}
...@@ -16,34 +16,86 @@ extensions = jinja2.ext.do ...@@ -16,34 +16,86 @@ extensions = jinja2.ext.do
extra-context = extra-context =
section slave_information slap-configuration section slave_information slap-configuration
{% set part_list = [] -%} {% set part_list = [] -%}
# Publish information for each slave
{%- set edgebot_software_type = 'edgebot' %} {%- set edgebot_software_type = 'edgebot' %}
{%- set edgebot_quantity = slapparameter_dict.pop('edgebot-quantity', '1') | int %}
{%- set edgebot_list = [] %}
{%- set edgebot_section_list = [] %}
{%- set slave_list_name = 'extra_slave_instance_list' %} {%- set slave_list_name = 'extra_slave_instance_list' %}
{%- set request_dict = {} %} {%- set request_dict = {} %}
{%- set namebase = "edgebot" %} {%- set namebase = "edgebot" %}
{%- set authorized_slave_list = [] %} {%- if 'region-dict' not in slapparameter_dict %}
{%- set monitor_base_url_dict = {} -%} {#- Be nice and allow to work with default configuration #}
{%- do slapparameter_dict.__setitem__('region-dict', {
'1': {
'sla-computer_guid': slap_configuration['computer'],
'state': slap_configuration['instance-state'],
'nameserver-list': slapparameter_dict.get('nameserver-list', []),
'check-frontend-ip-list': slapparameter_dict.get('check-frontend-ip-list', []),
}
}) %}
{%- endif %}
{%- set active_region_list = [] %}
{%- for region_name in sorted(slapparameter_dict['region-dict']) %}
{%- set region_parameter_dict = slapparameter_dict['region-dict'][region_name] %}
{%- if region_parameter_dict.get('state', 'started') == 'started' %}
{%- do active_region_list.append(region_name) %}
{%- endif %}
{%- endfor %}
{%- set authorized_slave_dict = {} %}
{%- set publish_slave_dict_dict = {} %}
{%- for slave in slave_instance_list | sort(attribute='slave_title') %} {%- for slave in slave_instance_list | sort(attribute='slave_title') %}
{%- do authorized_slave_list.append(slave) %} {%- set slave_reference = slave.pop('slave_reference') %}
{%- set publish_dict = {'assigned-region-dict': {}} %}
{%- if '_' in slave %}
{%- set base_slave_dict = json_module.loads(slave.pop('_')) %} {#- XXX: Unsafe! #}
{%- do base_slave_dict.__setitem__('-slave-title', slave['slave_title']) %}
{%- do base_slave_dict.__setitem__('-slave-reference', slave_reference) %}
{%- set slave_region_dict = base_slave_dict.pop('region-dict', {}) %}
{%- if slave_region_dict == {} %}
{%- for region in active_region_list %}
{%- do slave_region_dict.__setitem__(region, {}) %}
{%- endfor %}
{%- endif %}
{%- for region in slave_region_dict %}
{%- if region in active_region_list %}
{%- set region_info = {
'nameserver-list': slapparameter_dict['region-dict'][region].get('nameserver-list') or slapparameter_dict.get('slapparameter_dict') or [],
'check-frontend-ip-list': slave_region_dict[region].get('check-frontend-ip-list') or base_slave_dict.get('check-frontend-ip-list') or slapparameter_dict['region-dict'][region].get('check-frontend-ip-list') or slapparameter_dict.get('check-frontend-ip-list') or [],
} %}
{%- do publish_dict['assigned-region-dict'].__setitem__(region, region_info) %}
{%- set slave_dict = base_slave_dict.copy() %}
{%- do slave_dict.update(region_info) %}
{%- if region not in authorized_slave_dict %}
{%- do authorized_slave_dict.__setitem__(region, [slave_dict]) %}
{%- else %}
{%- do authorized_slave_dict[region].append(slave_dict) %}
{%- endif %}
{%- endif %}
{%- endfor %}
{%- endif %}
{%- do publish_slave_dict_dict.__setitem__(slave_reference, publish_dict) %}
{%- endfor %} {%- endfor %}
{%- set monitor_base_port = int(slap_configuration['configuration.monitor-base-port']) %} {%- set monitor_base_port = int(slap_configuration['configuration.monitor-base-port']) %}
{%- for i in range(1, edgebot_quantity + 1) %} {%- set number = {'i': 1} %}
{%- set edgebot_name = "%s-%s" % (namebase, i) %} {%- for region_name in sorted(slapparameter_dict['region-dict']) %}
{%- set request_section_title = 'request-%s' % edgebot_name %} {%- set region_parameter_dict = slapparameter_dict['region-dict'][region_name] %}
{%- do edgebot_list.append(edgebot_name) %} {%- set edgebot_name = "%s-%s" % (namebase, region_name) %}
{%- do edgebot_section_list.append(request_section_title) %} {%- set request_section_title = 'request-%s' % (hashlib_module.md5(edgebot_name.encode('utf-8')).hexdigest(),) %}
{%- do part_list.append(request_section_title) %} {%- do part_list.append(request_section_title) %}
{%- do request_dict.__setitem__(request_section_title, {#- Note: monitor-httpd-port will vary on regions being added and removed,
{ but this is accepted, as it's only internal trick #}
'config': {'monitor-httpd-port': monitor_base_port + i}, {%- do request_dict.__setitem__(
'name': edgebot_name, request_section_title,
'sla': {}, {
'state': 'started', 'config': {
}) %} 'monitor-httpd-port': monitor_base_port + number['i'],
'check-frontend-ip-list': region_parameter_dict.get('check-frontend-ip-list', []),
'nameserver-list': region_parameter_dict.get('nameserver-list', []),
'extra_slave_instance_list': authorized_slave_dict.get(region_name, [])
},
'name': edgebot_name,
'sla': {'computer_guid': region_parameter_dict['sla-computer_guid']},
'state': region_parameter_dict.get('state', 'started'),
}) %}
{%- do number.__setitem__('i', number['i'] + 1) %}
{%- endfor %} {%- endfor %}
[replicate] [replicate]
...@@ -54,54 +106,58 @@ config-monitor-username = ${monitor-instance-parameter:username} ...@@ -54,54 +106,58 @@ config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-password = ${monitor-htpasswd:passwd} config-monitor-password = ${monitor-htpasswd:passwd}
software-url = ${slap-connection:software-release-url} software-url = ${slap-connection:software-release-url}
software-type = {{edgebot_software_type}} software-type = {{edgebot_software_type}}
return = monitor-base-url
{% for section, edgebot_request in request_dict.items() %} {%- set monitor_base_url_dict = {} -%}
{% for section, edgebot_request in request_dict.items() %}
[{{section}}] [{{section}}]
<= replicate <= replicate
name = {{ edgebot_request.get('name') }} name = {{ edgebot_request['name'] }}
{%- if edgebot_request.get('state') %} state = {{ edgebot_request['state'] }}
state = {{ edgebot_request.get('state') }} {%- if edgebot_request['state'] != 'destroyed' %}
{%- endif%} {%- do monitor_base_url_dict.__setitem__(section, '${' ~ section ~ ':connection-monitor-base-url}') %}
{%- set slave_configuration_dict = slapparameter_dict %} return = monitor-base-url
{%- do slave_configuration_dict.update(edgebot_request.get('config')) %} {%- endif %}
{%- do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list)) %} {%- set edgebot_configuration_dict = edgebot_request['config'] %}
{%- for config_key, config_value in slave_configuration_dict.items() %} {%- for config_key, config_value in edgebot_configuration_dict.items() %}
config-{{ config_key }} = {{ dumps(config_value) }} config-{{ config_key }} = {{ dumps(config_value) }}
{% endfor -%} {% endfor -%}
{%- if edgebot_request.get('sla') %} {%- for parameter, value in edgebot_request['sla'].items() %}
{%- for parameter, value in edgebot_request.get('sla').items() %}
sla-{{ parameter }} = {{ value }} sla-{{ parameter }} = {{ value }}
{%- endfor %} {%- endfor %}
{%- else %}
# As no SLA was provided, by default it is requested on the same computer
sla-computer_guid = ${slap-connection:computer-id}
{% endif %}
{%- do monitor_base_url_dict.__setitem__(section, '${' ~ section ~ ':connection-monitor-base-url}') -%}
{%- endfor %} {%- endfor %}
{%- set directory_list = [] -%} # Publish information for each slave
{%- for slave_instance in slave_instance_list -%} {%- for slave_reference, publish_dict in publish_slave_dict_dict.items() -%}
{%- set publish_section_title = 'publish-%s' % slave_instance.get('slave_reference') -%} {%- set publish_section_title = 'publish-%s' % (hashlib_module.md5(slave_reference.encode('utf-8')).hexdigest(),) -%}
{%- do part_list.append(publish_section_title) %} {%- do part_list.append(publish_section_title) %}
[{{ publish_section_title }}] [{{ publish_section_title }}]
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish.serialised
-slave-reference = {{ slave_instance.get('slave_reference') }} available-region-list = {{ dumps(list(active_region_list)) }}
{% endfor %} -slave-reference = {{ slave_reference }}
{%- for key, value in publish_dict.items() %}
{{ key }} = {{ dumps(value) }}
{%- endfor %}
{% endfor %}
[monitor-conf-parameters] [monitor-conf-parameters]
monitor-title = Monitor monitor-title = Monitor
password = ${monitor-htpasswd:passwd} password = ${monitor-htpasswd:passwd}
[monitor-base-url-dict] [monitor-base-url-dict]
{% for key, value in monitor_base_url_dict.items() -%} {% for key, value in monitor_base_url_dict.items() -%}
{{ key }} = {{ value }} {{ key }} = {{ value }}
{% endfor %} {% endfor %}
[buildout] [buildout]
extends = {{ instance_base_monitor }} extends = {{ instance_base_monitor }}
parts += parts +=
slave-test-configuration slave-test-configuration
{% for part in part_list %} {% for part in part_list %}
{{ ' %s' % part }} {{ ' %s' % part }}
{%- endfor %} {%- endfor %}
{%- endif %}
[publish-connection-information]
recipe = slapos.cookbook:publish.serialised
active-region-list = {{ dumps(list(active_region_list)) }}
sla-computer_guid = {{ dumps(slap_configuration['computer']) }}
sla-instance_guid = {{ dumps(slap_configuration['instance-guid']) }}
{%- endif %} {#- if slap_software_type == software_type #}
...@@ -32,6 +32,7 @@ template = ${template-monitor-edgetest:target} ...@@ -32,6 +32,7 @@ template = ${template-monitor-edgetest:target}
rendered = $${buildout:directory}/template-monitor-base-edgetest.cfg rendered = $${buildout:directory}/template-monitor-base-edgetest.cfg
extensions = jinja2.ext.do extensions = jinja2.ext.do
context = import json_module json context = import json_module json
import hashlib_module hashlib
key develop_eggs_directory buildout:develop-eggs-directory key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration key slapparameter_dict slap-configuration:configuration
...@@ -53,6 +54,7 @@ surykatka-binary = ${buildout:bin-directory}/${surykatka:script-name} ...@@ -53,6 +54,7 @@ surykatka-binary = ${buildout:bin-directory}/${surykatka:script-name}
template-surykatka-ini = ${template-surykatka-ini:target} template-surykatka-ini = ${template-surykatka-ini:target}
context = import json_module json context = import json_module json
import hashlib_module hashlib
key develop_eggs_directory buildout:develop-eggs-directory key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory key eggs_directory buildout:eggs-directory
section slap_configuration slap-configuration section slap_configuration slap-configuration
...@@ -75,13 +77,8 @@ url = $${slap-connection:server-url} ...@@ -75,13 +77,8 @@ url = $${slap-connection:server-url}
key = $${slap-connection:key-file} key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file} cert = $${slap-connection:cert-file}
# Defaults # Defaults
configuration.check-status-code = 200 configuration.nameserver-list =
configuration.check-http-header-dict = {} configuration.check-frontend-ip-list =
configuration.nameserver =
configuration.check-frontend-ip =
configuration.check-certificate-expiration-days = 15
configuration.check-maximum-elapsed-time = 2
configuration.failure-amount = 2
# use monitor-base-port to have monitor listening on each instance # use monitor-base-port to have monitor listening on each instance
# on different port and also on different port than other services # on different port and also on different port than other services
# it makes it possible to instantiate it correctly on signle IP, for # it makes it possible to instantiate it correctly on signle IP, for
......
...@@ -15,14 +15,17 @@ ...@@ -15,14 +15,17 @@
"description": "Cluster of bots to perform a distributed monitoring ", "description": "Cluster of bots to perform a distributed monitoring ",
"request": "instance-edgetest-input-schema.json", "request": "instance-edgetest-input-schema.json",
"response": "instance-default-output-schema.json", "response": "instance-default-output-schema.json",
"serialisation": "json-in-xml",
"index": 1 "index": 1
}, },
"edgetest-slave": { "edgetest-slave": {
"title": "Edge Test Slave", "title": "Edge Test Slave",
"shared": true,
"software-type": "edgetest", "software-type": "edgetest",
"description": "Cluster of bots to perform a distributed monitoring ", "description": "Cluster of bots to perform a distributed monitoring ",
"request": "instance-edgetest-slave-input-schema.json", "request": "instance-edgetest-slave-input-schema.json",
"response": "instance-default-output-schema.json", "response": "instance-default-output-schema.json",
"serialisation": "json-in-xml",
"index": 2 "index": 2
} }
} }
......
...@@ -2,7 +2,6 @@ ...@@ -2,7 +2,6 @@
INTERVAL = 120 INTERVAL = 120
TIMEOUT = {{ timeout }} TIMEOUT = {{ timeout }}
SQLITE = {{ db }} SQLITE = {{ db }}
{%- set nameserver_list = nameserver.split() %}
{%- if len(nameserver_list) > 0 %} {%- if len(nameserver_list) > 0 %}
NAMESERVER = NAMESERVER =
{%- for nameserver_entry in sorted(nameserver_list) %} {%- for nameserver_entry in sorted(nameserver_list) %}
...@@ -10,7 +9,7 @@ NAMESERVER = ...@@ -10,7 +9,7 @@ NAMESERVER =
{%- endfor %} {%- endfor %}
{% endif %} {% endif %}
URL = URL =
{%- for slave in slave_instance_list | sort(attribute='slave_title') %} {%- for slave in slave_instance_list | sort(attribute='-slave-title') %}
{%- if 'url' in slave %} {%- if 'url' in slave %}
{{ slave['url'] }} {{ slave['url'] }}
{%- endif -%} {%- endif -%}
......
This diff is collapsed.
...@@ -18,7 +18,7 @@ md5sum = 2bd1779425b7561682c0de5496d808ed ...@@ -18,7 +18,7 @@ md5sum = 2bd1779425b7561682c0de5496d808ed
[root-common] [root-common]
filename = root-common.cfg.in filename = root-common.cfg.in
md5sum = 8e28f599247ad604ec6e32df410412a8 md5sum = c13b4f1a5aa526a8d3f8e02bf6baf785
[instance-neo-admin] [instance-neo-admin]
filename = instance-neo-admin.cfg.in filename = instance-neo-admin.cfg.in
......
...@@ -47,7 +47,7 @@ ...@@ -47,7 +47,7 @@
"type": "object" "type": "object"
}, },
"sla-dict": { "sla-dict": {
"description": "[NEO SR only] Where to request instances. Each key is a query string for criterions (e.g. \"computer_guid=foo\"), and each value is a list of partition references ('node-0', 'node-1', ...). The prefix 'node-' is mandatory and the number must start from 0. The total number of nodes here must be equal to the length of node-list.", "description": "[NEO SR only] Where to request instances. Each key is a query string for criterions (e.g. \"computer_guid=foo\"), and each value is a list of partition references ('node-0', 'node-1', ...). The prefix 'node-' is mandatory and the number must start from 0. The total number of nodes here must be equal to the length of node-list. A node can be removed by requesting it without any admin, master and storage.",
"additionalProperties": { "additionalProperties": {
"type": "array", "type": "array",
"items": { "items": {
...@@ -95,7 +95,7 @@ ...@@ -95,7 +95,7 @@
"type": "integer" "type": "integer"
}, },
"storage-count": { "storage-count": {
"description": "Number of storage nodes to deploy. One master and one admin node is deployed with each storage.", "description": "Number of storage nodes to deploy. One master and one admin node is deployed with each storage. 0 to disable.",
"default": 1, "default": 1,
"type": "integer" "type": "integer"
}, },
......
...@@ -89,7 +89,7 @@ software-type = {{ software_type }} ...@@ -89,7 +89,7 @@ software-type = {{ software_type }}
config-autostart = {{ dumps(sum(storage_count)) }} config-autostart = {{ dumps(sum(storage_count)) }}
{%- do assert(replicas < len(node_list)) %} {%- do assert(replicas < len(node_list)) %}
{%- set monitor = set() %} {%- set monitor = set() %}
{%- for i, node in enumerate(node_list) %} {%- for node in node_list %}
{%- set port = node.get('monitor') %} {%- set port = node.get('monitor') %}
{%- if port %} {%- if port %}
{%- do monitor.add(node.get('admin') != 0 and port) %} {%- do monitor.add(node.get('admin') != 0 and port) %}
...@@ -101,13 +101,14 @@ config-autostart = {{ dumps(sum(storage_count)) }} ...@@ -101,13 +101,14 @@ config-autostart = {{ dumps(sum(storage_count)) }}
{%- endif %} {%- endif %}
{%- for i, node in enumerate(node_list) %} {%- for i, node in enumerate(node_list) %}
{%- set section_id = prefix ~ i %} {%- set section_id = prefix ~ i %}
{%- do section_id_list.append(section_id) %}
{%- if node.get('master') == 0 and not node.get('monitor') %} {%- if node.get('master') == 0 and not node.get('monitor') %}
{%- do node.setdefault('admin', 0) %} {%- do node.setdefault('admin', 0) %}
{%- endif %} {%- endif %}
{%- if monitor or node.get('admin') == 0 %} {%- if monitor or node.get('admin') == 0 %}
{%- do node.setdefault('monitor', 0) %} {%- do node.setdefault('monitor', 0) %}
{%- endif %} {%- endif %}
{%- for x in 'admin', 'master', 'storage-count' if node.get(x, 1) %}
{%- do section_id_list.append(section_id) %}
[{{section_id}}] [{{section_id}}]
<= {{ prefix }}request-common <= {{ prefix }}request-common
...@@ -136,6 +137,9 @@ config-monitor = {{ dumps(parameter_dict.get('monitor', {})) }} ...@@ -136,6 +137,9 @@ config-monitor = {{ dumps(parameter_dict.get('monitor', {})) }}
config-{{ k }} = {{ dumps(v) }} config-{{ k }} = {{ dumps(v) }}
{%- endfor %} {%- endfor %}
{{ sla(section_id) }} {{ sla(section_id) }}
{%- break %}
{%- endfor %}
{%- endfor %} {%- endfor %}
{%- do assert(len(monitor) == 1, monitor) %} {%- do assert(len(monitor) == 1, monitor) %}
......
...@@ -31,3 +31,7 @@ if returncode: ...@@ -31,3 +31,7 @@ if returncode:
subprocess.call( subprocess.call(
('tar', '-caf', result('db.tar.xz'), 'neo_tests'), ('tar', '-caf', result('db.tar.xz'), 'neo_tests'),
cwd=tempfile.gettempdir()) cwd=tempfile.gettempdir())
p = subprocess.Popen(('journalctl', '-o', 'export'), stdout=subprocess.PIPE)
with open(result('journalctl-export.xz'), 'wb') as f:
subprocess.call(('xz',), stdin=p.stdout, stdout=f)
p.wait()
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
[instance] [instance]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = f3aac995aa1129ca58f9a92851ad7091 md5sum = bfd488ba023f505be25d947ec830bab3
[yarn.lock] [yarn.lock]
filename = yarn.lock filename = yarn.lock
......
...@@ -7,7 +7,7 @@ theia-environment-parts = ...@@ -7,7 +7,7 @@ theia-environment-parts =
slapos-repository slapos-repository
runner-link runner-link
settings.json settings.json
request-script request-script-template
theia-parts = theia-parts =
frontend-reload frontend-reload
...@@ -230,7 +230,7 @@ recipe = plone.recipe.command ...@@ -230,7 +230,7 @@ recipe = plone.recipe.command
filename = logo.png filename = logo.png
full-path = $${directory:frontend-static}/$${:filename} full-path = $${directory:frontend-static}/$${:filename}
command = command =
cp -f ${logo.png:output} $${:full-path} cp --remove-destination ${logo.png:output} $${:full-path}
stop-on-error = true stop-on-error = true
[frontend-instance-slapos.css] [frontend-instance-slapos.css]
...@@ -551,17 +551,19 @@ recipe = slapos.cookbook:symbolic.link ...@@ -551,17 +551,19 @@ recipe = slapos.cookbook:symbolic.link
target-directory = $${directory:project} target-directory = $${directory:project}
link-binary = $${directory:runner} link-binary = $${directory:runner}
[request-script] [request-script-template]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
rendered = $${directory:project}/$${:_buildout_section_name_}.sh rendered = $${directory:project}/$${:_buildout_section_name_}.sh
mode = 0700 mode = 0700
template = template =
inline:#!/bin/sh inline:#!/bin/sh
software_name=html5as-base #replace the software name # This template is generated automatically, copy it in order to supply and request.
# Any manual change to this file may be lost.
software_name=html5as-base #replace the software name writen in ~/srv/project/slapos/software/
software_release_uri=~/srv/project/slapos/software/$software_name/software.cfg software_release_uri=~/srv/project/slapos/software/$software_name/software.cfg
# slapos supply is used to add the software to the software list to be supplied to a node. # slapos supply is used to add the software to the software list to be supplied to a node.
slapos supply $software_release_uri slaprunner slapos supply $software_release_uri slaprunner
# slapos request the allocation of an instance to the master. # slapos request the allocation of an instance to the master.
# slapos request also gets status and parameters of the instance if it has any # slapos request also gets status and parameters of the instance if it has any
# (slapos request is meant to be run multiple time until you get the status). # (slapos request is meant to be run multiple time until you get the status).
slapos request $software_name'_1' $software_release_uri slapos request $software_name'_1' $software_release_uri
...@@ -183,7 +183,7 @@ class TestTheia(TheiaTestCase): ...@@ -183,7 +183,7 @@ class TestTheia(TheiaTestCase):
self.computer_partition_root_path, self.computer_partition_root_path,
'srv', 'srv',
'project', 'project',
'request-script.sh', 'request-script-template.sh',
) )
self.assertTrue(os.path.exists(script_path)) self.assertTrue(os.path.exists(script_path))
......
...@@ -631,9 +631,6 @@ PasteDeploy = 1.5.2 ...@@ -631,9 +631,6 @@ PasteDeploy = 1.5.2
argparse = 1.4.0 argparse = 1.4.0
zope.dottedname = 4.1.0 zope.dottedname = 4.1.0
# test_UserManagerInterfaces in testERP5Security fails with 1.10.0.
Products.PluggableAuthService = 1.9.0
# modified version that works fine for buildout installation # modified version that works fine for buildout installation
SOAPpy = 0.12.0nxd001 SOAPpy = 0.12.0nxd001
...@@ -664,6 +661,7 @@ Products.GenericSetup = 1.8.6 ...@@ -664,6 +661,7 @@ Products.GenericSetup = 1.8.6
Products.LongRequestLogger = 2.1.0 Products.LongRequestLogger = 2.1.0
# Products.MimetypesRegistry 2.1 requires AccessControl>=3.0.0Acquisition. # Products.MimetypesRegistry 2.1 requires AccessControl>=3.0.0Acquisition.
Products.MimetypesRegistry = 2.0.10 Products.MimetypesRegistry = 2.0.10
Products.PluggableAuthService = 1.10.0
Products.PluginRegistry = 1.4 Products.PluginRegistry = 1.4
Products.TIDStorage = 5.5.0 Products.TIDStorage = 5.5.0
pyPdf = 1.13 pyPdf = 1.13
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment