Commit f9417613 authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼

Update Release Candidate

parents c4b68ad5 873a49f6
...@@ -20,7 +20,7 @@ patch-options = -p1 ...@@ -20,7 +20,7 @@ patch-options = -p1
patches = patches =
${:_profile_base_location_}/noroot.patch#05fc6333e05576ea8e5a49f27a6ef951 ${:_profile_base_location_}/noroot.patch#05fc6333e05576ea8e5a49f27a6ef951
configure-command = make configure-command = make
configure-options = makefiles CCARGS='-DUSE_SASL_AUTH -DUSE_CYRUS_SASL -DUSE_TLS -DHAS_PCRE -DHAS_DB -I${libdb:location}/include -I${pcre:location}/include -I${openssl-1.0:location}/include -I${cyrus-sasl:location}/include/sasl' AUXLIBS='-L${openssl-1.0:location}/lib -L${pcre:location}/lib -L${libdb:location}/lib -L${cyrus-sasl:location}/lib -lssl -lpcre -ldb -lcrypto -lsasl2 -Wl,-rpath=${openssl-1.0:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${libdb:location}/lib -Wl,-rpath=${cyrus-sasl:location}/lib' configure-options = makefiles CCARGS='-DUSE_SASL_AUTH -DUSE_CYRUS_SASL -DUSE_TLS -DHAS_PCRE -DHAS_DB -I${libdb:location}/include -I${pcre:location}/include -I${openssl:location}/include -I${cyrus-sasl:location}/include/sasl' AUXLIBS='-L${openssl:location}/lib -L${pcre:location}/lib -L${libdb:location}/lib -L${cyrus-sasl:location}/lib -lssl -lpcre -ldb -lcrypto -lsasl2 -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${libdb:location}/lib -Wl,-rpath=${cyrus-sasl:location}/lib'
make-targets = non-interactive-package install_root=${:location} make-targets = non-interactive-package install_root=${:location}
environment = environment =
PATH=${patch:location}/bin:${m4:location}/bin:%(PATH)s PATH=${patch:location}/bin:${m4:location}/bin:%(PATH)s
...@@ -15,15 +15,15 @@ ...@@ -15,15 +15,15 @@
[template] [template]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = e6d5c7bb627b4f1d3e7c99721b7c58fe md5sum = 399b398a8eabfa6126d2a521dc779f9b
[template-kvm] [template-kvm]
filename = instance-kvm.cfg.jinja2 filename = instance-kvm.cfg.jinja2
md5sum = 31b17b55200ea065cb97085283ef5568 md5sum = 704b6ac6bf42837bcd8f4582c5a746c0
[template-kvm-cluster] [template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in filename = instance-kvm-cluster.cfg.jinja2.in
md5sum = 28a00c28a972f42627849b25c2792abb md5sum = 80b9b70ba1ccbc09deb8f9cad60f352c
[template-kvm-resilient] [template-kvm-resilient]
filename = instance-kvm-resilient.cfg.jinja2 filename = instance-kvm-resilient.cfg.jinja2
...@@ -31,7 +31,7 @@ md5sum = 7de5756f59ef7d823cd8ed33e6d15230 ...@@ -31,7 +31,7 @@ md5sum = 7de5756f59ef7d823cd8ed33e6d15230
[template-kvm-import] [template-kvm-import]
filename = instance-kvm-import.cfg.jinja2.in filename = instance-kvm-import.cfg.jinja2.in
md5sum = bd7e5db872b0dbe7716ec49c3907c401 md5sum = 3e7ff2ba85762ca47b5f90495d492570
[template-kvm-import-script] [template-kvm-import-script]
filename = template/kvm-import.sh.jinja2 filename = template/kvm-import.sh.jinja2
...@@ -39,7 +39,7 @@ md5sum = cd0008f1689dfca9b77370bc4d275b70 ...@@ -39,7 +39,7 @@ md5sum = cd0008f1689dfca9b77370bc4d275b70
[template-kvm-export] [template-kvm-export]
filename = instance-kvm-export.cfg.jinja2 filename = instance-kvm-export.cfg.jinja2
md5sum = f12df4256eb5bd31a01c0ddc4b3897bb md5sum = c15f7600389b0c641622fcfdc42260d8
[template-kvm-export-script] [template-kvm-export-script]
filename = template/kvm-export.sh.jinja2 filename = template/kvm-export.sh.jinja2
...@@ -88,3 +88,11 @@ md5sum = 9c67058edcc4edae0b57956c0932a9fc ...@@ -88,3 +88,11 @@ md5sum = 9c67058edcc4edae0b57956c0932a9fc
[image-download-config-creator] [image-download-config-creator]
_update_hash_filename_ = template/image-download-config-creator.py _update_hash_filename_ = template/image-download-config-creator.py
md5sum = 54261e418ab9860efe73efd514c4d47f md5sum = 54261e418ab9860efe73efd514c4d47f
[whitelist-firewall-download-controller]
_update_hash_filename_ = template/whitelist-firewall-download-controller.py
md5sum = bc64e29546833817636261d1b28aa6dc
[whitelist-domains-default]
_update_hash_filename_ = template/whitelist-domains-default
md5sum = e9d40162ba77472775256637a2617d14
...@@ -556,6 +556,12 @@ ...@@ -556,6 +556,12 @@
"title": "FreeBSD 12.1 RELEASE bootonly x86_64" "title": "FreeBSD 12.1 RELEASE bootonly x86_64"
} }
] ]
},
"whitelist-domains": {
"title": "Whitelist domains",
"description": "List of whitelisted domain names to be accessed from the VM. They will be resolved to IPs depending on where the VM end up. IPs can be used too.",
"type": "string",
"textarea": true
} }
}, },
"type": "object" "type": "object"
......
...@@ -107,6 +107,12 @@ ...@@ -107,6 +107,12 @@
"title": "FreeBSD 12.1 RELEASE bootonly x86_64" "title": "FreeBSD 12.1 RELEASE bootonly x86_64"
} }
] ]
},
"whitelist-domains": {
"title": "Whitelist domains",
"description": "List of whitelisted domain names to be accessed from the VM. They will be resolved to IPs depending on where the VM end up. IPs can be used too.",
"type": "string",
"textarea": true
} }
}, },
"type": "object" "type": "object"
......
...@@ -24,7 +24,7 @@ global-ipv6 = {{ ipv6 }} ...@@ -24,7 +24,7 @@ global-ipv6 = {{ ipv6 }}
[slap-parameter] [slap-parameter]
{% for k, v in slapparameter_dict.items() -%} {% for k, v in slapparameter_dict.items() -%}
{{ k }} = {{ v }} {{ k }} = {{ dumps(v) }}
{% endfor -%} {% endfor -%}
[request-common] [request-common]
...@@ -135,14 +135,13 @@ config-document-host = ${apache-conf:ip} ...@@ -135,14 +135,13 @@ config-document-host = ${apache-conf:ip}
config-document-port = ${apache-conf:port} config-document-port = ${apache-conf:port}
config-document-path = ${hash-code:passwd} config-document-path = ${hash-code:passwd}
config-keyboard-layout-language = {{ dumps(kvm_parameter_dict.get('keyboard-layout-language', 'fr')) }} config-keyboard-layout-language = {{ dumps(kvm_parameter_dict.get('keyboard-layout-language', 'fr')) }}
{%- if 'boot-image-url-list' in kvm_parameter_dict %} {%- for k in ['boot-image-url-list', 'boot-image-url-select', 'whitelist-domains'] %}
{#- play nice: if parameter was not constructed by the original request, do not send it at all #} {#- play nice - use parameter only if present #}
config-boot-image-url-list = {{ kvm_parameter_dict['boot-image-url-list'] }} {%- if k in kvm_parameter_dict %}
{%- endif %} {#- play safe - dumps value #}
{%- if 'boot-image-url-select' in kvm_parameter_dict %} config-{{ k }} = {{ dumps(kvm_parameter_dict[k]) }}
{#- play nice: if parameter was not constructed by the original request, do not send it at all #} {%- endif %}
config-boot-image-url-select = {{ kvm_parameter_dict['boot-image-url-select'] }} {%- endfor %}
{%- endif %}
config-type = cluster config-type = cluster
{% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url', kvm_parameter_dict.get('bootstrap-script-url', '')) -%} {% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url', kvm_parameter_dict.get('bootstrap-script-url', '')) -%}
......
...@@ -5,24 +5,11 @@ extends = ...@@ -5,24 +5,11 @@ extends =
parts += parts +=
cron-entry-backup cron-entry-backup
${instance-kvm-parts:parts}
certificate-authority
certificate-authority-service
publish-connection-information
kvm-vnc-promise
kvm-disk-image-corruption-promise
websockify-sighandler
websockify-sighandler-service
novnc-promise
cron
cron-service
frontend-promise
# monitor parts
monitor-base
[slap-parameter] [slap-parameter]
{% for k, v in slapparameter_dict.items() -%} {% for k, v in slapparameter_dict.items() -%}
{{ k }} = {{ v }} {{ k }} = {{ dumps(v) }}
{% endfor -%} {% endfor -%}
# Create the exporter executable, which is a simple shell script # Create the exporter executable, which is a simple shell script
......
...@@ -14,7 +14,7 @@ extends = ...@@ -14,7 +14,7 @@ extends =
[slap-parameter] [slap-parameter]
{% for k, v in slapparameter_dict.items() -%} {% for k, v in slapparameter_dict.items() -%}
{{ k }} = {{ v }} {{ k }} = {{ dumps(v) }}
{% endfor -%} {% endfor -%}
[resilient-publish-connection-parameter] [resilient-publish-connection-parameter]
......
...@@ -419,6 +419,12 @@ ...@@ -419,6 +419,12 @@
"title": "FreeBSD 12.1 RELEASE bootonly x86_64" "title": "FreeBSD 12.1 RELEASE bootonly x86_64"
} }
] ]
},
"whitelist-domains": {
"title": "Whitelist domains",
"description": "List of whitelisted domain names to be accessed from the VM. They will be resolved to IPs depending on where the VM end up. IPs can be used too.",
"type": "string",
"textarea": true
} }
} }
} }
{# Workaround empty parameter passing #}
{# In case of resilient '' is converted to 'None' string, but with slapproxy '' becomes None #}
{% for k, v in slapparameter_dict.items() %}
{% if v == 'None' or v is none %}
{% do slapparameter_dict.__setitem__(k, '') %}
{% endif %}
{% endfor %}
{% set additional_frontend = slapparameter_dict.get('frontend-additional-instance-guid') %} {% set additional_frontend = slapparameter_dict.get('frontend-additional-instance-guid') %}
{% set enable_http = str(slapparameter_dict.get('enable-http-server', False)).lower() == 'true' -%} {% set enable_http = str(slapparameter_dict.get('enable-http-server', False)).lower() == 'true' -%}
{% set use_tap = str(slapparameter_dict.get('use-tap', True)).lower() == 'true' -%} {% set use_tap = str(slapparameter_dict.get('use-tap', True)).lower() == 'true' -%}
...@@ -9,6 +16,7 @@ ...@@ -9,6 +16,7 @@
{% set instance_type = slapparameter_dict.get('type', 'standalone') -%} {% set instance_type = slapparameter_dict.get('type', 'standalone') -%}
{% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%} {% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
{% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%} {% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%}
{% set whitelist_domains = slapparameter_dict.get('whitelist-domains', '') -%}
{% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %} {% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %}
{% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %} {% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %}
{% set cpu_max_count = dumps(slapparameter_dict.get('cpu-max-count', int(slapparameter_dict.get('cpu-count', 1)) + 1)) %} {% set cpu_max_count = dumps(slapparameter_dict.get('cpu-max-count', int(slapparameter_dict.get('cpu-count', 1)) + 1)) %}
...@@ -92,22 +100,14 @@ config-state = empty ...@@ -92,22 +100,14 @@ config-state = empty
config-url = ${monitor-base:base-url}/private/boot-image-url-select/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password} config-url = ${monitor-base:base-url}/private/boot-image-url-select/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
[boot-image-url-select-source-config] [boot-image-url-select-source-config]
# generates configuration of the image from the user parameter
# special "magic" is used, to properly support multiline boot-image-url-select
# but in the same time correctly generate the configuration file
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = inline: template = inline:
{#- Do special trick to support boot-image-url-select being None, if key is present with value "" #}
{%- raw %} {%- raw %}
{%- set boot_image_url_select = slap_parameter.get('boot-image-url-select') or '' %}
{%- if boot_image_url_select == 'None' %}
{#- That's insane protection, is it 'None' because of type = array? #}
{%- set boot_image_url_select = '' %}
{%- endif %}
{{ boot_image_url_select }} {{ boot_image_url_select }}
{% endraw -%} {% endraw -%}
boot-image-url-select = {{ dumps(slapparameter_dict['boot-image-url-select']) }}
context = context =
section slap_parameter slap-parameter key boot_image_url_select :boot-image-url-select
rendered = ${directory:etc}/boot-image-url-select.json rendered = ${directory:etc}/boot-image-url-select.json
[boot-image-url-select-processed-config] [boot-image-url-select-processed-config]
...@@ -199,17 +199,14 @@ config-state = empty ...@@ -199,17 +199,14 @@ config-state = empty
config-url = ${monitor-base:base-url}/private/boot-image-url-list/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password} config-url = ${monitor-base:base-url}/private/boot-image-url-list/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
[boot-image-url-list-source-config] [boot-image-url-list-source-config]
# generates configuration of the image from the user parameter
# special "magic" is used, to properly support multiline boot-image-url-list
# but in the same time correctly generate the configuration file
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = inline: template = inline:
{#- Do special trick to support boot-image-url-list being None, if key is present with value "" #}
{%- raw %} {%- raw %}
{{ slap_parameter.get('boot-image-url-list') or '' }} {{ boot_image_url_list }}
{% endraw -%} {% endraw -%}
boot-image-url-list = {{ dumps(slapparameter_dict['boot-image-url-list']) }}
context = context =
section slap_parameter slap-parameter key boot_image_url_list :boot-image-url-list
rendered = ${directory:etc}/boot-image-url-list.conf rendered = ${directory:etc}/boot-image-url-list.conf
[boot-image-url-list-processed-config] [boot-image-url-list-processed-config]
...@@ -976,7 +973,7 @@ keyboard-layout-language = fr ...@@ -976,7 +973,7 @@ keyboard-layout-language = fr
{% set key_list = v.split('\n') -%} {% set key_list = v.split('\n') -%}
{{ k }} = {{ k }} =
{{ key_list | join('\n ') }} {{ key_list | join('\n ') }}
{% elif k == 'boot-image-url-list' %} {% elif k in ['boot-image-url-list', 'boot-image-url-select', 'whitelist-domains'] %}
{# needs to decorate possibly multiline or maybe unsafe value #} {# needs to decorate possibly multiline or maybe unsafe value #}
{{ k }} = {{ dumps(v) }} {{ k }} = {{ dumps(v) }}
{% else -%} {% else -%}
...@@ -1042,7 +1039,62 @@ command-line = ...@@ -1042,7 +1039,62 @@ command-line =
{% endif -%} {% endif -%}
[buildout] {% do part_list.append('whitelist-firewall') -%}
[whitelist-firewall]
recipe = slapos.cookbook:wrapper
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
wrapper-path = ${directory:scripts}/${:_buildout_section_name_}
command-line =
{{ dnsresolver_executable }}
--style list
--output ${:output}
${:source}
source =
${whitelist-domains-request:rendered}
{{ whitelist_domains_default }}
${whitelist-domains-resolv.conf:output}
${whitelist-domains-download:output}
output = ${buildout:directory}/.slapos-whitelist-firewall
[whitelist-firewall-directory]
recipe = plone.recipe.command
location = ${buildout:parts-directory}/whitelist-firewall
command = mkdir -p ${:location}
update-command = ${:command}
[whitelist-domains-request]
recipe = slapos.recipe.template:jinja2
template = inline:
{%- raw %}
{%- for domain in whitelist_domains.split() %}
{{ domain }}
{%- endfor %}
{% endraw -%}
rendered = ${whitelist-firewall-directory:location}/${:_buildout_section_name_}.txt
whitelist-domains = {{ dumps(whitelist_domains) }}
extensions = jinja2.ext.do
context =
key whitelist_domains :whitelist-domains
[whitelist-domains-resolv.conf]
recipe = plone.recipe.command
output = ${whitelist-firewall-directory:location}/${:_buildout_section_name_}.txt
update-command = ${:command}
command =
egrep ^nameserver /etc/resolv.conf | cut -d ' ' -f 2 > ${:output}
[whitelist-domains-download]
recipe = slapos.cookbook:wrapper
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
wrapper-path = ${directory:scripts}/${:_buildout_section_name_}
output = ${whitelist-firewall-directory:location}/${:_buildout_section_name_}.txt
interval = 3600
command-line = {{ python_executable }} {{ whitelist_firewall_download_controller }} {{ curl_executable_location }} 3600 ${:output} ${:url}
url = https://stream.nxdcdn.com/rapidspace-whitelist-domains
[instance-kvm-parts]
parts = parts =
certificate-authority certificate-authority
certificate-authority-service certificate-authority-service
...@@ -1081,6 +1133,9 @@ parts = ...@@ -1081,6 +1133,9 @@ parts =
# Complete parts with sections # Complete parts with sections
{{ part_list | join('\n ') }} {{ part_list | join('\n ') }}
[buildout]
parts = ${instance-kvm-parts:parts}
extends = extends =
# Add extends list # Add extends list
{{ extends_list | join('\n ') }} {{ extends_list | join('\n ') }}
......
...@@ -80,9 +80,12 @@ extra-context = ...@@ -80,9 +80,12 @@ extra-context =
raw ansible_promise_tpl ${template-ansible-promise:location}/${template-ansible-promise:filename} raw ansible_promise_tpl ${template-ansible-promise:location}/${template-ansible-promise:filename}
raw curl_executable_location ${curl:location}/bin/curl raw curl_executable_location ${curl:location}/bin/curl
raw dash_executable_location ${dash:location}/bin/dash raw dash_executable_location ${dash:location}/bin/dash
raw dnsresolver_executable ${buildout:bin-directory}/dnsresolver
raw dcron_executable_location ${dcron:location}/sbin/crond raw dcron_executable_location ${dcron:location}/sbin/crond
raw debian_amd64_netinst_location ${debian-amd64-netinst.iso:location}/${debian-amd64-netinst.iso:filename} raw debian_amd64_netinst_location ${debian-amd64-netinst.iso:location}/${debian-amd64-netinst.iso:filename}
raw file_download_script ${file-download-script:location}/${file-download-script:filename} raw file_download_script ${file-download-script:location}/${file-download-script:filename}
raw whitelist_domains_default ${whitelist-domains-default:location}/${whitelist-domains-default:filename}
raw whitelist_firewall_download_controller ${whitelist-firewall-download-controller:target}
raw image_download_controller ${image-download-controller:target} raw image_download_controller ${image-download-controller:target}
raw image_download_config_creator ${image-download-config-creator:target} raw image_download_config_creator ${image-download-config-creator:target}
raw logrotate_cfg ${template-logrotate-base:rendered} raw logrotate_cfg ${template-logrotate-base:rendered}
......
...@@ -182,6 +182,10 @@ path = download_file.in ...@@ -182,6 +182,10 @@ path = download_file.in
filename = download_file filename = download_file
on-update = true on-update = true
[whitelist-domains-default]
<= download-template-base
filename = whitelist-domains-default
[template-httpd] [template-httpd]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-kvm-http.cfg.in template = ${:_profile_base_location_}/instance-kvm-http.cfg.in
...@@ -199,6 +203,10 @@ mode = 640 ...@@ -199,6 +203,10 @@ mode = 640
[image-download-config-creator] [image-download-config-creator]
<= image-download-controller <= image-download-controller
[whitelist-firewall-download-controller]
<= image-download-controller
[versions] [versions]
websockify = 0.9.0 websockify = 0.9.0
......
# Minimal whitelisted domains needed to instantiate the instance
# Does not guarantee good usage of the guest VM itself
# The full list shall end up in whitelist-domains-download:url
# shcache.nxdcdn.com is default source for a lot of operations
shacache.nxdcdn.com
# stream.nxdcdn.com is needed by partition itself
stream.nxdcdn.com
# partition has to access default SlapOS Master
slap.vifib.com
slapos.vifib.com
# Partition needs access to SlapOS Master related resources
hnode.cdn.vifib.com
node.cdn.vifib.com
#!/usr/bin/env python
import os
import subprocess
import sys
import time
import logging
# Note: Assuring only one running instance is not done, as this script is only
# run from supervisord, which does it already
if __name__ == "__main__":
curl, sleep, output, url = sys.argv[1:]
sleep = int(sleep)
tmp_output = output + '.tmp'
logging.basicConfig(
format='%%(asctime)s [%%(levelname)s] %s : %%(message)s' % (url,),
level=logging.DEBUG)
logging.info('Redownloading each %is', sleep)
while True:
logging.info('Fetching')
try:
subprocess.check_output([
curl,
'--location', # follow redirects
'--no-progress-meter', # do not tell too much
'--max-time', '600', # 10 minutes is maximum
'--fail', # fail in case of wrong HTTP code
'--output', tmp_output, url],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
logging.error('Problem while downloading: %r', e.output.strip())
if os.path.exists(tmp_output):
logging.info('Stored output')
os.rename(tmp_output, output)
logging.info('Sleeping for %is', sleep)
time.sleep(sleep)
...@@ -37,10 +37,17 @@ import sqlite3 ...@@ -37,10 +37,17 @@ import sqlite3
from six.moves.urllib.parse import parse_qs, urlparse from six.moves.urllib.parse import parse_qs, urlparse
import unittest import unittest
import subprocess import subprocess
import tempfile
import SocketServer
import SimpleHTTPServer
import multiprocessing
import time
import shutil
from slapos.recipe.librecipe import generateHashFromFiles from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.slap.standalone import SlapOSNodeCommandError from slapos.slap.standalone import SlapOSNodeCommandError
from slapos.testing.utils import findFreeTCPPort
has_kvm = os.access('/dev/kvm', os.R_OK | os.W_OK) has_kvm = os.access('/dev/kvm', os.R_OK | os.W_OK)
skipUnlessKvm = unittest.skipUnless(has_kvm, 'kvm not loaded or not allowed') skipUnlessKvm = unittest.skipUnless(has_kvm, 'kvm not loaded or not allowed')
...@@ -105,34 +112,67 @@ bootstrap_machine_param_dict = { ...@@ -105,34 +112,67 @@ bootstrap_machine_param_dict = {
"enable-monitor": True, "enable-monitor": True,
"keyboard-layout-language": "fr" "keyboard-layout-language": "fr"
} }
@skipUnlessKvm
class ServicesTestCase(InstanceTestCase):
def test_hashes(self):
hash_files = [
'software_release/buildout.cfg',
]
expected_process_names = [
'6tunnel-10022-{hash}-on-watch',
'6tunnel-10080-{hash}-on-watch',
'6tunnel-10443-{hash}-on-watch',
'certificate_authority-{hash}-on-watch',
'crond-{hash}-on-watch',
'kvm-{hash}-on-watch',
'websockify-{hash}-on-watch',
]
class KvmMixin(object):
def getProcessInfo(self):
hash_value = generateHashFromFiles([
os.path.join(self.computer_partition_root_path, hash_file)
for hash_file in [
'software_release/buildout.cfg',
]
])
with self.slap.instance_supervisor_rpc as supervisor: with self.slap.instance_supervisor_rpc as supervisor:
process_names = [process['name'] running_process_info = '\n'.join(sorted([
for process in supervisor.getAllProcessInfo()] '%(group)s:%(name)s %(statename)s' % q for q
in supervisor.getAllProcessInfo()
if q['name'] != 'watchdog' and q['group'] != 'watchdog']))
return running_process_info.replace(hash_value, '{hash}')
hash_files = [os.path.join(self.computer_partition_root_path, path)
for path in hash_files]
for name in expected_process_names: @skipUnlessKvm
h = generateHashFromFiles(hash_files) class TestInstance(InstanceTestCase, KvmMixin):
expected_process_name = name.format(hash=h) __partition_reference__ = 'i'
self.assertIn(expected_process_name, process_names) def test(self):
connection_parameter_dict = self\
.computer_partition.getConnectionParameterDict()
present_key_list = []
assert_key_list = [
'backend-url', 'url', 'monitor-setup-url', 'ipv6-network-info',
'tap-ipv4', 'tap-ipv6']
for k in assert_key_list:
if k in connection_parameter_dict:
present_key_list.append(k)
connection_parameter_dict.pop(k)
self.assertEqual(
connection_parameter_dict,
{
'ipv6': self._ipv6_address,
'maximum-extra-disk-amount': '0',
'monitor-base-url': 'https://[%s]:8026' % (self._ipv6_address,),
'nat-rule-port-tcp-22': '%s : 10022' % (self._ipv6_address,),
'nat-rule-port-tcp-443': '%s : 10443' % (self._ipv6_address,),
'nat-rule-port-tcp-80': '%s : 10080' % (self._ipv6_address,),
}
)
self.assertEqual(set(present_key_list), set(assert_key_list))
self.assertEqual(
"""i0:6tunnel-10022-{hash}-on-watch RUNNING
i0:6tunnel-10080-{hash}-on-watch RUNNING
i0:6tunnel-10443-{hash}-on-watch RUNNING
i0:bootstrap-monitor EXITED
i0:certificate_authority-{hash}-on-watch RUNNING
i0:crond-{hash}-on-watch RUNNING
i0:kvm-{hash}-on-watch RUNNING
i0:kvm_controller EXITED
i0:monitor-httpd-{hash}-on-watch RUNNING
i0:monitor-httpd-graceful EXITED
i0:websockify-{hash}-on-watch RUNNING
i0:whitelist-domains-download-{hash} RUNNING
i0:whitelist-firewall-{hash} RUNNING""",
self.getProcessInfo()
)
class MonitorAccessMixin(object): class MonitorAccessMixin(object):
...@@ -394,7 +434,7 @@ class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase): ...@@ -394,7 +434,7 @@ class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase):
@skipIfPython3 @skipIfPython3
@skipUnlessKvm @skipUnlessKvm
class TestInstanceResilient(InstanceTestCase): class TestInstanceResilient(InstanceTestCase, KvmMixin):
__partition_reference__ = 'ir' __partition_reference__ = 'ir'
instance_max_retry = 20 instance_max_retry = 20
...@@ -403,22 +443,76 @@ class TestInstanceResilient(InstanceTestCase): ...@@ -403,22 +443,76 @@ class TestInstanceResilient(InstanceTestCase):
return 'kvm-resilient' return 'kvm-resilient'
def test(self): def test(self):
# just check that keys returned on requested partition are for resilient connection_parameter_dict = self\
self.assertSetEqual( .computer_partition.getConnectionParameterDict()
set(self.computer_partition.getConnectionParameterDict().keys()), present_key_list = []
set([ assert_key_list = [
'backend-url', 'monitor-password', 'takeover-kvm-1-password', 'backend-url', 'url',
'feed-url-kvm-1-pull', 'monitor-setup-url', 'ipv6-network-info']
'feed-url-kvm-1-push', for k in assert_key_list:
'ipv6', if k in connection_parameter_dict:
'ipv6-network-info', present_key_list.append(k)
'monitor-base-url', connection_parameter_dict.pop(k)
'monitor-password', self.assertEqual(
'monitor-setup-url', connection_parameter_dict,
'monitor-user', {
'takeover-kvm-1-password', 'feed-url-kvm-1-pull': 'http://[%s]:8088/get/local-ir0-kvm-1-pull' % (
'takeover-kvm-1-url', self._ipv6_address,),
'url'])) 'feed-url-kvm-1-push': 'http://[%s]:8088/get/local-ir0-kvm-1-push' % (
self._ipv6_address,),
'ipv6': self._ipv6_address,
'monitor-base-url': 'https://[%s]:8160' % (self._ipv6_address,),
'monitor-user': 'admin',
'takeover-kvm-1-url': 'http://[%s]:9263/' % (self._ipv6_address,),
}
)
self.assertEqual(set(present_key_list), set(assert_key_list))
self.assertEqual(
"""ir0:bootstrap-monitor EXITED
ir0:certificate_authority-{hash}-on-watch RUNNING
ir0:crond-{hash}-on-watch RUNNING
ir0:monitor-httpd-{hash}-on-watch RUNNING
ir0:monitor-httpd-graceful EXITED
ir1:bootstrap-monitor EXITED
ir1:certificate_authority-{hash}-on-watch RUNNING
ir1:crond-{hash}-on-watch RUNNING
ir1:equeue-on-watch RUNNING
ir1:monitor-httpd-{hash}-on-watch RUNNING
ir1:monitor-httpd-graceful EXITED
ir1:notifier-on-watch RUNNING
ir1:pbs_sshkeys_authority-on-watch RUNNING
ir2:6tunnel-10022-{hash}-on-watch RUNNING
ir2:6tunnel-10080-{hash}-on-watch RUNNING
ir2:6tunnel-10443-{hash}-on-watch RUNNING
ir2:bootstrap-monitor EXITED
ir2:certificate_authority-{hash}-on-watch RUNNING
ir2:crond-{hash}-on-watch RUNNING
ir2:equeue-on-watch RUNNING
ir2:kvm-{hash}-on-watch RUNNING
ir2:kvm_controller EXITED
ir2:monitor-httpd-{hash}-on-watch RUNNING
ir2:monitor-httpd-graceful EXITED
ir2:notifier-on-watch RUNNING
ir2:resilient_sshkeys_authority-on-watch RUNNING
ir2:sshd-graceful EXITED
ir2:sshd-on-watch RUNNING
ir2:websockify-{hash}-on-watch RUNNING
ir2:whitelist-domains-download-{hash} RUNNING
ir2:whitelist-firewall-{hash} RUNNING
ir3:bootstrap-monitor EXITED
ir3:certificate_authority-{hash}-on-watch RUNNING
ir3:crond-{hash}-on-watch RUNNING
ir3:equeue-on-watch RUNNING
ir3:monitor-httpd-{hash}-on-watch RUNNING
ir3:monitor-httpd-graceful EXITED
ir3:notifier-on-watch RUNNING
ir3:resilient-web-takeover-httpd-on-watch RUNNING
ir3:resilient_sshkeys_authority-on-watch RUNNING
ir3:sshd-graceful EXITED
ir3:sshd-on-watch RUNNING""",
self.getProcessInfo()
)
@skipIfPython3 @skipIfPython3
...@@ -485,9 +579,74 @@ class TestInstanceNbdServer(InstanceTestCase): ...@@ -485,9 +579,74 @@ class TestInstanceNbdServer(InstanceTestCase):
self.assertIn("WARNING", connection_parameter_dict['status_message']) self.assertIn("WARNING", connection_parameter_dict['status_message'])
class FakeImageHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
def log_message(self, *args):
if os.environ.get('SLAPOS_TEST_DEBUG'):
return SimpleHTTPServer.SimpleHTTPRequestHandler.log_message(self, *args)
else:
return
class FakeImageServerMixin(object):
def rerequestInstance(self, parameter_dict, state='started'):
software_url = self.getSoftwareURL()
software_type = self.getInstanceSoftwareType()
return self.slap.request(
software_release=software_url,
software_type=software_type,
partition_reference=self.default_partition_reference,
partition_parameter_kw=parameter_dict,
state=state)
def startImageHttpServer(self):
self.image_source_directory = tempfile.mkdtemp()
server = SocketServer.TCPServer(
(self._ipv4_address, findFreeTCPPort(self._ipv4_address)),
FakeImageHandler)
fake_image_content = 'fake_image_content'
self.fake_image_md5sum = hashlib.md5(fake_image_content).hexdigest()
with open(os.path.join(
self.image_source_directory, self.fake_image_md5sum), 'wb') as fh:
fh.write(fake_image_content)
fake_image2_content = 'fake_image2_content'
self.fake_image2_md5sum = hashlib.md5(fake_image2_content).hexdigest()
with open(os.path.join(
self.image_source_directory, self.fake_image2_md5sum), 'wb') as fh:
fh.write(fake_image2_content)
self.fake_image_wrong_md5sum = self.fake_image2_md5sum
url = 'http://%s:%s' % server.server_address
self.fake_image = '/'.join([url, self.fake_image_md5sum])
self.fake_image2 = '/'.join([url, self.fake_image2_md5sum])
old_dir = os.path.realpath(os.curdir)
os.chdir(self.image_source_directory)
try:
self.server_process = multiprocessing.Process(
target=server.serve_forever, name='FakeImageHttpServer')
self.server_process.start()
finally:
os.chdir(old_dir)
def stopImageHttpServer(self):
self.logger.debug('Stopping process %s' % (self.server_process,))
self.server_process.join(10)
self.server_process.terminate()
time.sleep(0.1)
if self.server_process.is_alive():
self.logger.warning(
'Process %s still alive' % (self.server_process, ))
shutil.rmtree(self.image_source_directory)
@skipUnlessKvm @skipUnlessKvm
class TestBootImageUrlList(InstanceTestCase): class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
__partition_reference__ = 'biul' __partition_reference__ = 'biul'
kvm_instance_partition_reference = 'biul0'
# variations # variations
key = 'boot-image-url-list' key = 'boot-image-url-list'
...@@ -520,6 +679,10 @@ class TestBootImageUrlList(InstanceTestCase): ...@@ -520,6 +679,10 @@ class TestBootImageUrlList(InstanceTestCase):
# start with empty, but working configuration # start with empty, but working configuration
return {} return {}
def setUp(self):
super(InstanceTestCase, self).setUp()
self.startImageHttpServer()
def tearDown(self): def tearDown(self):
# clean up the instance for other tests # clean up the instance for other tests
# 1st remove all images... # 1st remove all images...
...@@ -528,28 +691,8 @@ class TestBootImageUrlList(InstanceTestCase): ...@@ -528,28 +691,8 @@ class TestBootImageUrlList(InstanceTestCase):
# 2nd ...move instance to "default" state # 2nd ...move instance to "default" state
self.rerequestInstance({}) self.rerequestInstance({})
self.slap.waitForInstance(max_retry=10) self.slap.waitForInstance(max_retry=10)
self.stopImageHttpServer()
def rerequestInstance(self, parameter_dict, state='started'): super(InstanceTestCase, self).tearDown()
software_url = self.getSoftwareURL()
software_type = self.getInstanceSoftwareType()
return self.slap.request(
software_release=software_url,
software_type=software_type,
partition_reference=self.default_partition_reference,
partition_parameter_kw=parameter_dict,
state=state)
fake_image, = (
"https://shacache.nxdcdn.com/shacache/05105cd25d1ad798b71fd46a206c9b73d"
"a2c285a078af33d0e739525a595886785725a68811578bc21f75d0a97700a66d5e75bc"
"e5b2721ca4556a0734cb13e65",)
fake_image_md5sum = "c98825aa1b6c8087914d2bfcafec3058"
fake_image2, = (
"https://shacache.nxdcdn.com/shacache/54f8a83a32bbf52602d9d211d592ee705"
"99f0c6b6aafe99e44aeadb0c8d3036a0e673aa994ffdb28d9fb0de155720123f74d814"
"2a74b7675a8d8ca20476dba6e",)
fake_image2_md5sum = "d4316a4d05f527d987b9d6e43e4c2bc6"
fake_image_wrong_md5sum = "c98825aa1b6c8087914d2bfcafec3057"
def raising_waitForInstance(self, max_retry): def raising_waitForInstance(self, max_retry):
with self.assertRaises(SlapOSNodeCommandError): with self.assertRaises(SlapOSNodeCommandError):
...@@ -564,8 +707,10 @@ class TestBootImageUrlList(InstanceTestCase): ...@@ -564,8 +707,10 @@ class TestBootImageUrlList(InstanceTestCase):
self.rerequestInstance(partition_parameter_kw) self.rerequestInstance(partition_parameter_kw)
self.slap.waitForInstance(max_retry=10) self.slap.waitForInstance(max_retry=10)
# check that image is correctly downloaded and linked # check that image is correctly downloaded and linked
kvm_instance_partition = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference)
image_repository = os.path.join( image_repository = os.path.join(
self.computer_partition_root_path, 'srv', self.image_directory) kvm_instance_partition, 'srv', self.image_directory)
image = os.path.join(image_repository, self.fake_image_md5sum) image = os.path.join(image_repository, self.fake_image_md5sum)
image_link = os.path.join(image_repository, 'image_001') image_link = os.path.join(image_repository, 'image_001')
self.assertTrue(os.path.exists(image)) self.assertTrue(os.path.exists(image))
...@@ -597,25 +742,17 @@ class TestBootImageUrlList(InstanceTestCase): ...@@ -597,25 +742,17 @@ class TestBootImageUrlList(InstanceTestCase):
if entry.startswith('file') and 'media=cdrom' in entry: if entry.startswith('file') and 'media=cdrom' in entry:
# do cleanups # do cleanups
entry = entry.replace(software_root, '') entry = entry.replace(software_root, '')
entry = entry.replace(self.computer_partition_root_path, '') entry = entry.replace(kvm_instance_partition, '')
running_image_list.append(entry) running_image_list.append(entry)
return running_image_list return running_image_list
# check that the image is NOT YET available in kvm
self.assertEqual(
['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
'media=cdrom'],
getRunningImageList()
)
# mimic the requirement: restart the instance by requesting it stopped and # mimic the requirement: restart the instance by requesting it stopped and
# then started started, like user have to do it # then started started, like user have to do it
self.rerequestInstance(partition_parameter_kw, state='stopped') self.rerequestInstance(partition_parameter_kw, state='stopped')
self.slap.waitForInstance(max_retry=1) self.slap.waitForInstance(max_retry=1)
self.rerequestInstance(partition_parameter_kw, state='started') self.rerequestInstance(partition_parameter_kw, state='started')
self.slap.waitForInstance(max_retry=1) self.slap.waitForInstance(max_retry=3)
# now the image is available in the kvm, and its above default image
self.assertEqual( self.assertEqual(
[ [
'file=/srv/%s/image_001,media=cdrom' % (self.image_directory,), 'file=/srv/%s/image_001,media=cdrom' % (self.image_directory,),
...@@ -628,7 +765,8 @@ class TestBootImageUrlList(InstanceTestCase): ...@@ -628,7 +765,8 @@ class TestBootImageUrlList(InstanceTestCase):
# cleanup of images works, also asserts that configuration changes are # cleanup of images works, also asserts that configuration changes are
# reflected # reflected
self.rerequestInstance({self.key: ''}) partition_parameter_kw[self.key] = ''
self.rerequestInstance(partition_parameter_kw)
self.slap.waitForInstance(max_retry=2) self.slap.waitForInstance(max_retry=2)
self.assertEqual( self.assertEqual(
os.listdir(image_repository), os.listdir(image_repository),
...@@ -640,7 +778,7 @@ class TestBootImageUrlList(InstanceTestCase): ...@@ -640,7 +778,7 @@ class TestBootImageUrlList(InstanceTestCase):
self.rerequestInstance(partition_parameter_kw, state='stopped') self.rerequestInstance(partition_parameter_kw, state='stopped')
self.slap.waitForInstance(max_retry=1) self.slap.waitForInstance(max_retry=1)
self.rerequestInstance(partition_parameter_kw, state='started') self.rerequestInstance(partition_parameter_kw, state='started')
self.slap.waitForInstance(max_retry=1) self.slap.waitForInstance(max_retry=3)
# again only default image is available in the running process # again only default image is available in the running process
self.assertEqual( self.assertEqual(
...@@ -650,12 +788,15 @@ class TestBootImageUrlList(InstanceTestCase): ...@@ -650,12 +788,15 @@ class TestBootImageUrlList(InstanceTestCase):
) )
def assertPromiseFails(self, promise): def assertPromiseFails(self, promise):
partition_directory = os.path.join(
self.slap.instance_directory,
self.kvm_instance_partition_reference)
monitor_run_promise = os.path.join( monitor_run_promise = os.path.join(
self.computer_partition_root_path, 'software_release', 'bin', partition_directory, 'software_release', 'bin',
'monitor.runpromise' 'monitor.runpromise'
) )
monitor_configuration = os.path.join( monitor_configuration = os.path.join(
self.computer_partition_root_path, 'etc', 'monitor.conf') partition_directory, 'etc', 'monitor.conf')
self.assertNotEqual( self.assertNotEqual(
0, 0,
...@@ -708,9 +849,19 @@ class TestBootImageUrlList(InstanceTestCase): ...@@ -708,9 +849,19 @@ class TestBootImageUrlList(InstanceTestCase):
self.assertPromiseFails(self.config_state_promise) self.assertPromiseFails(self.config_state_promise)
@skipIfPython3
@skipUnlessKvm
class TestBootImageUrlListResilient(TestBootImageUrlList):
kvm_instance_partition_reference = 'biul2'
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-resilient'
@skipUnlessKvm @skipUnlessKvm
class TestBootImageUrlSelect(TestBootImageUrlList): class TestBootImageUrlSelect(TestBootImageUrlList):
__partition_reference__ = 'bius' __partition_reference__ = 'bius'
kvm_instance_partition_reference = 'bius0'
# variations # variations
key = 'boot-image-url-select' key = 'boot-image-url-select'
...@@ -754,7 +905,8 @@ class TestBootImageUrlSelect(TestBootImageUrlList): ...@@ -754,7 +905,8 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
for image_directory in [ for image_directory in [
'boot-image-url-list-repository', 'boot-image-url-select-repository']: 'boot-image-url-list-repository', 'boot-image-url-select-repository']:
image_repository = os.path.join( image_repository = os.path.join(
self.computer_partition_root_path, 'srv', image_directory) self.slap.instance_directory, self.kvm_instance_partition_reference,
'srv', image_directory)
image = os.path.join(image_repository, self.fake_image_md5sum) image = os.path.join(image_repository, self.fake_image_md5sum)
image_link = os.path.join(image_repository, 'image_001') image_link = os.path.join(image_repository, 'image_001')
self.assertTrue(os.path.exists(image)) self.assertTrue(os.path.exists(image))
...@@ -764,6 +916,9 @@ class TestBootImageUrlSelect(TestBootImageUrlList): ...@@ -764,6 +916,9 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
self.assertTrue(os.path.islink(image_link)) self.assertTrue(os.path.islink(image_link))
self.assertEqual(os.readlink(image_link), image) self.assertEqual(os.readlink(image_link), image)
kvm_instance_partition = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference)
def getRunningImageList(): def getRunningImageList():
running_image_list = [] running_image_list = []
with self.slap.instance_supervisor_rpc as instance_supervisor: with self.slap.instance_supervisor_rpc as instance_supervisor:
...@@ -777,25 +932,17 @@ class TestBootImageUrlSelect(TestBootImageUrlList): ...@@ -777,25 +932,17 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
if entry.startswith('file') and 'media=cdrom' in entry: if entry.startswith('file') and 'media=cdrom' in entry:
# do cleanups # do cleanups
entry = entry.replace(software_root, '') entry = entry.replace(software_root, '')
entry = entry.replace(self.computer_partition_root_path, '') entry = entry.replace(kvm_instance_partition, '')
running_image_list.append(entry) running_image_list.append(entry)
return running_image_list return running_image_list
# check that the image is NOT YET available in kvm
self.assertEqual(
['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
'media=cdrom'],
getRunningImageList()
)
# mimic the requirement: restart the instance by requesting it stopped and # mimic the requirement: restart the instance by requesting it stopped and
# then started started, like user have to do it # then started started, like user have to do it
self.rerequestInstance(partition_parameter_kw, state='stopped') self.rerequestInstance(partition_parameter_kw, state='stopped')
self.slap.waitForInstance(max_retry=1) self.slap.waitForInstance(max_retry=1)
self.rerequestInstance(partition_parameter_kw, state='started') self.rerequestInstance(partition_parameter_kw, state='started')
self.slap.waitForInstance(max_retry=1) self.slap.waitForInstance(max_retry=3)
# now the image is available in the kvm, and its above default image
self.assertEqual( self.assertEqual(
[ [
'file=/srv/boot-image-url-select-repository/image_001,media=cdrom', 'file=/srv/boot-image-url-select-repository/image_001,media=cdrom',
...@@ -814,18 +961,29 @@ class TestBootImageUrlSelect(TestBootImageUrlList): ...@@ -814,18 +961,29 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
for image_directory in [ for image_directory in [
'boot-image-url-list-repository', 'boot-image-url-select-repository']: 'boot-image-url-list-repository', 'boot-image-url-select-repository']:
image_repository = os.path.join( image_repository = os.path.join(
self.computer_partition_root_path, 'srv', image_directory) kvm_instance_partition, 'srv', image_directory)
self.assertEqual( self.assertEqual(
os.listdir(image_repository), os.listdir(image_repository),
[] []
) )
# cleanup of images works, also asserts that configuration changes are
# reflected
partition_parameter_kw[self.key] = ''
partition_parameter_kw['boot-image-url-list'] = ''
self.rerequestInstance(partition_parameter_kw)
self.slap.waitForInstance(max_retry=2)
self.assertEqual(
os.listdir(image_repository),
[]
)
# mimic the requirement: restart the instance by requesting it stopped and # mimic the requirement: restart the instance by requesting it stopped and
# then started started, like user have to do it # then started started, like user have to do it
self.rerequestInstance(partition_parameter_kw, state='stopped') self.rerequestInstance(partition_parameter_kw, state='stopped')
self.slap.waitForInstance(max_retry=1) self.slap.waitForInstance(max_retry=1)
self.rerequestInstance(partition_parameter_kw, state='started') self.rerequestInstance(partition_parameter_kw, state='started')
self.slap.waitForInstance(max_retry=1) self.slap.waitForInstance(max_retry=3)
# again only default image is available in the running process # again only default image is available in the running process
self.assertEqual( self.assertEqual(
...@@ -835,42 +993,44 @@ class TestBootImageUrlSelect(TestBootImageUrlList): ...@@ -835,42 +993,44 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
) )
@skipIfPython3
@skipUnlessKvm @skipUnlessKvm
class TestBootImageUrlListKvmCluster(InstanceTestCase): class TestBootImageUrlSelectResilient(TestBootImageUrlSelect):
kvm_instance_partition_reference = 'bius2'
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-resilient'
@skipUnlessKvm
class TestBootImageUrlListKvmCluster(InstanceTestCase, FakeImageServerMixin):
__partition_reference__ = 'biulkc' __partition_reference__ = 'biulkc'
@classmethod @classmethod
def getInstanceSoftwareType(cls): def getInstanceSoftwareType(cls):
return 'kvm-cluster' return 'kvm-cluster'
fake_image, = (
"https://shacache.nxdcdn.com/shacache/05105cd25d1ad798b71fd46a206c9b73d"
"a2c285a078af33d0e739525a595886785725a68811578bc21f75d0a97700a66d5e75bc"
"e5b2721ca4556a0734cb13e65",)
fake_image_md5sum = "c98825aa1b6c8087914d2bfcafec3058"
fake_image2, = (
"https://shacache.nxdcdn.com/shacache/54f8a83a32bbf52602d9d211d592ee705"
"99f0c6b6aafe99e44aeadb0c8d3036a0e673aa994ffdb28d9fb0de155720123f74d814"
"2a74b7675a8d8ca20476dba6e",)
fake_image2_md5sum = "d4316a4d05f527d987b9d6e43e4c2bc6"
input_value = "%s#%s" input_value = "%s#%s"
key = 'boot-image-url-list' key = 'boot-image-url-list'
config_file_name = 'boot-image-url-list.conf' config_file_name = 'boot-image-url-list.conf'
def setUp(self):
super(InstanceTestCase, self).setUp()
self.startImageHttpServer()
def tearDown(self):
self.stopImageHttpServer()
super(InstanceTestCase, self).tearDown()
@classmethod @classmethod
def getInstanceParameterDict(cls): def getInstanceParameterDict(cls):
return {'_': json.dumps({ return {'_': json.dumps({
"kvm-partition-dict": { "kvm-partition-dict": {
"KVM0": { "KVM0": {
"disable-ansible-promise": True, "disable-ansible-promise": True,
cls.key: cls.input_value % (
cls.fake_image, cls.fake_image_md5sum)
}, },
"KVM1": { "KVM1": {
"disable-ansible-promise": True, "disable-ansible-promise": True,
cls.key: cls.input_value % (
cls.fake_image2, cls.fake_image2_md5sum)
} }
} }
})} })}
...@@ -878,6 +1038,21 @@ class TestBootImageUrlListKvmCluster(InstanceTestCase): ...@@ -878,6 +1038,21 @@ class TestBootImageUrlListKvmCluster(InstanceTestCase):
def test(self): def test(self):
# Note: As there is no way to introspect nicely where partition landed # Note: As there is no way to introspect nicely where partition landed
# we assume ordering of the cluster requests # we assume ordering of the cluster requests
self.rerequestInstance({'_': json.dumps({
"kvm-partition-dict": {
"KVM0": {
"disable-ansible-promise": True,
self.key: self.input_value % (
self.fake_image, self.fake_image_md5sum)
},
"KVM1": {
"disable-ansible-promise": True,
self.key: self.input_value % (
self.fake_image2, self.fake_image2_md5sum)
}
}
})})
self.slap.waitForInstance(max_retry=10)
KVM0_config = os.path.join( KVM0_config = os.path.join(
self.slap.instance_directory, self.__partition_reference__ + '1', 'etc', self.slap.instance_directory, self.__partition_reference__ + '1', 'etc',
self.config_file_name) self.config_file_name)
...@@ -999,3 +1174,107 @@ class TestNatRulesKvmCluster(InstanceTestCase): ...@@ -999,3 +1174,107 @@ class TestNatRulesKvmCluster(InstanceTestCase):
class TestNatRulesKvmClusterComplex(TestNatRulesKvmCluster): class TestNatRulesKvmClusterComplex(TestNatRulesKvmCluster):
__partition_reference__ = 'nrkcc' __partition_reference__ = 'nrkcc'
nat_rules = ["100", "200 300"] nat_rules = ["100", "200 300"]
@skipUnlessKvm
class TestWhitelistFirewall(InstanceTestCase):
__partition_reference__ = 'wf'
kvm_instance_partition_reference = 'wf0'
def test(self):
slapos_whitelist_firewall = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference,
'.slapos-whitelist-firewall')
self.assertTrue(os.path.exists(slapos_whitelist_firewall))
with open(slapos_whitelist_firewall, 'rb') as fh:
content = fh.read().encode('utf-8')
try:
self.content_json = json.loads(content)
except ValueError:
self.fail('Failed to parse json of %s' % (content,))
self.assertTrue(isinstance(self.content_json, list))
# check /etc/resolv.conf
with open('/etc/resolv.conf', 'rb') as fh:
resolv_conf_ip_list = []
for line in fh.readlines():
line = line.encode('utf-8')
if line.startswith('nameserver'):
resolv_conf_ip_list.append(line.split()[1])
resolv_conf_ip_list = list(set(resolv_conf_ip_list))
self.assertFalse(len(resolv_conf_ip_list) == 0)
self.assertTrue(all([q in self.content_json for q in resolv_conf_ip_list]))
# there is something more
self.assertGreater(len(self.content_json), len(resolv_conf_ip_list))
@skipUnlessKvm
class TestWhitelistFirewallRequest(TestWhitelistFirewall):
whitelist_domains = '2.2.2.2 3.3.3.3\n4.4.4.4'
@classmethod
def getInstanceParameterDict(cls):
return {
'whitelist-domains': cls.whitelist_domains,
}
def test(self):
super(TestWhitelistFirewallRequest, self).test()
self.assertIn('2.2.2.2', self.content_json)
self.assertIn('3.3.3.3', self.content_json)
self.assertIn('4.4.4.4', self.content_json)
@skipUnlessKvm
class TestWhitelistFirewallResilient(TestWhitelistFirewall):
kvm_instance_partition_reference = 'wf2'
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-resilient'
@skipUnlessKvm
class TestWhitelistFirewallRequestResilient(TestWhitelistFirewallRequest):
kvm_instance_partition_reference = 'wf2'
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-resilient'
@skipUnlessKvm
class TestWhitelistFirewallCluster(TestWhitelistFirewall):
kvm_instance_partition_reference = 'wf1'
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-cluster'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({
"kvm-partition-dict": {
"KVM0": {
"disable-ansible-promise": True
}
}
})}
@skipUnlessKvm
class TestWhitelistFirewallRequestCluster(TestWhitelistFirewallRequest):
kvm_instance_partition_reference = 'wf1'
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-cluster'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({
"kvm-partition-dict": {
"KVM0": {
"whitelist-domains": cls.whitelist_domains,
"disable-ansible-promise": True
}
}
})}
...@@ -5,6 +5,7 @@ extends = ...@@ -5,6 +5,7 @@ extends =
../../component/openssl/buildout.cfg ../../component/openssl/buildout.cfg
../../component/logrotate/buildout.cfg ../../component/logrotate/buildout.cfg
../../component/apache/buildout.cfg ../../component/apache/buildout.cfg
../../component/geoip2/buildout.cfg
../../component/git/buildout.cfg ../../component/git/buildout.cfg
../../component/python-cryptography/buildout.cfg ../../component/python-cryptography/buildout.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
...@@ -36,11 +37,13 @@ eggs = ...@@ -36,11 +37,13 @@ eggs =
${python-cffi:egg} ${python-cffi:egg}
${python-cryptography:egg} ${python-cryptography:egg}
pyOpenSSL pyOpenSSL
re6stnet re6stnet[geoip]
initialization = initialization =
import os import os
path = os.environ.get('PATH', '') path = os.environ.get('PATH', '/bin:/usr/bin')
os.environ['PATH'] = '${git:location}/bin' + (path and ':' + path) os.environ['PATH'] = '${openssl:location}/bin:${git:location}/bin' + (
path and ':' + path)
os.environ['GEOIP2_MMDB'] = "${geolite2-country:location}/GeoLite2-Country.mmdb"
depends = ${re6stnet-develop:recipe} depends = ${re6stnet-develop:recipe}
[re6stnet-backup] [re6stnet-backup]
...@@ -59,6 +62,7 @@ template = ...@@ -59,6 +62,7 @@ template =
h=`sqlite3 "$2" .dump |git hash-object --stdin -w` h=`sqlite3 "$2" .dump |git hash-object --stdin -w`
git update-index --add --cacheinfo 0644 "$h" registry.sql git update-index --add --cacheinfo 0644 "$h" registry.sql
git diff --cached --quiet || GIT_WORK_TREE=$PWD GIT_DIR=$PWD git \ git diff --cached --quiet || GIT_WORK_TREE=$PWD GIT_DIR=$PWD git \
-c user.name="$${0##*/}" -c user.email="$USER@`hostname`" \
-c gc.auto=100 -c gc.autoDetach=false commit --allow-empty-message -qm '' -c gc.auto=100 -c gc.autoDetach=false commit --allow-empty-message -qm ''
[ ! "$3" ] || git push --mirror "$3" [ ! "$3" ] || git push --mirror "$3"
context = context =
......
...@@ -216,9 +216,9 @@ slapos.rebootstrap = 4.5 ...@@ -216,9 +216,9 @@ slapos.rebootstrap = 4.5
slapos.recipe.build = 0.46 slapos.recipe.build = 0.46
slapos.recipe.cmmi = 0.17 slapos.recipe.cmmi = 0.17
slapos.recipe.template = 4.5 slapos.recipe.template = 4.5
slapos.toolbox = 0.117 slapos.toolbox = 0.118
stevedore = 1.21.0 stevedore = 1.21.0
subprocess32 = 3.5.3 subprocess32 = 3.5.4
unicodecsv = 0.14.1 unicodecsv = 0.14.1
wcwidth = 0.2.5 wcwidth = 0.2.5
wheel = 0.35.1 wheel = 0.35.1
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment