Commit 72782b62 authored by Łukasz Nowak's avatar Łukasz Nowak

Feature/kvm whitelist firewall

See merge request !930
parents d77444b1 97225683
...@@ -15,15 +15,15 @@ ...@@ -15,15 +15,15 @@
[template] [template]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = e6d5c7bb627b4f1d3e7c99721b7c58fe md5sum = 399b398a8eabfa6126d2a521dc779f9b
[template-kvm] [template-kvm]
filename = instance-kvm.cfg.jinja2 filename = instance-kvm.cfg.jinja2
md5sum = 89796d6d6a25f694291f9b45181830ad md5sum = 704b6ac6bf42837bcd8f4582c5a746c0
[template-kvm-cluster] [template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in filename = instance-kvm-cluster.cfg.jinja2.in
md5sum = 8a6c5555efd63ac7d471b8fdabb69f7e md5sum = 80b9b70ba1ccbc09deb8f9cad60f352c
[template-kvm-resilient] [template-kvm-resilient]
filename = instance-kvm-resilient.cfg.jinja2 filename = instance-kvm-resilient.cfg.jinja2
...@@ -88,3 +88,11 @@ md5sum = 9c67058edcc4edae0b57956c0932a9fc ...@@ -88,3 +88,11 @@ md5sum = 9c67058edcc4edae0b57956c0932a9fc
[image-download-config-creator] [image-download-config-creator]
_update_hash_filename_ = template/image-download-config-creator.py _update_hash_filename_ = template/image-download-config-creator.py
md5sum = 54261e418ab9860efe73efd514c4d47f md5sum = 54261e418ab9860efe73efd514c4d47f
[whitelist-firewall-download-controller]
_update_hash_filename_ = template/whitelist-firewall-download-controller.py
md5sum = bc64e29546833817636261d1b28aa6dc
[whitelist-domains-default]
_update_hash_filename_ = template/whitelist-domains-default
md5sum = e9d40162ba77472775256637a2617d14
...@@ -556,6 +556,12 @@ ...@@ -556,6 +556,12 @@
"title": "FreeBSD 12.1 RELEASE bootonly x86_64" "title": "FreeBSD 12.1 RELEASE bootonly x86_64"
} }
] ]
},
"whitelist-domains": {
"title": "Whitelist domains",
"description": "List of whitelisted domain names to be accessed from the VM. They will be resolved to IPs depending on where the VM end up. IPs can be used too.",
"type": "string",
"textarea": true
} }
}, },
"type": "object" "type": "object"
......
...@@ -107,6 +107,12 @@ ...@@ -107,6 +107,12 @@
"title": "FreeBSD 12.1 RELEASE bootonly x86_64" "title": "FreeBSD 12.1 RELEASE bootonly x86_64"
} }
] ]
},
"whitelist-domains": {
"title": "Whitelist domains",
"description": "List of whitelisted domain names to be accessed from the VM. They will be resolved to IPs depending on where the VM end up. IPs can be used too.",
"type": "string",
"textarea": true
} }
}, },
"type": "object" "type": "object"
......
...@@ -135,7 +135,7 @@ config-document-host = ${apache-conf:ip} ...@@ -135,7 +135,7 @@ config-document-host = ${apache-conf:ip}
config-document-port = ${apache-conf:port} config-document-port = ${apache-conf:port}
config-document-path = ${hash-code:passwd} config-document-path = ${hash-code:passwd}
config-keyboard-layout-language = {{ dumps(kvm_parameter_dict.get('keyboard-layout-language', 'fr')) }} config-keyboard-layout-language = {{ dumps(kvm_parameter_dict.get('keyboard-layout-language', 'fr')) }}
{%- for k in ['boot-image-url-list', 'boot-image-url-select'] %} {%- for k in ['boot-image-url-list', 'boot-image-url-select', 'whitelist-domains'] %}
{#- play nice - use parameter only if present #} {#- play nice - use parameter only if present #}
{%- if k in kvm_parameter_dict %} {%- if k in kvm_parameter_dict %}
{#- play safe - dumps value #} {#- play safe - dumps value #}
......
...@@ -419,6 +419,12 @@ ...@@ -419,6 +419,12 @@
"title": "FreeBSD 12.1 RELEASE bootonly x86_64" "title": "FreeBSD 12.1 RELEASE bootonly x86_64"
} }
] ]
},
"whitelist-domains": {
"title": "Whitelist domains",
"description": "List of whitelisted domain names to be accessed from the VM. They will be resolved to IPs depending on where the VM end up. IPs can be used too.",
"type": "string",
"textarea": true
} }
} }
} }
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
{% set instance_type = slapparameter_dict.get('type', 'standalone') -%} {% set instance_type = slapparameter_dict.get('type', 'standalone') -%}
{% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%} {% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
{% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%} {% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%}
{% set whitelist_domains = slapparameter_dict.get('whitelist-domains', '') -%}
{% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %} {% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %}
{% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %} {% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %}
{% set cpu_max_count = dumps(slapparameter_dict.get('cpu-max-count', int(slapparameter_dict.get('cpu-count', 1)) + 1)) %} {% set cpu_max_count = dumps(slapparameter_dict.get('cpu-max-count', int(slapparameter_dict.get('cpu-count', 1)) + 1)) %}
...@@ -972,7 +973,7 @@ keyboard-layout-language = fr ...@@ -972,7 +973,7 @@ keyboard-layout-language = fr
{% set key_list = v.split('\n') -%} {% set key_list = v.split('\n') -%}
{{ k }} = {{ k }} =
{{ key_list | join('\n ') }} {{ key_list | join('\n ') }}
{% elif k in ['boot-image-url-list', 'boot-image-url-select'] %} {% elif k in ['boot-image-url-list', 'boot-image-url-select', 'whitelist-domains'] %}
{# needs to decorate possibly multiline or maybe unsafe value #} {# needs to decorate possibly multiline or maybe unsafe value #}
{{ k }} = {{ dumps(v) }} {{ k }} = {{ dumps(v) }}
{% else -%} {% else -%}
...@@ -1038,8 +1039,62 @@ command-line = ...@@ -1038,8 +1039,62 @@ command-line =
{% endif -%} {% endif -%}
{% do part_list.append('whitelist-firewall') -%}
[whitelist-firewall]
recipe = slapos.cookbook:wrapper
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
wrapper-path = ${directory:scripts}/${:_buildout_section_name_}
command-line =
{{ dnsresolver_executable }}
--style list
--output ${:output}
${:source}
source =
${whitelist-domains-request:rendered}
{{ whitelist_domains_default }}
${whitelist-domains-resolv.conf:output}
${whitelist-domains-download:output}
output = ${buildout:directory}/.slapos-whitelist-firewall
[whitelist-firewall-directory]
recipe = plone.recipe.command
location = ${buildout:parts-directory}/whitelist-firewall
command = mkdir -p ${:location}
update-command = ${:command}
[whitelist-domains-request]
recipe = slapos.recipe.template:jinja2
template = inline:
{%- raw %}
{%- for domain in whitelist_domains.split() %}
{{ domain }}
{%- endfor %}
{% endraw -%}
rendered = ${whitelist-firewall-directory:location}/${:_buildout_section_name_}.txt
whitelist-domains = {{ dumps(whitelist_domains) }}
extensions = jinja2.ext.do
context =
key whitelist_domains :whitelist-domains
[whitelist-domains-resolv.conf]
recipe = plone.recipe.command
output = ${whitelist-firewall-directory:location}/${:_buildout_section_name_}.txt
update-command = ${:command}
command =
egrep ^nameserver /etc/resolv.conf | cut -d ' ' -f 2 > ${:output}
[whitelist-domains-download]
recipe = slapos.cookbook:wrapper
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
wrapper-path = ${directory:scripts}/${:_buildout_section_name_}
output = ${whitelist-firewall-directory:location}/${:_buildout_section_name_}.txt
interval = 3600
command-line = {{ python_executable }} {{ whitelist_firewall_download_controller }} {{ curl_executable_location }} 3600 ${:output} ${:url}
url = https://stream.nxdcdn.com/rapidspace-whitelist-domains
[instance-kvm-parts] [instance-kvm-parts]
# Expose parts for easy addition in profiles which extend this one like resilient
parts = parts =
certificate-authority certificate-authority
certificate-authority-service certificate-authority-service
......
...@@ -80,9 +80,12 @@ extra-context = ...@@ -80,9 +80,12 @@ extra-context =
raw ansible_promise_tpl ${template-ansible-promise:location}/${template-ansible-promise:filename} raw ansible_promise_tpl ${template-ansible-promise:location}/${template-ansible-promise:filename}
raw curl_executable_location ${curl:location}/bin/curl raw curl_executable_location ${curl:location}/bin/curl
raw dash_executable_location ${dash:location}/bin/dash raw dash_executable_location ${dash:location}/bin/dash
raw dnsresolver_executable ${buildout:bin-directory}/dnsresolver
raw dcron_executable_location ${dcron:location}/sbin/crond raw dcron_executable_location ${dcron:location}/sbin/crond
raw debian_amd64_netinst_location ${debian-amd64-netinst.iso:location}/${debian-amd64-netinst.iso:filename} raw debian_amd64_netinst_location ${debian-amd64-netinst.iso:location}/${debian-amd64-netinst.iso:filename}
raw file_download_script ${file-download-script:location}/${file-download-script:filename} raw file_download_script ${file-download-script:location}/${file-download-script:filename}
raw whitelist_domains_default ${whitelist-domains-default:location}/${whitelist-domains-default:filename}
raw whitelist_firewall_download_controller ${whitelist-firewall-download-controller:target}
raw image_download_controller ${image-download-controller:target} raw image_download_controller ${image-download-controller:target}
raw image_download_config_creator ${image-download-config-creator:target} raw image_download_config_creator ${image-download-config-creator:target}
raw logrotate_cfg ${template-logrotate-base:rendered} raw logrotate_cfg ${template-logrotate-base:rendered}
......
...@@ -182,6 +182,10 @@ path = download_file.in ...@@ -182,6 +182,10 @@ path = download_file.in
filename = download_file filename = download_file
on-update = true on-update = true
[whitelist-domains-default]
<= download-template-base
filename = whitelist-domains-default
[template-httpd] [template-httpd]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-kvm-http.cfg.in template = ${:_profile_base_location_}/instance-kvm-http.cfg.in
...@@ -199,6 +203,10 @@ mode = 640 ...@@ -199,6 +203,10 @@ mode = 640
[image-download-config-creator] [image-download-config-creator]
<= image-download-controller <= image-download-controller
[whitelist-firewall-download-controller]
<= image-download-controller
[versions] [versions]
websockify = 0.9.0 websockify = 0.9.0
......
# Minimal whitelisted domains needed to instantiate the instance
# Does not guarantee good usage of the guest VM itself
# The full list shall end up in whitelist-domains-download:url
# shcache.nxdcdn.com is default source for a lot of operations
shacache.nxdcdn.com
# stream.nxdcdn.com is needed by partition itself
stream.nxdcdn.com
# partition has to access default SlapOS Master
slap.vifib.com
slapos.vifib.com
# Partition needs access to SlapOS Master related resources
hnode.cdn.vifib.com
node.cdn.vifib.com
#!/usr/bin/env python
import os
import subprocess
import sys
import time
import logging
# Note: Assuring only one running instance is not done, as this script is only
# run from supervisord, which does it already
if __name__ == "__main__":
curl, sleep, output, url = sys.argv[1:]
sleep = int(sleep)
tmp_output = output + '.tmp'
logging.basicConfig(
format='%%(asctime)s [%%(levelname)s] %s : %%(message)s' % (url,),
level=logging.DEBUG)
logging.info('Redownloading each %is', sleep)
while True:
logging.info('Fetching')
try:
subprocess.check_output([
curl,
'--location', # follow redirects
'--no-progress-meter', # do not tell too much
'--max-time', '600', # 10 minutes is maximum
'--fail', # fail in case of wrong HTTP code
'--output', tmp_output, url],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
logging.error('Problem while downloading: %r', e.output.strip())
if os.path.exists(tmp_output):
logging.info('Stored output')
os.rename(tmp_output, output)
logging.info('Sleeping for %is', sleep)
time.sleep(sleep)
...@@ -168,7 +168,9 @@ i0:kvm-{hash}-on-watch RUNNING ...@@ -168,7 +168,9 @@ i0:kvm-{hash}-on-watch RUNNING
i0:kvm_controller EXITED i0:kvm_controller EXITED
i0:monitor-httpd-{hash}-on-watch RUNNING i0:monitor-httpd-{hash}-on-watch RUNNING
i0:monitor-httpd-graceful EXITED i0:monitor-httpd-graceful EXITED
i0:websockify-{hash}-on-watch RUNNING""", i0:websockify-{hash}-on-watch RUNNING
i0:whitelist-domains-download-{hash} RUNNING
i0:whitelist-firewall-{hash} RUNNING""",
self.getProcessInfo() self.getProcessInfo()
) )
...@@ -496,6 +498,8 @@ ir2:resilient_sshkeys_authority-on-watch RUNNING ...@@ -496,6 +498,8 @@ ir2:resilient_sshkeys_authority-on-watch RUNNING
ir2:sshd-graceful EXITED ir2:sshd-graceful EXITED
ir2:sshd-on-watch RUNNING ir2:sshd-on-watch RUNNING
ir2:websockify-{hash}-on-watch RUNNING ir2:websockify-{hash}-on-watch RUNNING
ir2:whitelist-domains-download-{hash} RUNNING
ir2:whitelist-firewall-{hash} RUNNING
ir3:bootstrap-monitor EXITED ir3:bootstrap-monitor EXITED
ir3:certificate_authority-{hash}-on-watch RUNNING ir3:certificate_authority-{hash}-on-watch RUNNING
ir3:crond-{hash}-on-watch RUNNING ir3:crond-{hash}-on-watch RUNNING
...@@ -1170,3 +1174,107 @@ class TestNatRulesKvmCluster(InstanceTestCase): ...@@ -1170,3 +1174,107 @@ class TestNatRulesKvmCluster(InstanceTestCase):
class TestNatRulesKvmClusterComplex(TestNatRulesKvmCluster): class TestNatRulesKvmClusterComplex(TestNatRulesKvmCluster):
__partition_reference__ = 'nrkcc' __partition_reference__ = 'nrkcc'
nat_rules = ["100", "200 300"] nat_rules = ["100", "200 300"]
@skipUnlessKvm
class TestWhitelistFirewall(InstanceTestCase):
__partition_reference__ = 'wf'
kvm_instance_partition_reference = 'wf0'
def test(self):
slapos_whitelist_firewall = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference,
'.slapos-whitelist-firewall')
self.assertTrue(os.path.exists(slapos_whitelist_firewall))
with open(slapos_whitelist_firewall, 'rb') as fh:
content = fh.read().encode('utf-8')
try:
self.content_json = json.loads(content)
except ValueError:
self.fail('Failed to parse json of %s' % (content,))
self.assertTrue(isinstance(self.content_json, list))
# check /etc/resolv.conf
with open('/etc/resolv.conf', 'rb') as fh:
resolv_conf_ip_list = []
for line in fh.readlines():
line = line.encode('utf-8')
if line.startswith('nameserver'):
resolv_conf_ip_list.append(line.split()[1])
resolv_conf_ip_list = list(set(resolv_conf_ip_list))
self.assertFalse(len(resolv_conf_ip_list) == 0)
self.assertTrue(all([q in self.content_json for q in resolv_conf_ip_list]))
# there is something more
self.assertGreater(len(self.content_json), len(resolv_conf_ip_list))
@skipUnlessKvm
class TestWhitelistFirewallRequest(TestWhitelistFirewall):
whitelist_domains = '2.2.2.2 3.3.3.3\n4.4.4.4'
@classmethod
def getInstanceParameterDict(cls):
return {
'whitelist-domains': cls.whitelist_domains,
}
def test(self):
super(TestWhitelistFirewallRequest, self).test()
self.assertIn('2.2.2.2', self.content_json)
self.assertIn('3.3.3.3', self.content_json)
self.assertIn('4.4.4.4', self.content_json)
@skipUnlessKvm
class TestWhitelistFirewallResilient(TestWhitelistFirewall):
kvm_instance_partition_reference = 'wf2'
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-resilient'
@skipUnlessKvm
class TestWhitelistFirewallRequestResilient(TestWhitelistFirewallRequest):
kvm_instance_partition_reference = 'wf2'
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-resilient'
@skipUnlessKvm
class TestWhitelistFirewallCluster(TestWhitelistFirewall):
kvm_instance_partition_reference = 'wf1'
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-cluster'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({
"kvm-partition-dict": {
"KVM0": {
"disable-ansible-promise": True
}
}
})}
@skipUnlessKvm
class TestWhitelistFirewallRequestCluster(TestWhitelistFirewallRequest):
kvm_instance_partition_reference = 'wf1'
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-cluster'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({
"kvm-partition-dict": {
"KVM0": {
"whitelist-domains": cls.whitelist_domains,
"disable-ansible-promise": True
}
}
})}
...@@ -216,9 +216,9 @@ slapos.rebootstrap = 4.5 ...@@ -216,9 +216,9 @@ slapos.rebootstrap = 4.5
slapos.recipe.build = 0.46 slapos.recipe.build = 0.46
slapos.recipe.cmmi = 0.17 slapos.recipe.cmmi = 0.17
slapos.recipe.template = 4.5 slapos.recipe.template = 4.5
slapos.toolbox = 0.117 slapos.toolbox = 0.118
stevedore = 1.21.0 stevedore = 1.21.0
subprocess32 = 3.5.3 subprocess32 = 3.5.4
unicodecsv = 0.14.1 unicodecsv = 0.14.1
wcwidth = 0.2.5 wcwidth = 0.2.5
wheel = 0.35.1 wheel = 0.35.1
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment