KVM: several fixes

add more tests for KVM and fix them

See merge request !752

software/kvm/buildout.hash.cfg

@@ -19,11 +19,11 @@ md5sum = 092405e2fba77c22d4dc8cefcab677d8
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = 285558df4686116a92b39250f9e00f07
+md5sum = 4e2aecca03c64d0bcff669652b581dba
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in
-md5sum = 6f4c60f4366728021a6e438ad3dc6956
+md5sum = 73b09e75d617888f6d84d363c0ada9c5
 
 [template-kvm-resilient]
 filename = instance-kvm-resilient.cfg.jinja2

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -40,7 +40,7 @@ config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
 # Request kvm instances
 {% for instance_name, kvm_parameter_dict in slapparameter_dict.get('kvm-partition-dict', {'kvm-default': {}}).items() -%}
 {% set section = 'request-' ~ instance_name -%}
-{% set use_nat = kvm_parameter_dict.get('use-nat', 'True') -%}
+{% set use_nat = kvm_parameter_dict.get('use-nat', True) -%}
 [{{ section }}]
 <= request-common
 software-type = kvm
@@ -97,7 +97,7 @@ config-auto-ballooning = {{ dumps(kvm_parameter_dict.get('auto-ballooning', True
 {% set nat_rules_list = kvm_parameter_dict.get('nat-rules', []) -%}
 {{ setconfig('nat-rules', nat_rules_list | join(' ')) }}
 config-publish-nat-url = True
-config-use-nat = {{ use_nat }}
+config-use-nat = {{ dumps(use_nat) }}
 config-use-tap = {{ dumps(kvm_parameter_dict.get('use-tap', True)) }}
 config-nat-restrict-mode = {{ dumps(kvm_parameter_dict.get('nat-restrict-mode', False)) }}
 config-enable-vhost = {{ dumps(kvm_parameter_dict.get('enable-vhost', False)) }}
@@ -142,11 +142,11 @@ sla-fw_restricted_access = {{ dumps(slapparameter_dict.get('fw-restricted-access
 
 return = 
   url
-{% if frontend_dict.get('frontend-additional-instance-guid', '') %}
+{% if frontend_dict.get('frontend-additional-instance-guid') %}
   url-additional
 {% endif %}
   backend-url
-{% if str(use_nat).lower() == 'true' -%}
+{% if use_nat -%}
 {%   for port in nat_rules_list -%}
 {%     if ':' in port -%}
 {%       set proto, port = port.split(':') -%}
@@ -170,7 +170,7 @@ return =
 {% do monitor_base_url_dict.__setitem__(instance_name, '${' ~ section ~ ':connection-monitor-base-url}') -%}
 {% do publish_dict.__setitem__(instance_name ~ '-backend-url', '${' ~ section ~ ':connection-backend-url}') -%}
 {% do publish_dict.__setitem__(instance_name ~ '-url', '${' ~ section ~ ':connection-url}') -%}
-{% if frontend_dict.get('frontend-additional-instance-guid', '') %}
+{% if frontend_dict.get('frontend-additional-instance-guid') %}
 {% do publish_dict.__setitem__(instance_name ~ '-url-additional', '${' ~ section ~ ':connection-url-additional}') -%}
 {% endif %}
 {% do kvm_instance_dict.__setitem__(instance_name, (use_nat, nat_rules_list)) -%}

software/kvm/instance-kvm.cfg.jinja2

-{% set additional_frontend = (slapparameter_dict.get('frontend-additional-instance-guid', '').strip() != '') %}
-{% set enable_http = slapparameter_dict.get('enable-http-server', 'False').lower() -%}
-{% set use_tap = slapparameter_dict.get('use-tap', 'True').lower() -%}
-{% set use_nat = slapparameter_dict.get('use-nat', 'True').lower() -%}
-{% set wipe_disk = slapparameter_dict.get('wipe-disk-ondestroy', 'False').lower() -%}
-{% set nat_restrict = slapparameter_dict.get('nat-restrict-mode', 'False').lower() -%}
+{% set additional_frontend = slapparameter_dict.get('frontend-additional-instance-guid') %}
+{% set enable_http = slapparameter_dict.get('enable-http-server', False) -%}
+{% set use_tap = slapparameter_dict.get('use-tap', True) -%}
+{% set use_nat = slapparameter_dict.get('use-nat', True) -%}
+{% set wipe_disk = slapparameter_dict.get('wipe-disk-ondestroy', False) -%}
+{% set nat_restrict = slapparameter_dict.get('nat-restrict-mode', False) -%}
 {% set name = slapparameter_dict.get('name', 'localhost') -%}
-{% set disable_ansible_promise = slapparameter_dict.get('disable-ansible-promise', 'True').lower() -%}
-{% set enable_device_hotplug = slapparameter_dict.get('enable-device-hotplug', 'false').lower() -%}
+{% set disable_ansible_promise = slapparameter_dict.get('disable-ansible-promise', True) -%}
 {% set instance_type = slapparameter_dict.get('type', 'standalone') -%}
 {% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
 {% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%}
@@ -116,7 +115,7 @@ tap-mac-address = ${create-tap-mac:mac-address}
 use-tap = ${slap-parameter:use-tap}
 use-nat = ${slap-parameter:use-nat}
 nat-rules = {{ nat_rule_list }}
-nat-restrict= {{ nat_restrict }}
+nat-restrict= {{ dumps(nat_restrict) }}
 enable-vhost = ${slap-parameter:enable-vhost}
 
 virtual-hard-drive-url = ${slap-parameter:virtual-hard-drive-url}
@@ -137,7 +136,7 @@ external-disk-number = ${slap-parameter:external-disk-number}
 external-disk-size = ${slap-parameter:external-disk-size}
 external-disk-format = ${slap-parameter:external-disk-format}
 
-{% if enable_http == 'true' -%}
+{% if enable_http -%}
 httpd-port = ${slap-parameter:httpd-port}
 {% else -%}
 httpd-port = 0
@@ -187,7 +186,7 @@ wrapper-path = ${directory:services}/6tunnel-${:ipv6-port}
 command-line = {{ sixtunnel_executable_location }} -6 -4 -d -l ${:ipv6} ${:ipv6-port} ${:ipv4} ${:ipv4-port}
 hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
-{% if use_nat == 'true' and nat_rule_list -%}
+{% if use_nat and nat_rule_list -%}
 {%   for port in nat_rule_list.split(' ') -%}
 {%     if ':' in port -%}
 {%       set proto, port = port.split(':') -%}
@@ -253,7 +252,7 @@ module = check_command_execute
 name = kvm-disk-image-corruption.py
 config-command = ${kvm-disk-image-corruption-bin:output}
 
-{% if wipe_disk == 'true' -%}
+{% if wipe_disk -%}
 {% do part_list.append('wipe-disk-wrapper') -%}
 {% set wipe_file_list = '${kvm-parameter-dict:disk-path}' -%}
 {% if storage_dict -%}
@@ -418,7 +417,7 @@ name = frontend_additional_promise.py
 config-url = ${publish-connection-information:url-additional}
 {% endif %}
 
-{% if enable_http == 'true' %}
+{% if enable_http %}
 [httpd]
 recipe = slapos.cookbook:simplehttpserver
 host = ${slap-network-information:local-ipv4}
@@ -470,7 +469,7 @@ url-additional = ${request-slave-frontend-additional:connection-secure_access}/v
 {% set disk_number = len(storage_dict) -%}
 maximum-extra-disk-amount = {{ disk_number }}
 {% set iface = 'ens3' -%}
-{% if use_nat == 'true' -%}
+{% if use_nat -%}
 {%   set iface = 'ens4' -%}
 {%   if nat_rule_list -%}
 # Publish NAT port mapping status
@@ -488,14 +487,14 @@ nat-rule-url-{{proto}}-{{port}} = [${slap-network-information:global-ipv6}]:${6t
 {%     endfor -%}
 {%   endif -%}
 {% endif -%}
-{% if use_tap == 'true' -%}
+{% if use_tap -%}
 tap-ipv4 = {{ slap_configuration.get('tap-ipv4-addr', '') }}
 tap-ipv6 = {{ slap_configuration.get('tap-ipv6-addr', '') }}
 {% endif -%}
 
 {% set kvm_http = 'http://${slap-network-information:local-ipv4}:' ~ slapparameter_dict.get('httpd-port', 8081) -%}
-{% if enable_http == 'true' %}
-{%   if use_nat == 'true' -%}
+{% if enable_http %}
+{%   if use_nat -%}
 {%     set kvm_http = 'http://10.0.2.100' -%}
 {%   endif %}
 {%   if slapparameter_dict.get('authorized-key', '') and slapparameter_dict.get('type', '') == 'cluster' -%}
@@ -503,7 +502,7 @@ key_info = Get the publick key file in your VM with the command: wget {{ kvm_htt
 {%   endif %}
 {% endif %}
 
-{% if use_tap == 'true' and slap_configuration.get('tap-ipv4-addr') -%}
+{% if use_tap and slap_configuration.get('tap-ipv4-addr') -%}
 ipv4-network-info =
   PERMANENT SOLUTION: in your VM, add the lines below in /etc/network/interfaces and then run: "ifup {{ iface }}"
   auto {{ iface }}
@@ -511,7 +510,7 @@ ipv4-network-info =
   	address {{ slap_configuration.get('tap-ipv4-addr') }}
   	netmask {{ slap_configuration.get('tap-ipv4-netmask') }}
   	gateway {{ slap_configuration.get('tap-ipv4-gateway') }}
-{% if enable_http == 'true' %}
+{% if enable_http %}
   ${helper:blank-line}
   TEMPORARY SOLUTION: run in your VM the command: "wget -O- {{ kvm_http }}/${network-config-ipv4:filename} | /bin/sh -"
   (the configuration will be gone after the next reboot)
@@ -519,14 +518,14 @@ ipv4-network-info =
 {% endif %}
 
 ipv6-network-info =
-{% if use_tap == 'true' and slap_configuration.get('tap-ipv6-addr') %}
+{% if use_tap and slap_configuration.get('tap-ipv6-addr') %}
   PERMANENT SOLUTION: in your VM, add the lines below in /etc/network/interfaces and then run: "ifup {{ iface }}"
   auto {{ iface }}
   iface {{ iface }} inet6 static
   	address {{ slap_configuration.get('tap-ipv6-gateway') }}
   	netmask {{ slap_configuration.get('tap-ipv6-network').split('/')[1] }}
   	gateway {{ slap_configuration.get('tap-ipv6-addr') }}
-{% if enable_http == 'true' %}
+{% if enable_http %}
   ${helper:blank-line}
   TEMPORARY SOLUTION: run in your VM the command: "wget -O- {{ kvm_http }}/${network-config-ipv6:filename} | /bin/sh -"
   (the configuration will be gone after the next reboot)
@@ -534,14 +533,14 @@ ipv6-network-info =
 {% endif %}
 
 
-{% if use_tap == 'true' and slap_configuration.get('tap-ipv4-addr') -%}
+{% if use_tap and slap_configuration.get('tap-ipv4-addr') -%}
 [network-config-ipv4]
 recipe = plone.recipe.command
 filename = netconfig.sh
 path = ${directory:public}/${:filename}
 ipv4-add-address = ip -4 address add {{ slap_configuration.get('tap-ipv4-addr') }}/{{ slap_configuration.get('tap-ipv4-netmask') }} dev \$IFACE noprefixroute
 ipv4-add-gateway-route = ip -4 address add {{ slap_configuration.get('tap-ipv4-gateway') }} dev \$IFACE
-{%   if nat_restrict == 'true' -%}
+{%   if nat_restrict -%}
 ipv4-add-default-route = ip route add default via {{ slap_configuration.get('tap-ipv4-gateway') }} dev \$IFACE
 {%   elif global_ipv4_prefix -%}
 ipv4-add-default-route = ip route add {{ global_ipv4_prefix }} via {{ slap_configuration.get('tap-ipv4-gateway') }} dev \$IFACE src {{ slap_configuration.get('tap-ipv4-addr') }}
@@ -564,7 +563,7 @@ command =
 update-command = ${:command}
 {% endif -%}
 
-{% if use_tap == 'true' and slap_configuration.get('tap-ipv6-addr') -%}
+{% if use_tap and slap_configuration.get('tap-ipv6-addr') -%}
 [network-config-ipv6]
 recipe = plone.recipe.command
 filename = ipv6_config.sh
@@ -602,7 +601,7 @@ mode = {{ mode }}
 {{   writefile('get-authorized-key', '${directory:public}/authorized_keys', slapparameter_dict.get('authorized-key', ''), '700') }}
 {% endif -%}
 
-{% if use_tap == 'true' and nat_restrict == 'true' -%}
+{% if use_tap and nat_restrict -%}
 # Ask to set default to tap interface in the vm
 {{ writefile('set-default-interface', '${directory:public}/delDefaultIface', iface, '600') }}
 {% do part_list.append('set-default-interface') -%}
@@ -611,7 +610,7 @@ mode = {{ mode }}
 [publish-host-config]
 recipe = plone.recipe.command
 name = {{ slapparameter_dict.get('name', 'localhost') }}
-{% if use_tap == 'true' and slap_configuration.get('tap-ipv4-addr') -%}
+{% if use_tap and slap_configuration.get('tap-ipv4-addr') -%}
 local-ipv4 = {{ slap_configuration['tap-ipv4-addr'] }}
 gateway = {{ slap_configuration.get('tap-ipv4-gateway') }}
 netmask = {{ slap_configuration.get('tap-ipv4-network') }}
@@ -679,7 +678,7 @@ context =
 
 [ansible-vm-promise]
 <= monitor-promise-base
-module = check_execute_comand
+module = check_command_execute
 name = ansible_{{ name }}.py
 config-command = ${ansible-vm-bin:rendered}
 
@@ -756,7 +755,13 @@ data-to-vm =
 keyboard-layout-language = fr
 
 {% for k, v in slapparameter_dict.items() -%}
+{% if k == 'authorized-key' and v -%}
+{% set key_list =  v.split('\n') -%}
+{{ k }} =
+  {{ key_list | join('\n  ') }}
+{% else -%}
 {{ k }} = {{ v }}
+{% endif -%}
 {% endfor -%}
 
 #############################
@@ -769,12 +774,12 @@ keyboard-layout-language = fr
 # Set Additionals parts
 {%   do part_list.append('cluster-url-path') -%}
 {% endif -%}
-{% if enable_http == 'true' %}
+{% if enable_http %}
 {%   do part_list.extend(['httpd', 'httpd-service', 'httpd-promise', 'publish-host-config']) -%}
 {% if slapparameter_dict.get('data-to-vm', '') %}
 {%   do part_list.append('vm-data-content') -%}
 {% endif -%}
-{% if disable_ansible_promise == 'false' %}
+{% if not disable_ansible_promise %}
 {%   do part_list.extend(['ansible-vm-promise', 'logrotate-vm-bootstrap']) -%}
 {% endif -%}
 {% if slapparameter_dict.get('authorized-key', '') and slapparameter_dict.get('type', '') == 'cluster' %}

software/kvm/test/test.py

@@ -54,7 +54,45 @@ else:
       self.fail('This environment is not usable for kvm testing,'
                 ' as it lacks kvm_intel kernel module')
 
+bootstrap_common_param_dict = {
+    # the bootstrap script is vm-bootstrap
+    "bootstrap-script-url": "http://shacache.org/shacache/05105cd25d1ad798b71fd46a206c9b73da2c285a078af33d0e739525a595886785725a68811578bc21f75d0a97700a66d5e75bce5b2721ca4556a0734cb13e65#c98825aa1b6c8087914d2bfcafec3058",
+    "slave-frontend": {
+        "slave-frontend-dict": {}
+        },
+    "authorized-keys": [
+        "ssh-rsa %s key_one" % ("A" * 372),
+        "ssh-rsa %s key_two" % ("B" * 372),
+        "ssh-rsa %s key_three" % ("C" * 372)
+        ],
+    "fw-restricted-access": "off",
+    "fw-authorized-sources": [],
+    "fw-reject-sources": ["10.32.0.0/13"]
+    }
 
+bootstrap_machine_param_dict = {
+    "computer-guid": "local",
+    "disable-ansible-promise": True,
+    "state": "started",
+    "auto-ballooning": True,
+    "ram-size": 4096,
+    "cpu-count": 2,
+    "disk-size": 50,
+    # Debian 10 image
+    "virtual-hard-drive-url": "http://shacache.org/shacache/9d3e6d017754fdd08e5ecf78093dec27fd792fb183df6146006adf003b6f4b98c0388d5a11566627101f7855d77f60e3dd4ba7ce66850f4a8f030573b904d5ab",
+    "virtual-hard-drive-md5sum": "b7928d7b0a2b5e2888f5ddf68f5fe422",
+    "virtual-hard-drive-gzipped": False,
+    "hard-drive-url-check-certificate": False,
+    "use-tap": True,
+    "use-nat": True,
+    "nat-restrict-mode":True,
+    "enable-vhost": True,
+    "external-disk-number": 1,
+    "external-disk-size": 100,
+    "external-disk-format": "qcow2",
+    "enable-monitor": True,
+    "keyboard-layout-language": "fr"
+    }
 @skipUnlessKvm
 class ServicesTestCase(InstanceTestCase):
   def test_hashes(self):
@@ -198,6 +236,25 @@ class TestAccessDefaultAdditional(MonitorAccessMixin, InstanceTestCase):
     )
     self.assertIn('<title>noVNC</title>', result.text)
 
+@skipUnlessKvm
+class TestAccessDefaultBootstrap(MonitorAccessMixin, InstanceTestCase):
+  __partition_reference__ = 'adb'
+  expected_partition_with_monitor_base_url_count = 1
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'_': json.dumps(dict(bootstrap_common_param_dict, **bootstrap_machine_param_dict))}
+
+  def test(self):
+    connection_parameter_dict = self.computer_partition\
+      .getConnectionParameterDict()
+
+    result = requests.get(connection_parameter_dict['url'], verify=False)
+    self.assertEqual(
+      httplib.OK,
+      result.status_code
+    )
+    self.assertIn('<title>noVNC</title>', result.text)
 
 @skipUnlessKvm
 class TestAccessKvmCluster(MonitorAccessMixin, InstanceTestCase):
@@ -270,6 +327,45 @@ class TestAccessKvmClusterAdditional(MonitorAccessMixin, InstanceTestCase):
     )
     self.assertIn('<title>noVNC</title>', result.text)
 
+@skipUnlessKvm
+class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase):
+  __partition_reference__ = 'akcb'
+  expected_partition_with_monitor_base_url_count = 3
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'kvm-cluster'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'_': json.dumps(dict(bootstrap_common_param_dict, **{
+      "kvm-partition-dict": {
+          "test-machine1": bootstrap_machine_param_dict,
+          "test-machine2": dict(bootstrap_machine_param_dict, **{
+              # Debian 9 image
+              "virtual-hard-drive-url": "http://shacache.org/shacache/ce07873dbab7fa8501d1bf5565c2737b2eed6c8b9361b4997b21daf5f5d1590972db9ac00131cc5b27d9aa353f2f94071e073f9980cc61badd6d2427f592e6e8",
+              "virtual-hard-drive-md5sum": "2b113e3cd8276b9740189622603d6f99"
+              })
+          }
+      }))}
+
+  def test(self):
+    connection_parameter_dict = self.computer_partition\
+      .getConnectionParameterDict()
+    result = requests.get(connection_parameter_dict['KVM0-url'], verify=False)
+    self.assertEqual(
+      httplib.OK,
+      result.status_code
+    )
+    self.assertIn('<title>noVNC</title>', result.text)
+
+    result = requests.get(
+      connection_parameter_dict['KVM1-url'], verify=False)
+    self.assertEqual(
+      httplib.OK,
+      result.status_code
+    )
+    self.assertIn('<title>noVNC</title>', result.text)
 
 @skipUnlessKvm
 class TestInstanceResilient(InstanceTestCase):