Commit 78dd88df authored by Jérome Perrin's avatar Jérome Perrin

Merge remote-tracking branch 'upstream/master' into zope4py2

parents 49af104e 028fae15
...@@ -18,6 +18,7 @@ extends = ...@@ -18,6 +18,7 @@ extends =
../xorg/buildout.cfg ../xorg/buildout.cfg
../zlib/buildout.cfg ../zlib/buildout.cfg
../glib/buildout.cfg ../glib/buildout.cfg
../ceph/buildout.cfg
[gcc] [gcc]
min_version = 7.4 min_version = 7.4
...@@ -35,6 +36,7 @@ patch-binary = ${patch:location}/bin/patch ...@@ -35,6 +36,7 @@ patch-binary = ${patch:location}/bin/patch
pre-configure = pre-configure =
sed -i '/^libmigration\b/s/$/ dependencies: [zlib],/' meson.build sed -i '/^libmigration\b/s/$/ dependencies: [zlib],/' meson.build
sed -i 's/\bsnappy,/zlib, \0/' dump/meson.build sed -i 's/\bsnappy,/zlib, \0/' dump/meson.build
configure-rbd = --enable-rbd
configure-options = configure-options =
--meson=${meson:location}/bin/meson --meson=${meson:location}/bin/meson
--ninja=${ninja:location}/bin/ninja --ninja=${ninja:location}/bin/ninja
...@@ -55,11 +57,21 @@ configure-options = ...@@ -55,11 +57,21 @@ configure-options =
--enable-vnc-png --enable-vnc-png
--disable-vnc-jpeg --disable-vnc-jpeg
--disable-werror --disable-werror
${:configure-rbd}
CFLAGS-rbd = -I${librbd:location}/include/
LDFLAGS-rbd = -L${librbd:location}/lib -Wl,-rpath=${librbd:location}/lib
PKG_CONFIG_PATH-rbd = :${librbd:location}/lib/pkgconfig
environment = environment =
CFLAGS=-I${gettext:location}/include -I${libaio:location}/include -I${liburing:location}/include -I${libcap-ng:location}/include CFLAGS=${:CFLAGS-rbd} -I${gettext:location}/include -I${libaio:location}/include -I${liburing:location}/include -I${libcap-ng:location}/include
LDFLAGS=-L${gettext:location}/lib -L${libaio:location}/lib -L${libcap-ng:location}/lib -Wl,-rpath=${libcap-ng:location}/lib -Wl,-rpath=${glib:location}/lib -Wl,-rpath=${gnutls:location}/lib -Wl,-rpath=${nettle:location}/lib -Wl,-rpath=${pixman:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${libpng:location}/lib -Wl,-rpath=${libaio:location}/lib -Wl,-rpath=${liburing:location}/lib -Wl,-rpath=${libcap-ng:location}/lib LDFLAGS=${:LDFLAGS-rbd} -L${gettext:location}/lib -L${libaio:location}/lib -L${libcap-ng:location}/lib -Wl,-rpath=${libcap-ng:location}/lib -Wl,-rpath=${glib:location}/lib -Wl,-rpath=${gnutls:location}/lib -Wl,-rpath=${nettle:location}/lib -Wl,-rpath=${pixman:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${libpng:location}/lib -Wl,-rpath=${libaio:location}/lib -Wl,-rpath=${liburing:location}/lib -Wl,-rpath=${libcap-ng:location}/lib
PATH=${pkgconfig:location}/bin:${bzip2:location}/bin:%(PATH)s PATH=${pkgconfig:location}/bin:${bzip2:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${gnutls:location}/lib/pkgconfig:${gnutls:pkg-config-path}:${libpng:location}/lib/pkgconfig:${liburing:location}/lib/pkgconfig:${ncurses:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${gnutls:location}/lib/pkgconfig:${gnutls:pkg-config-path}:${libpng:location}/lib/pkgconfig:${liburing:location}/lib/pkgconfig:${ncurses:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig${:PKG_CONFIG_PATH-rbd}
[qemu:[int(q) for q in platform.libc_ver()[1].split(".")] < [2,25]]
configure-rbd =
CFLAGS-rbd =
LDFLAGS-rbd =
PKG_CONFIG_PATH-rbd =
[qemu:sys.version_info < (3,6)] [qemu:sys.version_info < (3,6)]
environment += environment +=
......
[instance-profile] [instance-profile]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = f0714597e23ebcb0f2a1368cd065a84e md5sum = 17004b2adb98b545b16c6be60e8165e8
...@@ -7,7 +7,6 @@ ...@@ -7,7 +7,6 @@
parts = parts =
publish-connection-parameter publish-connection-parameter
download-plc download-plc
compile-plc
beremiz-runtime beremiz-runtime
#beremiz-runtime-promise #beremiz-runtime-promise
...@@ -21,13 +20,6 @@ recipe = slapos.recipe.build:download-unpacked ...@@ -21,13 +20,6 @@ recipe = slapos.recipe.build:download-unpacked
offline = false offline = false
url = ${instance-parameter:configuration.runtime_plc_url} url = ${instance-parameter:configuration.runtime_plc_url}
[compile-plc]
recipe = plone.recipe.command
stop-on-error = true
command =
{{ buildout['bin-directory'] }}/pythonwitheggs {{ buildout['directory'] }}/parts/beremiz-source/Beremiz_cli.py --project-home ${directory:home}/parts/download-plc/ build
update-command = ${:command}
[instance-parameter] [instance-parameter]
recipe = slapos.cookbook:slapconfiguration recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id} computer = ${slap-connection:computer-id}
...@@ -54,8 +46,19 @@ log = ${:var}/log ...@@ -54,8 +46,19 @@ log = ${:var}/log
[beremiz-runtime] [beremiz-runtime]
logfile = ${directory:log}/beremiz-runtime.log logfile = ${directory:log}/beremiz-runtime.log
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
# adding BEREMIZPYTHONPATH is needed so we can override the Beremiz'
# internal code which tries to use sys.executable to spawn processes
# and in the context of SlapOS it's a plain Python without needed modules
# adding LIBRARY_PATH / COMPILER_PATH is needed so that gcc can find
# needed libraries and tools inside SlapOS context
environment =
BEREMIZPYTHONPATH = {{ buildout['bin-directory'] }}/pythonwitheggs
PATH=$PATH:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin
LIBRARY_PATH=/usr/lib/gcc/x86_64-linux-gnu/9/:/usr/lib/x86_64-linux-gnu:/usr/lib:/lib/x86_64-linux-gnu/:/lib:/usr/lib/x86_64-linux-gnu/
COMPILER_PATH=/usr/lib/gcc/x86_64-linux-gnu/9/:/usr/lib/gcc/x86_64-linux-gnu/
command-line = command-line =
{{ buildout['bin-directory'] }}/pythonwitheggs {{ buildout['directory'] }}/parts/beremiz-source/Beremiz_service.py -a ${instance-parameter:configuration.autostart} -p ${instance-parameter:configuration.port} -i ${instance-parameter:configuration.interface} -x 1 ${directory:home}/parts/download-plc/build {{ buildout['bin-directory'] }}/pythonwitheggs {{ buildout['directory'] }}/parts/beremiz-source/Beremiz_cli.py -k --project-home ${directory:home}/parts/download-plc/ build transfer run
wrapper-path = ${directory:service}/beremiz-runtime wrapper-path = ${directory:service}/beremiz-runtime
[beremiz-runtime-promise] [beremiz-runtime-promise]
......
...@@ -72,12 +72,6 @@ KVM instance parameters: ...@@ -72,12 +72,6 @@ KVM instance parameters:
- hard-drive-url-check-certificate (default: True) - hard-drive-url-check-certificate (default: True)
if virtual-hard-drive-url use self-signed https, then specify if https certificate should be verified or not if virtual-hard-drive-url use self-signed https, then specify if https certificate should be verified or not
- external-disk-number (default: 0)
Number of additional disk to attach to this VM. Need slapformat to be configured for this feature.
- external-disk-size (default: 20)
- external-disk-format (default: qcow2)
additional disk format. should be in this list: ['qcow2', 'raw', 'vdi', 'vmdk', 'cloop', 'qed']
- enable-http-server (default: False) - enable-http-server (default: False)
Configure server that will help to get some files into the vm from http Configure server that will help to get some files into the vm from http
require use-nat = True require use-nat = True
...@@ -126,3 +120,83 @@ Updating boot-image-url-select ...@@ -126,3 +120,83 @@ Updating boot-image-url-select
* update the ``boot-image-url-select`` in: * update the ``boot-image-url-select`` in:
* ``instance-kvm-input-schema.json`` * ``instance-kvm-input-schema.json``
* ``instance-kvm-cluster-input-schema.json`` * ``instance-kvm-cluster-input-schema.json``
Migration to modern external-disk parameter
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
**Note**: ``external-disk`` and old way are mutually exclusive, thus it will
result with not starting kvm and failing partition for sake of data
consistency.
Despite ``external-disk-number``, ``external-disk-size`` and
``external-disk-format`` are supported fully until unknown moment in the
future, it's advised to migrate to external-disk parameter as soon as possible,
as slapos.core ``slapos.cfg`` ``instance_storage_home`` can become obsoleted
and removed in future versions.
**Note**: Due to how technically ``instance_storage_home`` is implemented, such
migration requires full access to the Compute Node hosting given KVM instance.
Let's imagine that there is a kvm instance which was requested with parameters::
{
"external-disk-number": 2,
"external-disk-size": 10
}
After locating the partition on proper Compute Node, the node administrator
has to find the kvm processing running there with::
slapos node status slappartNN: | grep kvm-
The interesting part is the ``PID``, which can be used to find which disk paths
are configured for the running KVM process with::
ps axu | grep PID | grep --color DATA
It will be possible to find two entries pointing to ``DATA`` directory in the
partition::
-drive file=/srv/slapgrid/slappartNN/DATA/dataX/kvm_virtual_disk.qcow2,if=virtio
-drive file=/srv/slapgrid/slappartNN/DATA/dataY/kvm_virtual_disk.qcow2,if=virtio
**Attention**: Order of the disks is important.
The administrator shall provide absolute path to both for both disks::
readlink -f /srv/slapgrid/slappartNN/DATA/dataX/kvm_virtual_disk.qcow2 --> /<instance_storage_home>/dataX/slappartNN/kvm_virtual_disk.qcow2
readlink -f /srv/slapgrid/slappartNN/DATA/dataY/kvm_virtual_disk.qcow2 --> /<instance_storage_home>/dataY/slappartNN/kvm_virtual_disk.qcow2
And now it will be safe to use the paths in ``external-disk`` parameter::
{
"external-disk": {
"first": {
"path": "/<instance_storage_home>/dataX/slappartNN/kvm_virtual_disk.qcow2",
"index": 1
},
"second": {
"path": "/<instance_storage_home>/dataY/slappartNN/kvm_virtual_disk.qcow2",
"index": 2
}
}
}
Of course ``external-disk-number`` and ``external-disk-size`` HAVE TO be removed
from instance parameters before continuing.
For now such configuration will lead to no starting kvm process, so after
parameters are updated in SlapOS Master **and** are processed on the Compute
Node The administrator shall release the images from automatic detection by
removing files:
* ``etc/.data-disk-amount``
* ``etc/.data-disk-ids``
from the partition (typically ``/srv/slapgrid/slappartNN/`` directory).
The failure observed to confirm the situation can be found in
``.slappartNN_kvm-HASH.log`` with presence of message like::
qemu-system-x86_64: -drive file=/<instance_storage_home>/dataX/slappartNN/kvm_virtual_disk.qcow2,if=virtio,cache=writeback: Failed to get "write" lock
Is another process using the image [/<instance_storage_home>/dataX/slappartNN/kvm_virtual_disk.qcow2]?
...@@ -19,11 +19,11 @@ md5sum = 7e90da1f6dac4233e1aa3248f48e357c ...@@ -19,11 +19,11 @@ md5sum = 7e90da1f6dac4233e1aa3248f48e357c
[template-kvm] [template-kvm]
filename = instance-kvm.cfg.jinja2 filename = instance-kvm.cfg.jinja2
md5sum = 4a3f999a8d1705c0cabf58d563953d4d md5sum = 44dc93281f2fffe64d014754fae1b38a
[template-kvm-cluster] [template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in filename = instance-kvm-cluster.cfg.jinja2.in
md5sum = 406bf455afe1c71cb0e8cc04dd877f1d md5sum = c00608f94ff6cfd8d37a280267c7227f
[template-kvm-resilient] [template-kvm-resilient]
filename = instance-kvm-resilient.cfg.jinja2 filename = instance-kvm-resilient.cfg.jinja2
...@@ -55,7 +55,7 @@ md5sum = a8cf453d20f01c707f02c4b4014580d8 ...@@ -55,7 +55,7 @@ md5sum = a8cf453d20f01c707f02c4b4014580d8
[template-kvm-run] [template-kvm-run]
filename = template/template-kvm-run.in filename = template/template-kvm-run.in
md5sum = f3531d250327135603053bc43105aab5 md5sum = 7da4fcd815cb7abe318d8ac4b98c8cf3
[template-kvm-controller] [template-kvm-controller]
filename = template/kvm-controller-run.in filename = template/kvm-controller-run.in
......
...@@ -392,34 +392,6 @@ ...@@ -392,34 +392,6 @@
"type": "boolean", "type": "boolean",
"default": true "default": true
}, },
"external-disk-number": {
"title": "Number of additional disk to create for virtual machine",
"description": "Specify the number of additional disk to create for virtual machine in data folder of SlapOS Node. Requires instance_storage_home to be configured on SlapOS Node.",
"type": "integer",
"minimum": 0,
"default": 0
},
"external-disk-size": {
"title": "Size of additional disk to create for virtual machine, in Gigabytes",
"description": "Specify the size of additional disk to create for virtual machine in data folder of SlapOS Node. Requires instance_storage_home to be configured on SlapOS Node.",
"type": "integer",
"minimum": 5,
"default": 20
},
"external-disk-format": {
"title": "Type of external disk drive to create by QEMU.",
"description": "Type of QEMU disk drive, to create.",
"type": "string",
"default": "qcow2",
"enum": [
"qcow2",
"raw",
"vdi",
"vmdk",
"cloop",
"qed"
]
},
"wipe-disk-ondestroy": { "wipe-disk-ondestroy": {
"title": "Wipe disks when destroy the VM", "title": "Wipe disks when destroy the VM",
"description": "Say if disks should be wiped by writing new data over every single bit before delete them. This option is used to securely delete VM disks", "description": "Say if disks should be wiped by writing new data over every single bit before delete them. This option is used to securely delete VM disks",
...@@ -546,6 +518,58 @@ ...@@ -546,6 +518,58 @@
"description": "List of whitelisted domain names to be accessed from the VM. They will be resolved to IPs depending on where the VM end up. IPs can be used too.", "description": "List of whitelisted domain names to be accessed from the VM. They will be resolved to IPs depending on where the VM end up. IPs can be used too.",
"type": "string", "type": "string",
"textarea": true "textarea": true
},
"external-disk": {
"title": "External disks",
"description": "Allows to set paths and types of manually provided external disks. Use index to order disks. Changing this parameter leads to kvm process restart.",
"type": "object",
"default": {},
"patternProperties": {
".*": {
"properties": {
"path": {
"title": "Path of the provided image",
"description": "Absolute or relative path (to the partition root path) of the provided image. Image has to be provided, with read and write permissions of the partition user. Please note, that if the name starts with 'rbd:' it's considered special and won't be used as relative path.",
"type": "string"
},
"format": {
"title": "Format of a disk image",
"description": "Format of a disk image. Note: rbd type is experimental.",
"type": "string",
"default": "autodetect",
"enum": [
"autodetect",
"qcow2",
"rbd",
"raw",
"vdi",
"vmdk",
"cloop",
"qed"
]
},
"cache": {
"title": "Cache value for the disk",
"description": "Cache value directly passed to the kvm -drive invocation as cache=value.",
"type": "string",
"default": "writeback",
"enum": [
"writeback",
"none",
"unsafe",
"directsync",
"writethrough"
]
},
"index": {
"title": "Index of a disk",
"description": "An index value used to order disks for the VM, required if more than one disk is used to have stable ordering.",
"type": "integer",
"default": 0
}
}
}
}
} }
}, },
"type": "object" "type": "object"
......
...@@ -139,7 +139,7 @@ config-wipe-disk-iterations = {{ dumps(kvm_parameter_dict.get('wipe-disk-iterati ...@@ -139,7 +139,7 @@ config-wipe-disk-iterations = {{ dumps(kvm_parameter_dict.get('wipe-disk-iterati
config-document-host = ${apache-conf:ip} config-document-host = ${apache-conf:ip}
config-document-port = ${apache-conf:port} config-document-port = ${apache-conf:port}
config-document-path = ${hash-code:passwd} config-document-path = ${hash-code:passwd}
{%- for k in ['boot-image-url-list', 'boot-image-url-select', 'whitelist-domains'] %} {%- for k in ['boot-image-url-list', 'boot-image-url-select', 'whitelist-domains', 'external-disk'] %}
{#- play nice - use parameter only if present #} {#- play nice - use parameter only if present #}
{%- if k in kvm_parameter_dict %} {%- if k in kvm_parameter_dict %}
{#- play safe - dumps value #} {#- play safe - dumps value #}
......
...@@ -193,34 +193,6 @@ ...@@ -193,34 +193,6 @@
"type": "boolean", "type": "boolean",
"default": true "default": true
}, },
"external-disk-number": {
"title": "Number of additional disk to create for virtual machine",
"description": "Specify the number of additional disk to create for virtual machine in data folder of SlapOS Node. Requires instance_storage_home to be configured on SlapOS Node.",
"type": "integer",
"minimum": 0,
"default": 0
},
"external-disk-size": {
"title": "Size of additional disk to create for virtual machine, in Gigabytes",
"description": "Specify the size of additional disk to create for virtual machine in data folder of SlapOS Node. Requires instance_storage_home to be configured on SlapOS Node.",
"type": "integer",
"minimum": 1,
"default": 20
},
"external-disk-format": {
"title": "Type of external disk drive to create by QEMU.",
"description": "Type of QEMU disk drive, to create.",
"type": "string",
"default": "qcow2",
"enum": [
"qcow2",
"raw",
"vdi",
"vmdk",
"cloop",
"qed"
]
},
"wipe-disk-ondestroy": { "wipe-disk-ondestroy": {
"title": "Wipe disks when destroy the VM", "title": "Wipe disks when destroy the VM",
"description": "Say if disks should be wiped by writing new data over every single bit before delete them. This option is used to securely delete VM disks", "description": "Say if disks should be wiped by writing new data over every single bit before delete them. This option is used to securely delete VM disks",
...@@ -403,6 +375,58 @@ ...@@ -403,6 +375,58 @@
"description": "List of whitelisted domain names to be accessed from the VM. They will be resolved to IPs depending on where the VM end up. IPs can be used too.", "description": "List of whitelisted domain names to be accessed from the VM. They will be resolved to IPs depending on where the VM end up. IPs can be used too.",
"type": "string", "type": "string",
"textarea": true "textarea": true
},
"external-disk": {
"title": "External disks",
"description": "Allows to set paths and types of manually provided external disks. Use index to order disks. Changing this parameter leads to kvm process restart.",
"type": "object",
"default": {},
"patternProperties": {
".*": {
"properties": {
"path": {
"title": "Path of the provided image",
"description": "Absolute or relative path (to the partition root path) of the provided image. Image has to be provided, with read and write permissions of the partition user. Please note, that if the name starts with 'rbd:' it's considered special and won't be used as relative path.",
"type": "string"
},
"format": {
"title": "Format of a disk image",
"description": "Format of a disk image. Note: rbd type is experimental.",
"type": "string",
"default": "autodetect",
"enum": [
"autodetect",
"qcow2",
"rbd",
"raw",
"vdi",
"vmdk",
"cloop",
"qed"
]
},
"cache": {
"title": "Cache value for the disk",
"description": "Cache value directly passed to the kvm -drive invocation as cache=value.",
"type": "string",
"default": "writeback",
"enum": [
"writeback",
"none",
"unsafe",
"directsync",
"writethrough"
]
},
"index": {
"title": "Index of a disk",
"description": "An index value used to order disks for the VM, required if more than one disk is used to have stable ordering.",
"type": "integer",
"default": 0
}
}
}
}
} }
} }
} }
...@@ -471,6 +471,9 @@ disk-storage-list = ...@@ -471,6 +471,9 @@ disk-storage-list =
external-disk-number = ${slap-parameter:external-disk-number} external-disk-number = ${slap-parameter:external-disk-number}
external-disk-size = ${slap-parameter:external-disk-size} external-disk-size = ${slap-parameter:external-disk-size}
external-disk-format = ${slap-parameter:external-disk-format} external-disk-format = ${slap-parameter:external-disk-format}
# new external-disk parameter
external-disk = ${slap-parameter:external-disk}
instance-root = ${buildout:directory}
{% if enable_http -%} {% if enable_http -%}
httpd-port = ${slap-parameter:httpd-port} httpd-port = ${slap-parameter:httpd-port}
...@@ -1091,6 +1094,7 @@ hard-drive-url-check-certificate = True ...@@ -1091,6 +1094,7 @@ hard-drive-url-check-certificate = True
external-disk-number = 0 external-disk-number = 0
external-disk-size = 20 external-disk-size = 20
external-disk-format = qcow2 external-disk-format = qcow2
external-disk = {}
# Help to get some configuration files into the vm from http # Help to get some configuration files into the vm from http
enable-http-server = False enable-http-server = False
......
...@@ -12,6 +12,7 @@ from random import shuffle ...@@ -12,6 +12,7 @@ from random import shuffle
import glob import glob
import re import re
import json import json
import operator
# XXX: give all of this through parameter, don't use this as template, but as module # XXX: give all of this through parameter, don't use this as template, but as module
qemu_img_path = {{ repr(parameter_dict["qemu-img-path"]) }} qemu_img_path = {{ repr(parameter_dict["qemu-img-path"]) }}
...@@ -46,6 +47,10 @@ pid_file_path = '{{ parameter_dict.get("pid-file-path") }}' ...@@ -46,6 +47,10 @@ pid_file_path = '{{ parameter_dict.get("pid-file-path") }}'
external_disk_number = {{ parameter_dict.get("external-disk-number") }} external_disk_number = {{ parameter_dict.get("external-disk-number") }}
external_disk_size = {{ parameter_dict.get("external-disk-size") }} external_disk_size = {{ parameter_dict.get("external-disk-size") }}
external_disk_format = {{ repr(parameter_dict["external-disk-format"]) }} external_disk_format = {{ repr(parameter_dict["external-disk-format"]) }}
external_disk = {{ parameter_dict['external-disk'] }}
if int(external_disk_number) > 0 and len(external_disk) > 0:
raise ValueError("external-disk-number and external-disk are mutually exclusive.")
instance_root = '{{ parameter_dict['instance-root'] }}'
disk_storage_dict = {} disk_storage_dict = {}
disk_storage_list = """{{ parameter_dict.get("disk-storage-list") }}""".split('\n') disk_storage_list = """{{ parameter_dict.get("disk-storage-list") }}""".split('\n')
map_storage_list = [] map_storage_list = []
...@@ -305,6 +310,32 @@ for disk in additional_disk_list: ...@@ -305,6 +310,32 @@ for disk in additional_disk_list:
kvm_argument_list.extend([ kvm_argument_list.extend([
'-drive', 'file=%s,if=%s' % (disk, disk_type)]) '-drive', 'file=%s,if=%s' % (disk, disk_type)])
# support external-disk parameter
# allow empty index if only one disk is provided
if len(external_disk) > 1:
for key, value in external_disk.items():
if 'index' not in value:
raise ValueError('index is missing and more than one disk is present in external-disk configuration')
for disk_info in sorted(external_disk.values(), key=operator.itemgetter('index')):
if disk_info['path'].startswith('rbd:') or disk_info['path'].startswith('/'):
path = disk_info['path']
else:
path = os.path.join(instance_root, disk_info['path'])
drive_argument_list = [
"file=%s" %( path,),
"if=%s" % (disk_type,),
"cache=%s" % (disk_info.get('cache', 'writeback'),)
]
if disk_info.get('format', 'autodetect') != 'autodetect':
drive_argument_list.append(
"format=%s" % (disk_info['format'],)
)
kvm_argument_list += (
'-drive',
",".join(drive_argument_list)
)
if auto_ballooning: if auto_ballooning:
kvm_argument_list.extend(['-device', 'virtio-balloon-pci,id=balloon0']) kvm_argument_list.extend(['-device', 'virtio-balloon-pci,id=balloon0'])
......
...@@ -1868,8 +1868,25 @@ class TestParameterCluster(TestParameterDefault): ...@@ -1868,8 +1868,25 @@ class TestParameterCluster(TestParameterDefault):
return 'kvm-cluster' return 'kvm-cluster'
class ExternalDiskMixin(KvmMixin):
def getRunningDriveList(self, kvm_instance_partition):
_match_drive = re.compile('file.*if=virtio.*').match
with self.slap.instance_supervisor_rpc as instance_supervisor:
kvm_pid = next(q for q in instance_supervisor.getAllProcessInfo()
if 'kvm-' in q['name'])['pid']
drive_list = []
for entry in psutil.Process(kvm_pid).cmdline():
m = _match_drive(entry)
if m:
path = m.group(0)
drive_list.append(
path.replace(kvm_instance_partition, '${partition}')
)
return drive_list
@skipUnlessKvm @skipUnlessKvm
class TestExternalDisk(InstanceTestCase, KvmMixin): class TestExternalDisk(InstanceTestCase, ExternalDiskMixin):
__partition_reference__ = 'ed' __partition_reference__ = 'ed'
kvm_instance_partition_reference = 'ed0' kvm_instance_partition_reference = 'ed0'
...@@ -1958,21 +1975,6 @@ class TestExternalDisk(InstanceTestCase, KvmMixin): ...@@ -1958,21 +1975,6 @@ class TestExternalDisk(InstanceTestCase, KvmMixin):
super(TestExternalDisk, cls).tearDownClass() super(TestExternalDisk, cls).tearDownClass()
shutil.rmtree(cls.working_directory) shutil.rmtree(cls.working_directory)
def getRunningDriveList(self, kvm_instance_partition):
_match_drive = re.compile('file=(.+),if=virtio').match
with self.slap.instance_supervisor_rpc as instance_supervisor:
kvm_pid = next(q for q in instance_supervisor.getAllProcessInfo()
if 'kvm-' in q['name'])['pid']
dirve_list = []
for entry in psutil.Process(kvm_pid).cmdline():
m = _match_drive(entry)
if m:
path = m.group(1)
dirve_list.append(
path.replace(kvm_instance_partition, '${partition}')
)
return dirve_list
def test(self): def test(self):
kvm_instance_partition = os.path.join( kvm_instance_partition = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference) self.slap.instance_directory, self.kvm_instance_partition_reference)
...@@ -2005,3 +2007,191 @@ class TestExternalDisk(InstanceTestCase, KvmMixin): ...@@ -2005,3 +2007,191 @@ class TestExternalDisk(InstanceTestCase, KvmMixin):
class TestExternalDiskJson( class TestExternalDiskJson(
KvmMixinJson, TestExternalDisk): KvmMixinJson, TestExternalDisk):
pass pass
@skipUnlessKvm
class TestExternalDiskModern(InstanceTestCase, ExternalDiskMixin):
__partition_reference__ = 'edm'
kvm_instance_partition_reference = 'edm0'
@classmethod
def getInstanceSoftwareType(cls):
return 'default'
@classmethod
def setUpClass(cls):
super(TestExternalDiskModern, cls).setUpClass()
def getExternalDiskInstanceParameterDict(
self, first, second, third, update_dict=None):
parameter_dict = {
"external-disk": {
"second disk": {
"path": second,
"index": 2,
},
"third disk": {
"path": third,
"index": 3,
"cache": "none"
},
"first disk": {
"path": first,
"index": 1,
"format": "qcow"
},
}
}
if update_dict is not None:
parameter_dict.update(update_dict)
return parameter_dict
def test(self):
# Disks can't be created in /tmp, as it's specially mounted on testnodes
# and then KVM can't use them:
# -drive file=/tmp/tmpX/third_disk,if=virtio,cache=none: Could not open
# '/tmp/tmpX/third_disk': filesystem does not support O_DIRECT
self.working_directory = tempfile.mkdtemp(dir=self.slap.instance_directory)
self.addCleanup(shutil.rmtree, self.working_directory)
kvm_instance_partition = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference)
# find qemu_img from the tested SR via it's partition parameter, as
# otherwise qemu-kvm would be dependency of test suite
with open(
os.path.join(self.computer_partition_root_path, 'buildout.cfg')) as fh:
qemu_img = [
q for q in fh.readlines()
if 'raw qemu_img_executable_location' in q][0].split()[-1]
self.first_disk = os.path.join(self.working_directory, 'first_disk')
subprocess.check_call([
qemu_img, "create", "-f", "qcow", self.first_disk, "1M"])
second_disk = 'second_disk'
self.second_disk = os.path.join(kvm_instance_partition, second_disk)
subprocess.check_call([
qemu_img, "create", "-f", "qcow2", os.path.join(
kvm_instance_partition, self.second_disk), "1M"])
self.third_disk = os.path.join(self.working_directory, 'third_disk')
subprocess.check_call([
qemu_img, "create", "-f", "qcow2", self.third_disk, "1M"])
self.rerequestInstance({'_': json.dumps(
self.getExternalDiskInstanceParameterDict(
self.first_disk, second_disk, self.third_disk))})
self.waitForInstance()
drive_list = self.getRunningDriveList(kvm_instance_partition)
self.assertEqual(
drive_list,
[
'file=${partition}/srv/virtual.qcow2,if=virtio,discard=on,'
'format=qcow2',
'file=%s/first_disk,if=virtio,cache=writeback,format=qcow' % (
self.working_directory,),
'file=${partition}/second_disk,if=virtio,cache=writeback',
'file=%s/third_disk,if=virtio,cache=none' % (
self.working_directory,)
]
)
update_dict = {
"external-disk-number": 1,
"external-disk-size": 100,
"external-disk-format": "qcow2",
}
parameter_dict = self.getExternalDiskInstanceParameterDict(
self.first_disk, second_disk, self.third_disk, update_dict)
# assert mutual exclusivity
self.rerequestInstance({'_': json.dumps(parameter_dict)})
self.raising_waitForInstance(3)
@skipUnlessKvm
class TestExternalDiskModernCluster(TestExternalDiskModern):
kvm_instance_partition_reference = 'edm1'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({
"kvm-partition-dict": {
"kvm-default": {
"disable-ansible-promise": True,
}
}
})}
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-cluster'
def getExternalDiskInstanceParameterDict(self, *args, **kwargs):
partition_dict = super(
TestExternalDiskModernCluster, self
).getExternalDiskInstanceParameterDict(*args, **kwargs)
partition_dict.update({"disable-ansible-promise": True})
return {
"kvm-partition-dict": {
"kvm-default": partition_dict
}
}
@skipUnlessKvm
class TestExternalDiskModernIndexRequired(InstanceTestCase, ExternalDiskMixin):
__partition_reference__ = 'edm'
kvm_instance_partition_reference = 'edm0'
@classmethod
def getInstanceSoftwareType(cls):
return 'default'
@classmethod
def setUpClass(cls):
super(TestExternalDiskModernIndexRequired, cls).setUpClass()
def getExternalDiskInstanceParameterDict(self, first, second, third):
return {
"external-disk": {
"second disk": {
"path": second,
},
"third disk": {
"path": third,
"index": 3,
},
"first disk": {
"path": first,
"index": 1,
},
}
}
def test(self):
# Disks can't be created in /tmp, as it's specially mounted on testnodes
# and then KVM can't use them:
# -drive file=/tmp/tmpX/third_disk,if=virtio,cache=none: Could not open
# '/tmp/tmpX/third_disk': filesystem does not support O_DIRECT
self.working_directory = tempfile.mkdtemp(dir=self.slap.instance_directory)
self.addCleanup(shutil.rmtree, self.working_directory)
kvm_instance_partition = os.path.join(
self.slap.instance_directory, self.kvm_instance_partition_reference)
# find qemu_img from the tested SR via it's partition parameter, as
# otherwise qemu-kvm would be dependency of test suite
with open(
os.path.join(self.computer_partition_root_path, 'buildout.cfg')) as fh:
qemu_img = [
q for q in fh.readlines()
if 'raw qemu_img_executable_location' in q][0].split()[-1]
self.first_disk = os.path.join(self.working_directory, 'first_disk')
subprocess.check_call([
qemu_img, "create", "-f", "qcow", self.first_disk, "1M"])
second_disk = 'second_disk'
self.second_disk = os.path.join(kvm_instance_partition, second_disk)
subprocess.check_call([
qemu_img, "create", "-f", "qcow2", os.path.join(
kvm_instance_partition, self.second_disk), "1M"])
self.third_disk = os.path.join(self.working_directory, 'third_disk')
subprocess.check_call([
qemu_img, "create", "-f", "qcow2", self.third_disk, "1M"])
self.rerequestInstance({'_': json.dumps(
self.getExternalDiskInstanceParameterDict(
self.first_disk, second_disk, self.third_disk))})
self.raising_waitForInstance(10)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment