Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
slapos
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Łukasz Nowak
slapos
Commits
45c3e4eb
Commit
45c3e4eb
authored
Mar 13, 2024
by
Łukasz Nowak
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
software/kvm: Switch to incremental backups
parent
b56ede93
Changes
9
Show whitespace changes
Inline
Side-by-side
Showing
9 changed files
with
325 additions
and
64 deletions
+325
-64
software/kvm/buildout.hash.cfg
software/kvm/buildout.hash.cfg
+9
-5
software/kvm/instance-kvm-export.cfg.jinja2
software/kvm/instance-kvm-export.cfg.jinja2
+20
-9
software/kvm/instance-kvm-import.cfg.jinja2.in
software/kvm/instance-kvm-import.cfg.jinja2.in
+2
-4
software/kvm/instance.cfg.in
software/kvm/instance.cfg.in
+21
-3
software/kvm/software.cfg
software/kvm/software.cfg
+17
-1
software/kvm/template/check-backup-directory.sh
software/kvm/template/check-backup-directory.sh
+37
-0
software/kvm/template/kvm-export.sh.jinja2
software/kvm/template/kvm-export.sh.jinja2
+30
-17
software/kvm/template/kvm-import.sh.jinja2
software/kvm/template/kvm-import.sh.jinja2
+24
-6
software/kvm/test/test.py
software/kvm/test/test.py
+165
-19
No files found.
software/kvm/buildout.hash.cfg
View file @
45c3e4eb
...
@@ -15,7 +15,7 @@
...
@@ -15,7 +15,7 @@
[template]
[template]
filename = instance.cfg.in
filename = instance.cfg.in
md5sum =
b6204319cca4264b3c351d4dd1f2b5d0
md5sum =
8ca34c4a0abc36c076da93919c75465f
[template-kvm]
[template-kvm]
filename = instance-kvm.cfg.jinja2
filename = instance-kvm.cfg.jinja2
...
@@ -31,19 +31,19 @@ md5sum = 839fc16c112d3b87e2dbd2e382e326de
...
@@ -31,19 +31,19 @@ md5sum = 839fc16c112d3b87e2dbd2e382e326de
[template-kvm-import]
[template-kvm-import]
filename = instance-kvm-import.cfg.jinja2.in
filename = instance-kvm-import.cfg.jinja2.in
md5sum =
a463a5e3cd2287d275d6943c2a11b7e4
md5sum =
58647cb7c170086cbbebcf957c6701b7
[template-kvm-import-script]
[template-kvm-import-script]
filename = template/kvm-import.sh.jinja2
filename = template/kvm-import.sh.jinja2
md5sum =
cd0008f1689dfca9b77370bc4d275b70
md5sum =
843e8205cd08437a04f673bcb842b162
[template-kvm-export]
[template-kvm-export]
filename = instance-kvm-export.cfg.jinja2
filename = instance-kvm-export.cfg.jinja2
md5sum =
34d1b7cc8ca62bfdfce759a1dfbbaccd
md5sum =
bb440d85dd0265ca7a2b0a818eaa8e33
[template-kvm-export-script]
[template-kvm-export-script]
filename = template/kvm-export.sh.jinja2
filename = template/kvm-export.sh.jinja2
md5sum =
64aa1ce8785f6b94aabd787fa3443082
md5sum =
4b0cfe333df0c2834d2282d5fc545e27
[template-nbd]
[template-nbd]
filename = instance-nbd.cfg.jinja2
filename = instance-nbd.cfg.jinja2
...
@@ -96,3 +96,7 @@ md5sum = b4f6ffef08685bace1b9c01a3bd2620d
...
@@ -96,3 +96,7 @@ md5sum = b4f6ffef08685bace1b9c01a3bd2620d
[whitelist-domains-default]
[whitelist-domains-default]
filename = template/whitelist-domains-default
filename = template/whitelist-domains-default
md5sum = e9d40162ba77472775256637a2617d14
md5sum = e9d40162ba77472775256637a2617d14
[check-backup-directory.sh]
filename = template/check-backup-directory.sh
md5sum = 368b6fd76c3f64b99d2b123b57ae5cff
software/kvm/instance-kvm-export.cfg.jinja2
View file @
45c3e4eb
...
@@ -6,6 +6,7 @@ extends =
...
@@ -6,6 +6,7 @@ extends =
parts +=
parts +=
cron-entry-backup
cron-entry-backup
${instance-kvm-parts:parts}
${instance-kvm-parts:parts}
check-backup-directory-promise
[slap-parameter]
[slap-parameter]
{% for k, v in slapparameter_dict.items() -%}
{% for k, v in slapparameter_dict.items() -%}
...
@@ -16,13 +17,10 @@ parts +=
...
@@ -16,13 +17,10 @@ parts +=
{%- endif %}
{%- endif %}
{% endfor -%}
{% endfor -%}
# Create the exporter executable, which is a simple shell script
[directory]
[exporter]
tmp = ${buildout:directory}/tmp
recipe = slapos.recipe.template:jinja2
url = {{ template_kvm_export }}
[disk]
output = ${directory:bin}/${slap-parameter:namebase}-exporter
# Resilient stack wants a "wrapper" parameter
wrapper = ${:output}
{%- set disk_type = slapparameter_dict.get('disk-type', 'virtio') %}
{%- set disk_type = slapparameter_dict.get('disk-type', 'virtio') %}
{%- if disk_type == "virtio" %}
{%- if disk_type == "virtio" %}
device = virtio0
device = virtio0
...
@@ -32,12 +30,19 @@ device = ide0-hd0
...
@@ -32,12 +30,19 @@ device = ide0-hd0
{%- else %}
{%- else %}
# unsupported disk-type {{ disk_type }}
# unsupported disk-type {{ disk_type }}
{%- endif %}
{%- endif %}
[exporter]
recipe = slapos.recipe.template:jinja2
url = {{ template_kvm_export }}
output = ${directory:bin}/${slap-parameter:namebase}-exporter
# Resilient stack wants a "wrapper" parameter
wrapper = ${:output}
context =
context =
section directory directory
section directory directory
section buildout buildout
section buildout buildout
key socket_path kvm-instance:socket-path
key socket_path kvm-instance:socket-path
key device :device
section disk disk
raw
gzip_binary {{ gzip_binary
}}
raw
qmpbackup {{ qmpbackup
}}
# Extends publish section with resilient parameters
# Extends publish section with resilient parameters
[publish-connection-information]
[publish-connection-information]
...
@@ -56,3 +61,9 @@ username = {{ slapparameter_dict['monitor-username'] }}
...
@@ -56,3 +61,9 @@ username = {{ slapparameter_dict['monitor-username'] }}
password = {{ slapparameter_dict['monitor-password'] }}
password = {{ slapparameter_dict['monitor-password'] }}
{% endif -%}
{% endif -%}
[check-backup-directory-promise]
# Check that disk image is not corrupted
<= monitor-promise-base
promise = check_command_execute
name = check-backup-directory.py
config-command = {{ check_backup_directory }} ${directory:backup}/${disk:device} ${directory:tmp}
software/kvm/instance-kvm-import.cfg.jinja2.in
View file @
45c3e4eb
...
@@ -48,6 +48,7 @@ bin = ${buildout:directory}/bin
...
@@ -48,6 +48,7 @@ bin = ${buildout:directory}/bin
srv = ${buildout:directory}/srv
srv = ${buildout:directory}/srv
var = ${buildout:directory}/var
var = ${buildout:directory}/var
log = ${:var}/log
log = ${:var}/log
tmp = ${buildout:directory}/tmp
scripts = ${:etc}/run
scripts = ${:etc}/run
services = ${:etc}/service
services = ${:etc}/service
novnc-conf = ${:etc}/novnc
novnc-conf = ${:etc}/novnc
...
@@ -65,10 +66,7 @@ output = ${directory:bin}/${slap-parameter:namebase}-importer
...
@@ -65,10 +66,7 @@ output = ${directory:bin}/${slap-parameter:namebase}-importer
wrapper = ${:output}
wrapper = ${:output}
context =
context =
section directory directory
section directory directory
raw zcat_binary {{ zcat_binary }}
raw qmprebase {{ qmprebase }}
raw gzip_binary {{ gzip_binary }}
backup-disk-path = ${directory:backup}/virtual.qcow2
[kvm-disk-image-corruption-bin]
[kvm-disk-image-corruption-bin]
recipe = collective.recipe.template
recipe = collective.recipe.template
...
...
software/kvm/instance.cfg.in
View file @
45c3e4eb
...
@@ -126,6 +126,15 @@ template-replicated-destination = ${template-replicated:target}
...
@@ -126,6 +126,15 @@ template-replicated-destination = ${template-replicated:target}
import-list = file parts :template-parts-destination
import-list = file parts :template-parts-destination
file replicated :template-replicated-destination
file replicated :template-replicated-destination
[qmpbackup-binary]
recipe = slapos.cookbook:wrapper
environment =
PATH=${qemu:location}/bin
wrapper-path =
$${buildout:bin-directory}/qmpbackup
command-line =
${buildout:bin-directory}/qmpbackup
[dynamic-template-kvm-export]
[dynamic-template-kvm-export]
recipe = slapos.recipe.template:jinja2
recipe = slapos.recipe.template:jinja2
url = ${template-kvm-export:location}/instance-kvm-export.cfg.jinja2
url = ${template-kvm-export:location}/instance-kvm-export.cfg.jinja2
...
@@ -137,8 +146,18 @@ context =
...
@@ -137,8 +146,18 @@ context =
raw kvm_template $${dynamic-template-kvm:output}
raw kvm_template $${dynamic-template-kvm:output}
raw template_kvm_export ${template-kvm-export-script:target}
raw template_kvm_export ${template-kvm-export-script:target}
key pbsready_export_template template-pbsready-export:output
key pbsready_export_template template-pbsready-export:output
raw gzip_binary ${gzip:location}/bin/gzip
key slapparameter_dict slap-configuration:configuration
key slapparameter_dict slap-configuration:configuration
key qmpbackup qmpbackup-binary:wrapper-path
raw check_backup_directory ${check-backup-directory.sh:output}
[qmprebase-binary]
recipe = slapos.cookbook:wrapper
environment =
PATH=${qemu:location}/bin
wrapper-path =
$${buildout:bin-directory}/qmprebase
command-line =
${buildout:bin-directory}/qmprebase
[dynamic-template-kvm-import]
[dynamic-template-kvm-import]
recipe = slapos.recipe.template:jinja2
recipe = slapos.recipe.template:jinja2
...
@@ -152,8 +171,7 @@ context =
...
@@ -152,8 +171,7 @@ context =
raw template_kvm_import ${template-kvm-import-script:target}
raw template_kvm_import ${template-kvm-import-script:target}
key pbsready_import_template template-pbsready-import:output
key pbsready_import_template template-pbsready-import:output
key slapparameter_dict slap-configuration:configuration
key slapparameter_dict slap-configuration:configuration
raw zcat_binary ${gzip:location}/bin/zcat
key qmprebase qmprebase-binary:wrapper-path
raw gzip_binary ${gzip:location}/bin/gzip
[dynamic-template-nbd]
[dynamic-template-nbd]
<= jinja2-template-base
<= jinja2-template-base
...
...
software/kvm/software.cfg
View file @
45c3e4eb
...
@@ -10,7 +10,6 @@ extends =
...
@@ -10,7 +10,6 @@ extends =
../../component/netcat/buildout.cfg
../../component/netcat/buildout.cfg
../../component/nginx/buildout.cfg
../../component/nginx/buildout.cfg
../../component/pycurl/buildout.cfg
../../component/pycurl/buildout.cfg
../../component/gzip/buildout.cfg
../../stack/slapos.cfg
../../stack/slapos.cfg
../../stack/resilient/buildout.cfg
../../stack/resilient/buildout.cfg
buildout.hash.cfg
buildout.hash.cfg
...
@@ -19,6 +18,7 @@ extends =
...
@@ -19,6 +18,7 @@ extends =
# to avoid versioning issues
# to avoid versioning issues
common-parts =
common-parts =
template
template
qmpbackup
# XXX: we have to manually add this for resilience
# XXX: we have to manually add this for resilience
pbs-recipe-egg
pbs-recipe-egg
...
@@ -32,6 +32,18 @@ parts = ${:common-parts}
...
@@ -32,6 +32,18 @@ parts = ${:common-parts}
# In qemu builtin vnc server, and make it available only for localhost
# In qemu builtin vnc server, and make it available only for localhost
# so that only novnc can listen to it.
# so that only novnc can listen to it.
[qmpbackup]
recipe = zc.recipe.egg
eggs =
qemu.qmp
qmpbackup
find-links +=
https://github.com/abbbi/qmpbackup/releases/download/v0.29/qmpbackup-0.29.tar.gz
[versions]
qemu.qmp = 0.0.3:whl
qmpbackup = 0.29
[python-with-eggs]
[python-with-eggs]
recipe = zc.recipe.egg
recipe = zc.recipe.egg
interpreter = ${:_buildout_section_name_}
interpreter = ${:_buildout_section_name_}
...
@@ -129,3 +141,7 @@ context =
...
@@ -129,3 +141,7 @@ context =
[whitelist-domains-default]
[whitelist-domains-default]
<= download-base
<= download-base
[check-backup-directory.sh]
<= template-base
output = ${buildout:parts-directory}/${:_buildout_section_name_}/check-backup-directory.sh
software/kvm/template/check-backup-directory.sh
0 → 100644
View file @
45c3e4eb
#!/bin/bash
directory
=
$1
tmp
=
$2
# support a case of not ready yet directory
if
[
!
-d
$directory
]
;
then
exit
0
fi
tmpfile
=
$(
mktemp
-p
$tmp
)
trap
"rm -fr
$tmpfile
"
EXIT TERM INT
find
$directory
-type
f
-name
'FULL*qcow2'
-printf
'%f\n'
>
$tmpfile
full_amount
=
$(
wc
-l
$tmpfile
|
cut
-d
' '
-f
1
)
if
[
$full_amount
-ne
1
]
&&
[
$full_amount
-ne
0
]
;
then
echo
"Too many FULL backups"
cat
$tmpfile
exit
1
fi
find
$directory
-type
f
-name
'INC*qcow2'
-printf
'%f\n'
>
$tmpfile
if
[
$(
wc
-l
$tmpfile
|
cut
-d
' '
-f
1
)
-gt
0
]
&&
[
$full_amount
-eq
0
]
;
then
echo
"INC present but no FULL backup"
cat
$tmpfile
exit
1
fi
find
$directory
-type
f
-name
'*.partial'
-printf
'%f\n'
>
$tmpfile
if
[
$(
wc
-l
$tmpfile
|
cut
-d
' '
-f
1
)
-ne
0
]
;
then
echo
"Partial file present"
cat
$tmpfile
exit
1
fi
exit
0
software/kvm/template/kvm-export.sh.jinja2
View file @
45c3e4eb
...
@@ -6,25 +6,38 @@ set -e
...
@@ -6,25 +6,38 @@ set -e
LC_ALL
=
C
LC_ALL
=
C
export
LC_ALL
export
LC_ALL
BACKUP_DIR
={{
directory[
'backup'
]
}}
BACKUP_DIR
={{
directory[
'backup'
]
}}
BACKUP_FILE
=
virtual.qcow2
QMP_CLIENT
={{
buildout[
'directory'
]
}}
/software_release/bin/qemu-qmp-client
log
=
$(
mktemp
--tmpdir
={{
directory[
'tmp'
]
}}
)
trap
"rm -f
$log
"
EXIT TERM INT
$QMP_CLIENT
--socket
{{
socket_path
}}
--drive-backup
$BACKUP_DIR
/
$BACKUP_FILE
{{
device
}}
set
+e
# Due to the way qmp works, the VM file cannot be compressed on the fly.
qmpbackup
=
"{{ qmpbackup }} --socket {{ socket_path }} backup --compress --target
$BACKUP_DIR
--include {{ disk['device'] }}"
# Although the compression step is optional, the importer uses the .gz file
$qmpbackup
--level
auto
>
$log
# if present. So, remove it if you are disabling the compression.
RESULT
=
$?
cat
$log
# The downside of compression, here, is the temporary usage of more disk space
if
[
$RESULT
-ne
0
]
;
then
# in the exporter node. The goal is to minimize disk usage on the PBS server.
# recover from unfinished previous backup
if
egrep
-q
'Partial backup found in.*{{ disk['
device
']}}.*possible broken backup chain. Execute new full backup'
$log
;
then
find
$BACKUP_DIR
/
{{
disk[
'device'
]
}}
-name
'*.partial'
-delete
$qmpbackup
--level
auto
||
exit
$?
echo
"Recovered from partial backup by removing partial"
else
exit
$RESULT
fi
fi
set
-e
# If you want to compress the file in-place:
# as new style backup went find delete eventual old style backup
# truncate -s $(gzip -c $BACKUP_FILE | dd of=$BACKUP_FILE conv=notrunc 2>&1 | sed -n '$ s/ .*$// p') $BACKUP_FILE
rm
-f
$BACKUP_DIR
/virtual.qcow2
{
,.gz
}
# but the importer script would have to be adapted.
echo
"Compressing backup..."
# cleanup the backup directory from too old backups, especially important after take-over
{{
gzip_binary
}}
--force
--rsyncable
$BACKUP_DIR
/
$BACKUP_FILE
recent_full
=
$(
find
$BACKUP_DIR
-type
f
-name
'FULL-*.qcow2'
-exec
ls
-t1
{}
+ |
head
-n1
)
if
[
x
"
$recent_full
"
!=
x
""
]
;
then
for
f
in
$(
find
$BACKUP_DIR
-type
f
-name
'*qcow2'
\!
-newer
$recent_full
)
;
do
if
[
"
$f
"
!=
"
$recent_full
"
]
;
then
rm
-vf
$f
fi
done
fi
cd
$BACKUP_DIR
&&
find
-type
f
!
-name
backup.signature
-print0
| xargs
-0
sha256sum
|
LC_ALL
=
C
sort
-k
66
>
backup.signature
cd
$BACKUP_DIR
&&
find
-type
f
!
-name
backup.signature
-print0
| xargs
-0
sha256sum
|
LC_ALL
=
C
sort
-k
66
>
backup.signature
software/kvm/template/kvm-import.sh.jinja2
View file @
45c3e4eb
...
@@ -2,7 +2,6 @@
...
@@ -2,7 +2,6 @@
VM_DIR
={{
directory[
'srv'
]
}}
VM_DIR
={{
directory[
'srv'
]
}}
BACKUP_DIR
={{
directory[
'backup'
]
}}
BACKUP_DIR
={{
directory[
'backup'
]
}}
VM_FILE
=
virtual.qcow2
LC_ALL
=
C
LC_ALL
=
C
export
LC_ALL
export
LC_ALL
umask
077
umask
077
...
@@ -16,11 +15,30 @@ write_backup_proof () {
...
@@ -16,11 +15,30 @@ write_backup_proof () {
# For now we just make the diff before
# For now we just make the diff before
write_backup_proof
write_backup_proof
if
[
-f
"
$BACKUP_DIR
/
${
VM_FILE
}
.gz"
]
;
then
tmpfile
=
$(
mktemp
--tmpdir
={{
directory[
'tmp'
]
}}
)
{{
gzip_binary
}}
-t
"
$BACKUP_DIR
/
${
VM_FILE
}
.gz"
||
exit
10
# use temporary space inside of the partition, as it can be quite big
{{
zcat_binary
}}
"
$BACKUP_DIR
/
${
VM_FILE
}
.gz"
>
$VM_DIR
/
$VM_FILE
tmpdir
=
$(
mktemp
-d
--tmpdir
={{
directory[
'tmp'
]
}}
)
# assure the temporary directory is cleaned up
trap
"rm -fr
$tmpdir
$tmpfile
"
EXIT TERM INT
cp
-a
$BACKUP_DIR
/
*
$tmpdir
if
[
-d
$tmpdir
/ide0-hd0
]
;
then
disk_type
=
"ide0-hd0"
elif
[
-d
$tmpdir
/virtio0
]
;
then
disk_type
=
virtio0
else
else
rm
$VM_DIR
/
$VM_FILE
echo
"Unsupported disk type"
cp
$BACKUP_DIR
/
$VM_FILE
$VM_DIR
/
$VM_FILE
exit
1
fi
fi
if
[
$(
find
$tmpdir
/
$disk_type
-name
'INC-*'
|
wc
-l
|
cut
-d
' '
-f
1
)
-gt
0
]
;
then
{{
qmprebase
}}
rebase
--dir
$tmpdir
/
$disk_type
fi
# assert that restore went well and there is only one file
find
$tmpdir
/
$disk_type
-type
f
-name
'FULL*qcow2'
-printf
'%f\n'
>
$tmpfile
if
[
$(
wc
-l
$tmpfile
|
cut
-d
' '
-f
1
)
-ne
1
]
;
then
echo
"Wrong amount of FULL backups"
cat
$tmpfile
exit
1
fi
cp
$tmpdir
/
$disk_type
/
*
$VM_DIR
/virtual.qcow2
software/kvm/test/test.py
View file @
45c3e4eb
...
@@ -656,6 +656,171 @@ class TestAccessKvmClusterBootstrap(MonitorAccessMixin, KVMTestCase):
...
@@ -656,6 +656,171 @@ class TestAccessKvmClusterBootstrap(MonitorAccessMixin, KVMTestCase):
self
.
assertIn
(
'<title>noVNC</title>'
,
result
.
text
)
self
.
assertIn
(
'<title>noVNC</title>'
,
result
.
text
)
class
TestInstanceResilientBackupMixin
(
KvmMixin
):
__partition_reference__
=
'irb'
instance_max_retry
=
20
disk_type
=
'virtio'
disk_type_backup_mapping
=
{
'virtio'
:
'virtio0'
,
'ide'
:
'ide0-hd0'
,
}
@
classmethod
def
getInstanceParameterDict
(
cls
):
parameter_dict
=
{}
if
cls
.
disk_type
!=
'virtio'
:
parameter_dict
[
'disk-type'
]
=
cls
.
disk_type
return
parameter_dict
@
classmethod
def
getInstanceSoftwareType
(
cls
):
return
'kvm-resilient'
def
setUp
(
self
):
super
().
setUp
()
exporter_partition
=
glob
.
glob
(
os
.
path
.
join
(
self
.
slap
.
instance_directory
,
'*'
,
'template-kvm-export.cfg'
))
self
.
assertEqual
(
1
,
len
(
exporter_partition
))
self
.
exporter_partition
=
os
.
path
.
dirname
(
exporter_partition
[
0
])
self
.
exporter
=
os
.
path
.
join
(
self
.
exporter_partition
,
'bin'
,
'exporter'
)
self
.
backup_directory
=
os
.
path
.
join
(
self
.
exporter_partition
,
'srv'
,
'backup'
,
'kvm'
,
self
.
disk_type_backup_mapping
[
self
.
disk_type
])
importer_partition
=
glob
.
glob
(
os
.
path
.
join
(
self
.
slap
.
instance_directory
,
'*'
,
'template-kvm-import.cfg'
))
self
.
assertEqual
(
1
,
len
(
importer_partition
))
self
.
importer_partition
=
os
.
path
.
dirname
(
importer_partition
[
0
])
def
call_exporter
(
self
,
exporter
=
None
):
if
exporter
is
None
:
exporter
=
self
.
exporter
try
:
return
(
0
,
subprocess
.
check_output
(
[
exporter
],
stderr
=
subprocess
.
STDOUT
).
decode
(
'utf-8'
))
except
subprocess
.
CalledProcessError
as
e
:
return
(
e
.
returncode
,
e
.
output
.
decode
(
'utf-8'
))
@
skipUnlessKvm
class
TestInstanceResilientBackupImporter
(
TestInstanceResilientBackupMixin
,
KVMTestCase
):
def
test
(
self
):
equeue_file
=
os
.
path
.
join
(
self
.
importer_partition
,
'var'
,
'log'
,
'equeue.log'
)
destination_qcow2
=
os
.
path
.
join
(
self
.
importer_partition
,
'srv'
,
'virtual.qcow2'
)
destination_backup
=
os
.
path
.
join
(
self
.
importer_partition
,
'srv'
,
'backup'
,
'kvm'
,
self
.
disk_type_backup_mapping
[
self
.
disk_type
])
# sanity check - no export/import happened yet
self
.
assertFalse
(
os
.
path
.
exists
(
self
.
backup_directory
))
status_code
,
status_text
=
self
.
call_exporter
()
self
.
assertEqual
(
0
,
status_code
,
status_text
)
def
awaitBackup
(
equeue_file
):
for
f
in
range
(
30
):
with
open
(
equeue_file
,
'r'
)
as
fh
:
equeue_log
=
fh
.
read
()
if
'finished successfully'
in
equeue_log
:
break
time
.
sleep
(
1
)
else
:
self
.
fail
(
'Backup not finished: %s'
%
(
equeue_log
))
return
equeue_log
equeue_log
=
awaitBackup
(
equeue_file
)
self
.
assertNotIn
(
'qemu-img rebase'
,
equeue_log
)
self
.
assertEqual
(
os
.
listdir
(
self
.
backup_directory
),
os
.
listdir
(
destination_backup
)
)
self
.
assertTrue
(
os
.
path
.
exists
(
destination_qcow2
))
# clean up equeue file for precise assertion
with
open
(
equeue_file
,
'w'
)
as
fh
:
fh
.
write
(
''
)
# drop backup destination to assert it's recreation
os
.
unlink
(
destination_qcow2
)
status_code
,
status_text
=
self
.
call_exporter
()
self
.
assertEqual
(
0
,
status_code
,
status_text
)
equeue_log
=
awaitBackup
(
equeue_file
)
self
.
assertIn
(
'qemu-img rebase'
,
equeue_log
)
self
.
assertEqual
(
os
.
listdir
(
self
.
backup_directory
),
os
.
listdir
(
destination_backup
)
)
self
.
assertTrue
(
os
.
path
.
exists
(
destination_qcow2
))
# takeover
connection_parameter
=
self
.
computer_partition
.
getConnectionParameterDict
()
takeover_result
=
requests
.
post
(
connection_parameter
[
'takeover-kvm-1-url'
],
data
=
{
'password'
:
connection_parameter
[
'takeover-kvm-1-password'
]})
self
.
assertEqual
(
httplib
.
OK
,
takeover_result
.
status_code
)
self
.
assertTrue
(
takeover_result
.
text
.
startswith
(
'Success.'
))
# the real assertions comes from re-stabilizing the instance tree
self
.
slap
.
waitForInstance
(
max_retry
=
10
)
# check that all stabilizes after backup after takeover
new_exporter_partition
=
glob
.
glob
(
os
.
path
.
join
(
self
.
slap
.
instance_directory
,
'*'
,
'template-kvm-export.cfg'
))
self
.
assertEqual
(
1
,
len
(
new_exporter_partition
))
new_exporter_partition
=
os
.
path
.
dirname
(
new_exporter_partition
[
0
])
new_exporter
=
os
.
path
.
join
(
new_exporter_partition
,
'bin'
,
'exporter'
)
self
.
assertNotEqual
(
new_exporter
,
self
.
exporter
)
status_code
,
status_text
=
self
.
call_exporter
(
new_exporter
)
self
.
assertEqual
(
0
,
status_code
,
status_text
)
self
.
slap
.
waitForInstance
(
max_retry
=
10
)
@
skipUnlessKvm
class
TestInstanceResilientBackupImporterIde
(
TestInstanceResilientBackupImporter
):
disk_type
=
'ide'
@
skipUnlessKvm
class
TestInstanceResilientBackupExporter
(
TestInstanceResilientBackupMixin
,
KVMTestCase
):
def
test
(
self
):
status_code
,
status_text
=
self
.
call_exporter
()
self
.
assertEqual
(
0
,
status_code
,
status_text
)
self
.
assertEqual
(
len
(
glob
.
glob
(
os
.
path
.
join
(
self
.
backup_directory
,
'FULL-*.qcow2'
))),
1
)
self
.
assertEqual
(
len
(
glob
.
glob
(
os
.
path
.
join
(
self
.
backup_directory
,
'INC-*.qcow2'
))),
0
)
self
.
assertNotIn
(
'Recovered from partial backup by removing partial'
,
status_text
)
# cover .partial file in the backup directory with fallback to full
current_backup
=
glob
.
glob
(
os
.
path
.
join
(
self
.
backup_directory
,
'FULL-*'
))[
0
]
with
open
(
current_backup
+
'.partial'
,
'w'
)
as
fh
:
fh
.
write
(
''
)
status_code
,
status_text
=
self
.
call_exporter
()
self
.
assertEqual
(
0
,
status_code
,
status_text
)
self
.
assertEqual
(
len
(
glob
.
glob
(
os
.
path
.
join
(
self
.
backup_directory
,
'FULL-*.qcow2'
))),
1
)
self
.
assertEqual
(
len
(
glob
.
glob
(
os
.
path
.
join
(
self
.
backup_directory
,
'INC-*.qcow2'
))),
1
)
self
.
assertIn
(
'Recovered from partial backup by removing partial'
,
status_text
)
self
.
assertTrue
(
os
.
path
.
exists
(
os
.
path
.
join
(
self
.
exporter_partition
,
'etc'
,
'plugin'
,
'check-backup-directory.py'
)))
@
skipUnlessKvm
class
TestInstanceResilientBackupExporterIde
(
TestInstanceResilientBackupExporter
):
disk_type
=
'ide'
@
skipUnlessKvm
@
skipUnlessKvm
class
TestInstanceResilient
(
KVMTestCase
,
KvmMixin
):
class
TestInstanceResilient
(
KVMTestCase
,
KvmMixin
):
__partition_reference__
=
'ir'
__partition_reference__
=
'ir'
...
@@ -672,25 +837,6 @@ class TestInstanceResilient(KVMTestCase, KvmMixin):
...
@@ -672,25 +837,6 @@ class TestInstanceResilient(KVMTestCase, KvmMixin):
cls
.
kvm0_ipv6
=
cls
.
getPartitionIPv6
(
cls
.
getPartitionId
(
'kvm0'
))
cls
.
kvm0_ipv6
=
cls
.
getPartitionIPv6
(
cls
.
getPartitionId
(
'kvm0'
))
cls
.
kvm1_ipv6
=
cls
.
getPartitionIPv6
(
cls
.
getPartitionId
(
'kvm1'
))
cls
.
kvm1_ipv6
=
cls
.
getPartitionIPv6
(
cls
.
getPartitionId
(
'kvm1'
))
def
test_kvm_exporter
(
self
):
exporter_partition
=
os
.
path
.
join
(
self
.
slap
.
instance_directory
,
self
.
__partition_reference__
+
'2'
)
backup_path
=
os
.
path
.
join
(
exporter_partition
,
'srv'
,
'backup'
,
'kvm'
,
'virtual.qcow2.gz'
)
exporter
=
os
.
path
.
join
(
exporter_partition
,
'bin'
,
'exporter'
)
if
os
.
path
.
exists
(
backup_path
):
os
.
unlink
(
backup_path
)
def
call_exporter
():
try
:
return
(
0
,
subprocess
.
check_output
(
[
exporter
],
stderr
=
subprocess
.
STDOUT
).
decode
(
'utf-8'
))
except
subprocess
.
CalledProcessError
as
e
:
return
(
e
.
returncode
,
e
.
output
.
decode
(
'utf-8'
))
status_code
,
status_text
=
call_exporter
()
self
.
assertEqual
(
0
,
status_code
,
status_text
)
def
test
(
self
):
def
test
(
self
):
connection_parameter_dict
=
self
\
connection_parameter_dict
=
self
\
.
computer_partition
.
getConnectionParameterDict
()
.
computer_partition
.
getConnectionParameterDict
()
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment