Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
slapos
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
104
Merge Requests
104
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Jobs
Commits
Open sidebar
nexedi
slapos
Commits
f5086bc4
Commit
f5086bc4
authored
Oct 01, 2014
by
Marco Mariani
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
pbs: refactored, use template for push_raw, quote with shlex for security
parent
7f1e3771
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
126 additions
and
86 deletions
+126
-86
slapos/recipe/pbs.py
slapos/recipe/pbs.py
+126
-86
No files found.
slapos/recipe/pbs.py
View file @
f5086bc4
...
...
@@ -36,6 +36,7 @@ from slapos.recipe.librecipe import GenericSlapRecipe
from
slapos.recipe.dropbear
import
KnownHostsFile
from
slapos.recipe.notifier
import
Notify
from
slapos.recipe.notifier
import
Callback
from
slapos.recipe.librecipe
import
shlex
def
promise
(
args
):
...
...
@@ -62,89 +63,67 @@ def promise(args):
class
Recipe
(
GenericSlapRecipe
,
Notify
,
Callback
):
def
add_slave
(
self
,
entry
,
known_hosts_file
):
path_list
=
[]
url
=
entry
.
get
(
'url'
)
if
not
url
:
raise
ValueError
(
'Missing URL parameter for PBS recipe'
)
parsed_url
=
urlparse
.
urlparse
(
url
)
slave_type
=
entry
[
'type'
]
if
not
slave_type
in
[
'pull'
,
'push'
]:
raise
ValueError
(
'type parameter must be either pull or push.'
)
slave_id
=
entry
[
'notification-id'
]
print
'Processing PBS slave %s with type %s'
%
(
slave_id
,
slave_type
)
promise_path
=
os
.
path
.
join
(
self
.
options
[
'promises-directory'
],
slave_id
)
promise_dict
=
dict
(
ssh_client
=
self
.
options
[
'sshclient-binary'
],
user
=
parsed_url
.
username
,
host
=
parsed_url
.
hostname
,
port
=
parsed_url
.
port
)
promise
=
self
.
createPythonScript
(
promise_path
,
__name__
+
'.promise'
,
promise_dict
)
path_list
.
append
(
promise
)
host
=
parsed_url
.
hostname
known_hosts_file
[
host
]
=
entry
[
'server-key'
]
def
wrapper_push
(
self
,
remote_schema
,
local_dir
,
remote_dir
,
rdiff_wrapper_path
):
# Create a simple rdiff-backup wrapper that will push
notifier_wrapper_path
=
os
.
path
.
join
(
self
.
options
[
'wrappers-directory'
],
slave_id
)
rdiff_wrapper_path
=
notifier_wrapper_path
+
'_raw'
template
=
textwrap
.
dedent
(
"""
\
#!/bin/sh
#
# Push data to a PBS *-import instance.
#
# Create the rdiff-backup wrapper
# It is useful to separate it from the notifier so that we can run it
# Manually.
rdiffbackup_parameter_list
=
[]
LC_ALL=C
export LC_ALL
RDIFF_BACKUP=%(rdiffbackup_binary)s
until $RDIFF_BACKUP
\
\
--remote-schema %(remote_schema)s
\
\
--restore-as-of now
\
\
--force
\
\
%(local_dir)s
\
\
%(remote_dir)s; do
echo "repeating rdiff-backup..."
sleep 10
done
"""
)
# XXX use -y because the host might not yet be in the
# trusted hosts file until the next time slapgrid is run.
rdiffbackup_remote_schema
=
'%(ssh)s -y -K 300 -p %%s %(user)s@%(host)s'
%
{
'ssh'
:
self
.
options
[
'sshclient-binary'
],
'user'
:
parsed_url
.
username
,
'host'
:
parsed_url
.
hostname
,
template_dict
=
{
'rdiffbackup_binary'
:
shlex
.
quote
(
self
.
options
[
'rdiffbackup-binary'
]),
'remote_schema'
:
shlex
.
quote
(
remote_schema
),
'remote_dir'
:
shlex
.
quote
(
remote_dir
),
'local_dir'
:
shlex
.
quote
(
local_dir
)
}
remote_directory
=
'%(port)s::%(path)s'
%
{
'port'
:
parsed_url
.
port
,
'path'
:
parsed_url
.
path
}
local_directory
=
self
.
createDirectory
(
self
.
options
[
'directory'
],
entry
[
'name'
])
if
slave_type
==
'push'
:
# Create a simple rdiff-backup wrapper that will push
rdiffbackup_parameter_list
.
extend
([
'--remote-schema'
,
rdiffbackup_remote_schema
])
rdiffbackup_parameter_list
.
extend
([
'--restore-as-of'
,
'now'
])
rdiffbackup_parameter_list
.
append
(
'--force'
)
rdiffbackup_parameter_list
.
append
(
local_directory
)
rdiffbackup_parameter_list
.
append
(
remote_directory
)
comments
=
[
''
,
'Push data to a PBS *-import instance.'
,
''
]
rdiff_wrapper
=
self
.
createWrapper
(
return
self
.
createFile
(
name
=
rdiff_wrapper_path
,
command
=
self
.
options
[
'rdiffbackup-binary'
],
parameters
=
rdiffbackup_parameter_list
,
comments
=
comments
,
pidfile
=
os
.
path
.
join
(
self
.
options
[
'run-directory'
],
'%s_raw.pid'
%
slave_id
),
content
=
template
%
template_dict
,
mode
=
0o700
)
elif
slave_type
==
'pull'
:
def
wrapper_pull
(
self
,
remote_schema
,
local_dir
,
remote_dir
,
rdiff_wrapper_path
,
remove_backup_older_than
):
# Wrap rdiff-backup call into a script that checks consistency of backup
# We need to manually escape the remote schema
rdiffbackup_parameter_list
.
extend
([
'--remote-schema'
,
'"%s"'
%
rdiffbackup_remote_schema
])
rdiffbackup_parameter_list
.
append
(
remote_directory
)
rdiffbackup_parameter_list
.
append
(
local_directory
)
comments
=
[
''
,
'Pull data from a PBS *-export instance.'
,
''
]
rdiff_wrapper_template
=
textwrap
.
dedent
(
"""
\
template
=
textwrap
.
dedent
(
"""
\
#!/bin/sh
%(comment)s
#
# Pull data from a PBS *-export instance.
#
LC_ALL=C
export LC_ALL
is_first_backup=$(test -d %(local_directory)s/rdiff-backup-data || echo yes)
RDIFF_BACKUP="%(rdiffbackup_binary)s"
$RDIFF_BACKUP %(rdiffbackup_parameter)s
is_first_backup=$(test -d %(rdiff_backup_data)s || echo yes)
RDIFF_BACKUP=%(rdiffbackup_binary)s
$RDIFF_BACKUP
\
\
--remote-schema %(remote_schema)s
\
\
%(remote_dir)s
\
\
%(local_dir)s
if [ ! $? -eq 0 ]; then
# Check the backup, go to the last consistent backup, so that next
# run will be okay.
echo "Checking backup directory..."
$RDIFF_BACKUP --check-destination-dir %(local_directory
)s
$RDIFF_BACKUP --check-destination-dir %(local_dir
)s
if [ ! $? -eq 0 ]; then
# Here, two possiblities:
if [ is_first_backup ]; then
...
...
@@ -160,11 +139,11 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
fi
else
# Everything's okay, cleaning up...
$RDIFF_BACKUP --remove-older-than %(remove_backup_older_than)s --force %(local_directory
)s
$RDIFF_BACKUP --remove-older-than %(remove_backup_older_than)s --force %(local_dir
)s
fi
if [ -e /srv/slapgrid/slappart17/srv/backup/pbs/COMP-1867-slappart6-runner-2/backup.signature ]; them
cd %(local_directory
)s
if [ -e %(backup_signature)s ]; then
cd %(local_dir
)s
find -type f ! -name backup.signature ! -wholename "./rdiff-backup-data/*" -print0 | xargs -P4 -0 sha256sum | LC_ALL=C sort -k 66 > ../proof.signature
diff -ruw backup.signature ../proof.signature > ../backup.diff
# XXX If there is a difference on the backup, we should publish the
...
...
@@ -172,19 +151,80 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
# instead do a push it to the clone.
fi
"""
)
rdiff_wrapper_content
=
rdiff_wrapper_template
%
{
'comment'
:
''
.
join
((
'# %s
\
n
'
%
comment_line
)
for
comment_line
in
comments
),
'rdiffbackup_binary'
:
self
.
options
[
'rdiffbackup-binary'
],
'local_directory'
:
local_directory
,
'rdiffbackup_parameter'
:
'
\
\
\
n
'
.
join
(
rdiffbackup_parameter_list
),
# XXX: only 3 increments is not enough by default.
'remove_backup_older_than'
:
entry
.
get
(
'remove-backup-older-than'
,
'3B'
)
template_dict
=
{
'rdiffbackup_binary'
:
shlex
.
quote
(
self
.
options
[
'rdiffbackup-binary'
]),
'rdiff_backup_data'
:
shlex
.
quote
(
os
.
path
.
join
(
local_dir
,
'rdiff-backup-data'
)),
'backup_signature'
:
shlex
.
quote
(
os
.
path
.
join
(
local_dir
,
'backup.signature'
)),
'remote_schema'
:
shlex
.
quote
(
remote_schema
),
'remote_dir'
:
shlex
.
quote
(
remote_dir
),
'local_dir'
:
shlex
.
quote
(
local_dir
),
'remove_backup_older_than'
:
shlex
.
quote
(
remove_backup_older_than
)
}
rdiff_wrapper
=
self
.
createFile
(
return
self
.
createFile
(
name
=
rdiff_wrapper_path
,
content
=
rdiff_wrapper_content
,
mode
=
0700
content
=
template
%
template_dict
,
mode
=
0o700
)
def
add_slave
(
self
,
entry
,
known_hosts_file
):
path_list
=
[]
url
=
entry
.
get
(
'url'
)
if
not
url
:
raise
ValueError
(
'Missing URL parameter for PBS recipe'
)
parsed_url
=
urlparse
.
urlparse
(
url
)
slave_type
=
entry
[
'type'
]
if
not
slave_type
in
[
'pull'
,
'push'
]:
raise
ValueError
(
'type parameter must be either pull or push.'
)
slave_id
=
entry
[
'notification-id'
]
print
'Processing PBS slave %s with type %s'
%
(
slave_id
,
slave_type
)
promise_path
=
os
.
path
.
join
(
self
.
options
[
'promises-directory'
],
slave_id
)
promise_dict
=
dict
(
ssh_client
=
self
.
options
[
'sshclient-binary'
],
user
=
parsed_url
.
username
,
host
=
parsed_url
.
hostname
,
port
=
parsed_url
.
port
)
promise
=
self
.
createPythonScript
(
promise_path
,
__name__
+
'.promise'
,
promise_dict
)
path_list
.
append
(
promise
)
known_hosts_file
[
parsed_url
.
hostname
]
=
entry
[
'server-key'
]
notifier_wrapper_path
=
os
.
path
.
join
(
self
.
options
[
'wrappers-directory'
],
slave_id
)
rdiff_wrapper_path
=
notifier_wrapper_path
+
'_raw'
# Create the rdiff-backup wrapper
# It is useful to separate it from the notifier so that we can run it manually.
# XXX use -y because the host might not yet be in the
# trusted hosts file until the next time slapgrid is run.
remote_schema
=
'{ssh} -y -K 300 -p %s {username}@{hostname}'
.
format
(
ssh
=
self
.
options
[
'sshclient-binary'
],
username
=
parsed_url
.
username
,
hostname
=
parsed_url
.
hostname
)
remote_dir
=
'{port}::{path}'
.
format
(
port
=
parsed_url
.
port
,
path
=
parsed_url
.
path
)
local_dir
=
self
.
createDirectory
(
self
.
options
[
'directory'
],
entry
[
'name'
])
if
slave_type
==
'push'
:
rdiff_wrapper
=
self
.
wrapper_push
(
remote_schema
,
local_dir
,
remote_dir
,
rdiff_wrapper_path
)
elif
slave_type
==
'pull'
:
# XXX: only 3 increments is not enough by default.
rdiff_wrapper
=
self
.
wrapper_pull
(
remote_schema
,
local_dir
,
remote_dir
,
rdiff_wrapper_path
,
entry
.
get
(
'remove-backup-older-than'
,
'3B'
))
path_list
.
append
(
rdiff_wrapper
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment