Commit f5086bc4 authored by Marco Mariani's avatar Marco Mariani

pbs: refactored, use template for push_raw, quote with shlex for security

parent 7f1e3771
......@@ -36,6 +36,7 @@ from slapos.recipe.librecipe import GenericSlapRecipe
from slapos.recipe.dropbear import KnownHostsFile
from slapos.recipe.notifier import Notify
from slapos.recipe.notifier import Callback
from slapos.recipe.librecipe import shlex
def promise(args):
......@@ -62,89 +63,67 @@ def promise(args):
class Recipe(GenericSlapRecipe, Notify, Callback):
def add_slave(self, entry, known_hosts_file):
path_list = []
url = entry.get('url')
if not url:
raise ValueError('Missing URL parameter for PBS recipe')
parsed_url = urlparse.urlparse(url)
slave_type = entry['type']
if not slave_type in ['pull', 'push']:
raise ValueError('type parameter must be either pull or push.')
slave_id = entry['notification-id']
print 'Processing PBS slave %s with type %s' % (slave_id, slave_type)
promise_path = os.path.join(self.options['promises-directory'], slave_id)
promise_dict = dict(ssh_client=self.options['sshclient-binary'],
user=parsed_url.username,
host=parsed_url.hostname,
port=parsed_url.port)
promise = self.createPythonScript(promise_path,
__name__ + '.promise',
promise_dict)
path_list.append(promise)
host = parsed_url.hostname
known_hosts_file[host] = entry['server-key']
def wrapper_push(self, remote_schema, local_dir, remote_dir, rdiff_wrapper_path):
# Create a simple rdiff-backup wrapper that will push
notifier_wrapper_path = os.path.join(self.options['wrappers-directory'], slave_id)
rdiff_wrapper_path = notifier_wrapper_path + '_raw'
template = textwrap.dedent("""\
#!/bin/sh
#
# Push data to a PBS *-import instance.
#
# Create the rdiff-backup wrapper
# It is useful to separate it from the notifier so that we can run it
# Manually.
rdiffbackup_parameter_list = []
LC_ALL=C
export LC_ALL
RDIFF_BACKUP=%(rdiffbackup_binary)s
until $RDIFF_BACKUP \\
--remote-schema %(remote_schema)s \\
--restore-as-of now \\
--force \\
%(local_dir)s \\
%(remote_dir)s; do
echo "repeating rdiff-backup..."
sleep 10
done
""")
# XXX use -y because the host might not yet be in the
# trusted hosts file until the next time slapgrid is run.
rdiffbackup_remote_schema = '%(ssh)s -y -K 300 -p %%s %(user)s@%(host)s' % {
'ssh': self.options['sshclient-binary'],
'user': parsed_url.username,
'host': parsed_url.hostname,
template_dict = {
'rdiffbackup_binary': shlex.quote(self.options['rdiffbackup-binary']),
'remote_schema': shlex.quote(remote_schema),
'remote_dir': shlex.quote(remote_dir),
'local_dir': shlex.quote(local_dir)
}
remote_directory = '%(port)s::%(path)s' % {'port': parsed_url.port,
'path': parsed_url.path}
local_directory = self.createDirectory(self.options['directory'], entry['name'])
if slave_type == 'push':
# Create a simple rdiff-backup wrapper that will push
rdiffbackup_parameter_list.extend(['--remote-schema', rdiffbackup_remote_schema])
rdiffbackup_parameter_list.extend(['--restore-as-of', 'now'])
rdiffbackup_parameter_list.append('--force')
rdiffbackup_parameter_list.append(local_directory)
rdiffbackup_parameter_list.append(remote_directory)
comments = ['', 'Push data to a PBS *-import instance.', '']
rdiff_wrapper = self.createWrapper(
return self.createFile(
name=rdiff_wrapper_path,
command=self.options['rdiffbackup-binary'],
parameters=rdiffbackup_parameter_list,
comments=comments,
pidfile=os.path.join(self.options['run-directory'], '%s_raw.pid' % slave_id),
content=template % template_dict,
mode=0o700
)
elif slave_type == 'pull':
def wrapper_pull(self, remote_schema, local_dir, remote_dir, rdiff_wrapper_path, remove_backup_older_than):
# Wrap rdiff-backup call into a script that checks consistency of backup
# We need to manually escape the remote schema
rdiffbackup_parameter_list.extend(['--remote-schema', '"%s"' % rdiffbackup_remote_schema])
rdiffbackup_parameter_list.append(remote_directory)
rdiffbackup_parameter_list.append(local_directory)
comments = ['', 'Pull data from a PBS *-export instance.', '']
rdiff_wrapper_template = textwrap.dedent("""\
template = textwrap.dedent("""\
#!/bin/sh
%(comment)s
#
# Pull data from a PBS *-export instance.
#
LC_ALL=C
export LC_ALL
is_first_backup=$(test -d %(local_directory)s/rdiff-backup-data || echo yes)
RDIFF_BACKUP="%(rdiffbackup_binary)s"
$RDIFF_BACKUP %(rdiffbackup_parameter)s
is_first_backup=$(test -d %(rdiff_backup_data)s || echo yes)
RDIFF_BACKUP=%(rdiffbackup_binary)s
$RDIFF_BACKUP \\
--remote-schema %(remote_schema)s \\
%(remote_dir)s \\
%(local_dir)s
if [ ! $? -eq 0 ]; then
# Check the backup, go to the last consistent backup, so that next
# run will be okay.
echo "Checking backup directory..."
$RDIFF_BACKUP --check-destination-dir %(local_directory)s
$RDIFF_BACKUP --check-destination-dir %(local_dir)s
if [ ! $? -eq 0 ]; then
# Here, two possiblities:
if [ is_first_backup ]; then
......@@ -160,11 +139,11 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
fi
else
# Everything's okay, cleaning up...
$RDIFF_BACKUP --remove-older-than %(remove_backup_older_than)s --force %(local_directory)s
$RDIFF_BACKUP --remove-older-than %(remove_backup_older_than)s --force %(local_dir)s
fi
if [ -e /srv/slapgrid/slappart17/srv/backup/pbs/COMP-1867-slappart6-runner-2/backup.signature ]; them
cd %(local_directory)s
if [ -e %(backup_signature)s ]; then
cd %(local_dir)s
find -type f ! -name backup.signature ! -wholename "./rdiff-backup-data/*" -print0 | xargs -P4 -0 sha256sum | LC_ALL=C sort -k 66 > ../proof.signature
diff -ruw backup.signature ../proof.signature > ../backup.diff
# XXX If there is a difference on the backup, we should publish the
......@@ -172,19 +151,80 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
# instead do a push it to the clone.
fi
""")
rdiff_wrapper_content = rdiff_wrapper_template % {
'comment': ''.join(('# %s\n' % comment_line) for comment_line in comments),
'rdiffbackup_binary': self.options['rdiffbackup-binary'],
'local_directory': local_directory,
'rdiffbackup_parameter': ' \\\n '.join(rdiffbackup_parameter_list),
# XXX: only 3 increments is not enough by default.
'remove_backup_older_than': entry.get('remove-backup-older-than', '3B')
template_dict = {
'rdiffbackup_binary': shlex.quote(self.options['rdiffbackup-binary']),
'rdiff_backup_data': shlex.quote(os.path.join(local_dir, 'rdiff-backup-data')),
'backup_signature': shlex.quote(os.path.join(local_dir, 'backup.signature')),
'remote_schema': shlex.quote(remote_schema),
'remote_dir': shlex.quote(remote_dir),
'local_dir': shlex.quote(local_dir),
'remove_backup_older_than': shlex.quote(remove_backup_older_than)
}
rdiff_wrapper = self.createFile(
return self.createFile(
name=rdiff_wrapper_path,
content=rdiff_wrapper_content,
mode=0700
content=template % template_dict,
mode=0o700
)
def add_slave(self, entry, known_hosts_file):
path_list = []
url = entry.get('url')
if not url:
raise ValueError('Missing URL parameter for PBS recipe')
parsed_url = urlparse.urlparse(url)
slave_type = entry['type']
if not slave_type in ['pull', 'push']:
raise ValueError('type parameter must be either pull or push.')
slave_id = entry['notification-id']
print 'Processing PBS slave %s with type %s' % (slave_id, slave_type)
promise_path = os.path.join(self.options['promises-directory'], slave_id)
promise_dict = dict(ssh_client=self.options['sshclient-binary'],
user=parsed_url.username,
host=parsed_url.hostname,
port=parsed_url.port)
promise = self.createPythonScript(promise_path,
__name__ + '.promise',
promise_dict)
path_list.append(promise)
known_hosts_file[parsed_url.hostname] = entry['server-key']
notifier_wrapper_path = os.path.join(self.options['wrappers-directory'], slave_id)
rdiff_wrapper_path = notifier_wrapper_path + '_raw'
# Create the rdiff-backup wrapper
# It is useful to separate it from the notifier so that we can run it manually.
# XXX use -y because the host might not yet be in the
# trusted hosts file until the next time slapgrid is run.
remote_schema = '{ssh} -y -K 300 -p %s {username}@{hostname}'.format(
ssh=self.options['sshclient-binary'],
username=parsed_url.username,
hostname=parsed_url.hostname
)
remote_dir = '{port}::{path}'.format(port=parsed_url.port, path=parsed_url.path)
local_dir = self.createDirectory(self.options['directory'], entry['name'])
if slave_type == 'push':
rdiff_wrapper = self.wrapper_push(remote_schema,
local_dir,
remote_dir,
rdiff_wrapper_path)
elif slave_type == 'pull':
# XXX: only 3 increments is not enough by default.
rdiff_wrapper = self.wrapper_pull(remote_schema,
local_dir,
remote_dir,
rdiff_wrapper_path,
entry.get('remove-backup-older-than', '3B'))
path_list.append(rdiff_wrapper)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment