Switch rdiff backup recipe to slave instance based recipe.

slapos/recipe/dropbear.py

@@ -28,6 +28,32 @@ import os
 import itertools
 from slapos.recipe.librecipe import GenericBaseRecipe
 
+class KnownHostsFile(dict):
+
+  def __init__(self, filename):
+    self._filename = filename
+
+  def _load(self):
+    if os.path.exists(self._filename):
+      with open(self._filename, 'r') as keyfile:
+        for line in keyfile:
+          host, key = [column.strip() for column in line.split(' ', 1)]
+          self[host] = key
+
+  def _dump(self):
+    with open(self._filename, 'w') as keyfile:
+      for key, value in self.items():
+        keyfile.write('%(host)s %(key)s\n' % {'host': key,
+                                              'key': value})
+
+  def __enter__(self):
+    self._load()
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    self._dump()
+
+
+
 class AuthorizedKeysFile(object):
 
   def __init__(self, filename):

slapos/recipe/rdiffbackup.py

@@ -24,11 +24,15 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from json import loads as unjson
+from hashlib import sha512
+from urlparse import urlparse
 import os
 import subprocess
 import sys
 
 from slapos.recipe.librecipe import GenericBaseRecipe
+from slapos.recipe.dropbear import KnownHostsFile
 from slapos import slap as slapmodule
 
 
@@ -41,9 +45,7 @@ def promise(args):
 
   # Rdiff Backup protocol quit command
   quitcommand = 'q' + chr(255) + chr(0) * 7
-  ssh_cmdline = [args['ssh_client'], '-T',
-                 '-i', args['key'],
-                 '-y', '%(user)s@%(host)s/%(port)s' % args]
+  ssh_cmdline = [args['ssh_client'], '%(user)s@%(host)s/%(port)s' % args]
 
   ssh = subprocess.Popen(ssh_cmdline, stdin=subprocess.PIPE,
                          stdout=open(os.devnull), stderr=open(os.devnull))
@@ -58,48 +60,98 @@ def promise(args):
 
   return ssh.returncode
 
+
+
 class Recipe(GenericBaseRecipe):
 
-  def install(self):
-    command = [self.options['rdiffbackup-binary']]
+  def add_slave(self, entry, known_hosts_file):
+    path_list = []
+
+    url = entry.get('url')
+    if url is None:
+      url = ''
+
+    # We assume that thanks to sha512 there's no collisions
+    url_hash = sha512(url).hexdigest()
+    name_hash = sha512(entry['name']).hexdigest()
+
+    promise_path = os.path.join(self.options['promises-directory'],
+                                url_hash)
+    parsed_url = urlparse(url)
+    promise_dict = self.promise_base_dict.copy()
+    promise_dict.update(user=parsed_url.username,
+                        host=parsed_url.hostname,
+                        port=parsed_url.port)
+    promise = self.createPythonScript(promise_path,
+                                      __name__ + '.promise',
+                                      promise_dict)
+    path_list.append(promise)
 
-    if self.optionIsTrue('client', True):
-      self.logger.info("Client mode")
 
-      # XXX-Antoine: Automaticaly accept unknown key with -y
-      # we should generate a known_host file.
-      remote_schema = '%(ssh)s -i %%s -T -y %(user)s@%(host)s/%(port)s' % \
+    host = parsed_url.hostname
+    known_hosts_file[host] = entry['server-key']
+
+    remote_schema = '%(ssh)s -p %%s %(user)s@%(host)s' % \
       {
         'ssh': self.options['sshclient-binary'],
-          'user': self.options['user'],
-          'host': self.options['host'],
-          'port': self.options['port'],
+        'user': parsed_url.username,
+        'host': parsed_url.hostname,
       }
 
+    command = [self.options['rdiffbackup-binary']]
     command.extend(['--remote-schema', remote_schema])
-      command.append('%(key)s::%(path)s' % {'key': self.options['key'],
-                                            'path': self.options['path']})
-      command.append(self.options['localpath'])
 
-      if 'promise' in self.options:
+    remote_directory = '%(port)s::%(path)s' % {'port': parsed_url.port,
+                                               'path': parsed_url.path}
+
+    local_directory = self.createDirectory(self.options['directory'],
+                                           name_hash)
+
+    if entry['type'] == 'push':
+      command.extend(['--restore-as-of', 'now'])
+      command.extend([local_directory, remote_directory])
+    else:
+      command.extend([remote_directory, local_directory])
+
+    wrapper = self.createPythonScript(
+      os.path.join(self.options['wrappers-directory'], url_hash),
+      'slapos.recipe.librecipe.execute.execute',
+      command
+    )
+    path_list.append(wrapper)
+
+    cron_entry = os.path.join(self.options['cron-entries'], url_hash)
+    with open(cron_entry, 'w') as cron_entry_file:
+      cron_entry_file.write('%s %s' % (entry['frequency'], wrapper))
+    path_list.append(cron_entry)
+
+    return path_list
+
+  def install(self):
+    path_list = []
+
+
+    if self.optionIsTrue('client', True):
+      self.logger.info("Client mode")
+
       slap_connection = self.buildout['slap-connection']
-        self.createPythonScript(self.options['promise'],
-          __name__ + '.promise',
-          dict(
+      self.promise_base_dict = dict(
         server_url=slap_connection['server-url'],
         computer_id=slap_connection['computer-id'],
         cert_file=slap_connection.get('cert-file'),
         key_file=slap_connection.get('key-file'),
         partition_id=slap_connection['partition-id'],
         ssh_client=self.options['sshclient-binary'],
-            user=self.options['user'],
-            host=self.options['host'],
-            port=self.options['port'],
-            key=self.options['key'],
-          ),
       )
 
+      slaves = unjson(self.buildout['slap-parameter']['slave_instance_list'])
+      known_hosts = KnownHostsFile(self.options['known-hosts'])
+      with known_hosts:
+        for slave in slaves:
+          path_list.extend(self.add_slave(slave, known_hosts))
+
     else:
+      command = [self.options['rdiffbackup-binary']]
       self.logger.info("Server mode")
       command.extend(['--restrict', self.options['path']])
       command.append('--server')
@@ -108,5 +160,6 @@ class Recipe(GenericBaseRecipe):
         self.options['wrapper'],
         'slapos.recipe.librecipe.execute.execute',
         command)
+      path_list.append(wrapper)
 
-    return [wrapper]
+    return path_list