resilient: new tool directory for resiliency.

This first commit adds a script which generate a RSS feed from the statistics from rdiff-backup

setup.py

@@ -92,6 +92,7 @@ setup(name=name,
           'pubsubnotifier = slapos.pubsub.notifier:main',
           'pubsubserver = slapos.pubsub:main',
           'qemu-qmp-client = slapos.qemuqmpclient:main',
+          'rdiffbackup.genstatrss = slapos.resilient.rdiffBackupStat2RSS:main',
           'slapos-kill = slapos.systool:kill',
           'slaprunnertest = slapos.runner.runnertest:main',
           'slaprunnerteststandalone = slapos.runner.runnertest:runStandaloneUnitTest',

slapos/resilient/rdiffBackupStat2RSS.py

+import argparse
+import datetime
+import os
+import re
+import time
+import PyRSS2Gen as RSS2
+
+from collections import OrderedDict
+
+def parseArguments():
+  """
+  Parse arguments for rdiff-backup statistics Rss Generator.
+  """
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--rdiff_backup_data_folder',
+                      help='Path where to find rdiff-backup statistical files')
+  parser.add_argument('--output',
+                      help='Path where to save the feed')
+  parser.add_argument('--feed_url',
+                      help='Url of this feed file.')
+
+  return parser.parse_args()
+
+def makeDictFromStatFile(text_content):
+  m = re.search("([a-zA-Z]*) ([0-9 :.]*) \(([a-zA-Z0-9 :.]*)\)", text_content)
+  if m:
+    return {'metric': m.group(1),
+            'value': m.group(2),
+            'human_readable_value': m.group(3)}
+  m = re.search("([a-zA-Z]*) ([0-9]*)", text_content)
+  if m:
+    return {'metric': m.group(1),
+            'value': m.group(2),
+            'human_readable_value': m.group(2)}
+
+def getRSSItemFromDict(item, option):
+  description = "Metric;Value;Human Readable Value\n"
+  for entry in item:
+    description += "%s;%s;%s\n" % (entry['metric'], entry['value'], entry['human_readable_value'])
+    if entry['metric'] == "EndTime":
+      pubDate = datetime.datetime.fromtimestamp(float(entry['value']))
+  return RSS2.RSSItem(
+    title="Rdiff-Backup Transfer Statistics",
+    link=option.feed_url,
+    pubDate=pubDate,
+    description=description)
+
+def genRSS(option):
+  """
+  Read statistics file from rdiff-backup and generate a RSS feed entry from it
+  """
+  stat_file_list = sorted([file for file in os.listdir(option.rdiff_backup_data_folder)
+                           if file.startswith('session_statistics')])
+
+  item_dict = OrderedDict()
+  for stat_file in stat_file_list:
+    with open(os.path.join(option.rdiff_backup_data_folder, stat_file), 'r') as file:
+      item_dict[stat_file] = [makeDictFromStatFile(line.strip()) for line in file.readlines()]
+  title="Rdiff-Backup Statistics"
+  rss_feed = RSS2.RSS2(
+    title=title,
+    link=option.feed_url,
+    description=title,
+    items = [getRSSItemFromDict(item_dict[item], option) for item in item_dict])
+
+  return rss_feed.to_xml()
+
+def main():
+  option = parseArguments()
+  feed = genRSS(option)
+
+  with open(option.output, 'w') as rss_file:
+    rss_file.write(feed)
+
+  exit(0)

slapos/test/test_rdiffBackupStat2RSS.py

+import csv
+import feedparser
+import os
+import shutil
+import tempfile
+import unittest
+
+from slapos.resilient.rdiffBackupStat2RSS import genRSS
+
+class Option(dict):
+    def __init__(self, **kw): 
+        self.__dict__.update(kw)
+    def __setitem__(i, y):
+        self.__dict__[i] = y
+
+class TestRdiffBackupStat2RSS(unittest.TestCase):
+  def setUp(self):
+    self.data_directory = tempfile.mkdtemp(dir='.')
+    self.feed_path = os.path.join(self.data_directory)
+
+  def tearDown(self):
+    shutil.rmtree(self.data_directory)
+
+  def getOptionObject(self, **kw):
+    """
+    Returns an object containing options as properties, to simulate a call
+    to the tested script
+    """
+    option = {
+      'rdiff_backup_data_folder': self.data_directory,
+      'output': self.feed_path,
+      'feed_url': 'http://exemple.com',
+    }
+    option.update(kw)
+    return Option(**option)
+
+  def createSample(self):
+    """
+    Writes 2 statistics file in rdiff-backup format
+    """
+    with open(os.path.join(self.data_directory, 'session_statistics_1'), 'w') as stat_file:
+      stat_file.write("""\
+        StartTime 1473339659.00 (Thu Sep  8 15:00:59 2016)
+        EndTime 1473339667.81 (Thu Sep  8 15:01:07 2016)
+        ElapsedTime 8.81 (8.81 seconds)
+        SourceFiles 2381
+        SourceFileSize 142096473 (136 MB)
+        MirrorFiles 1
+        MirrorFileSize 0 (0 bytes)
+        NewFiles 2380
+        NewFileSize 142096473 (136 MB)
+        DeletedFiles 0
+        DeletedFileSize 0 (0 bytes)
+        ChangedFiles 1
+        ChangedSourceSize 0 (0 bytes)
+        ChangedMirrorSize 0 (0 bytes)
+        IncrementFiles 0
+        IncrementFileSize 0 (0 bytes)
+        TotalDestinationSizeChange 142096473 (136 MB)
+        Errors 0""")
+    with open(os.path.join(self.data_directory, 'session_statistics_2'), 'w') as stat_file:
+      stat_file.write("""\
+        StartTime 1473340154.00 (Thu Sep  8 15:09:14 2016)
+        EndTime 1473340154.95 (Thu Sep  8 15:09:14 2016)
+        ElapsedTime 0.95 (0.95 seconds)
+        SourceFiles 2381
+        SourceFileSize 142096473 (136 MB)
+        MirrorFiles 2381
+        MirrorFileSize 142096473 (136 MB)
+        NewFiles 0
+        NewFileSize 0 (0 bytes)
+        DeletedFiles 0
+        DeletedFileSize 0 (0 bytes)
+        ChangedFiles 15
+        ChangedSourceSize 230112 (225 KB)
+        ChangedMirrorSize 230112 (225 KB)
+        IncrementFiles 15
+        IncrementFileSize 2122 (2.07 KB)
+        TotalDestinationSizeChange 2122 (2.07 KB)
+        Errors 0""")
+
+  def test_generatedRSSIsCorrect(self):
+    self.createSample()
+    option = self.getOptionObject()
+    feed_content = genRSS(option)
+    feed = feedparser.parse(feed_content)
+
+    self.assertFalse(feed.bozo)
+    self.assertTrue(len(feed.entries), 2)
+    self.assertLess(feed.entries[0].published_parsed, feed.entries[1].published_parsed)
+
+
+if __name__ == '__main__':
+  unittest.main()