resilient: adds a promise to test integrity and the status of the last run of...

resilient: adds a promise to test integrity and the status of the last run of the feeds generated by notifier scripts

stack/resilient/buildout.cfg

@@ -14,6 +14,7 @@ parts =
   pbsready
   pbsready-import
   pbsready-export
+  notifier-feed-promise-template
   template-replicated
   template-parts
   instance-frozen
@@ -41,7 +42,7 @@ eggs =
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready.cfg.in
 output = ${buildout:directory}/pbsready.cfg
-md5sum = dfaef2081cc08b0f046b6989f9a9462f
+md5sum = 0df8fe9b69f7943c3d5a2d30d4640557
 mode = 0644
 
 [pbsready-import]
@@ -50,7 +51,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready-import.cfg.in
 output = ${buildout:directory}/pbsready-import.cfg
-md5sum = 9c3567ec52dcdb00ac1e8b2acd1941ff
+md5sum = 5d5e4ad35c1a97ea5f7a15a4f5f766a8
 mode = 0644
 
 [pbsready-export]
@@ -59,14 +60,14 @@ mode = 0644
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/pbsready-export.cfg.in
 filename = pbsready-export.cfg.in
-md5sum = 6f912998b88de02ef52660e4a9c336f2
+md5sum = 1b38292c42702f91f620cb99d1a88952
 mode = 0644
 
 [template-pull-backup]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-pull-backup.cfg.in
 output = ${buildout:directory}/instance-pull-backup.cfg
-md5sum = e5642321d44c9a94383826b220df28ae
+md5sum = 3063ab8d4b1e4a4133ec3893349cefaa
 mode = 0644
 
 [template-replicated]
@@ -115,6 +116,12 @@ output = ${buildout:directory}/template-wrapper.cfg
 mode = 0644
 md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
 
+[notifier-feed-promise-template]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/templates/notifier-feed-promise.py.in
+md5sum = d75346911dbc4cfcdb39a21e56cd5016
+mode = 0644
+
 ##################
 # Monitor element
 #

stack/resilient/instance-pull-backup.cfg.in

@@ -15,6 +15,7 @@ parts =
   backup-signature-link
   cron-pbs-status-feed
   pull-push-stalled-promise
+  notifier-feed-status-promise
 
 ## Monitor for pbs
   monitor-base
@@ -320,6 +321,16 @@ recipe = slapos.cookbook:wrapper
 command-line = ${buildout:bin-directory}/check-feed-as-promise --feed-path $${pbs-resilient-status-feed:feed-path} --title --ok-pattern 'OK' --time-buffer 90000
 wrapper-path = $${basedirectory:promises}/stalled-pull-push
 
+[notifier-feed-status-promise]
+recipe = slapos.recipe.template:jinja2
+template = ${notifier-feed-promise-template:target}
+rendered = $${basedirectory:promises}/notifier-feed-check-malformed-or-failure.py
+mode = 700
+context =
+  key notifier_feed_directory directory:notifier-feeds
+  raw base_url http://[$${notifier:host}]:$${notifier:port}/get/
+  raw python_executable ${buildout:executable}
+
 #----------------
 #--
 #-- Publish instance parameters.

stack/resilient/pbsready-export.cfg.in

@@ -12,6 +12,7 @@ parts =
   cron
   cron-entry-logrotate
   cron-entry-notifier-status-feed
+  notifier-feed-status-promise
   notifier-stalled-promise
   resilient-sshkeys-authority
   sshd-raw-server

stack/resilient/pbsready-import.cfg.in

@@ -13,6 +13,7 @@ parts =
   cron
   cron-entry-logrotate
   cron-entry-notifier-status-feed
+  notifier-feed-status-promise
   notifier-stalled-promise
   resilient-sshkeys-authority
   sshd-raw-server

stack/resilient/pbsready.cfg.in

@@ -9,6 +9,7 @@ parts =
   cron
   cron-entry-logrotate
   cron-entry-notifier-status-feed
+  notifier-feed-status-promise
   notifier-stalled-promise
   resilient-sshkeys-authority
   sshd-graceful
@@ -306,6 +307,20 @@ input = inline:#!${bash:location}/bin/bash
 output = $${basedirectory:promises}/public-key-existence
 mode = 700
 
+#----------------
+#--
+#-- Promises
+
+[notifier-feed-status-promise]
+recipe = slapos.recipe.template:jinja2
+template = ${notifier-feed-promise-template:target}
+rendered = $${basedirectory:promises}/notifier-feed-check-malformed-or-failure.py
+mode = 700
+context =
+  key notifier_feed_directory directory:notifier-feeds
+  raw base_url http://[$${notifier:host}]:$${notifier:port}/get/
+  raw python_executable ${buildout:executable}
+
 #----------------
 #--
 #-- Connection informations to re-use.

stack/resilient/templates/notifier-feed-promise.py.in

+#!{{ python_executable }}
+import csv
+import os
+import sys
+import urllib2
+
+csv.field_size_limit(sys.maxsize)
+
+notifier_feed_directory = '{{ notifier_feed_directory }}'
+base_url = "{{ base_url }}"
+
+feed_file_list = os.listdir(notifier_feed_directory)
+
+for feed_file_name in feed_file_list:
+  url = base_url + feed_file_name
+  # Try feed consistency
+  try:
+    feed = urllib2.urlopen(url)
+    body = feed.read()
+  except urllib2.HTTPError as e:
+    sys.exit("%s is unavailable: %s" % (feed_file_name, e))
+  with open(os.path.join(notifier_feed_directory, feed_file_name)) as feed_file:
+    reader = csv.reader(feed_file)
+    # Get last row because we only care about last run
+    for row in reader:
+      pass
+    try:
+      timestamp, title, content, guid = row
+      if content.startswith('OK'):
+        continue
+      else:
+        sys.exit("Last run of %s failed" % feed_file_name)
+    except ValueError:
+      sys.exit("Notifier feed %s is malformed" % notifier_feed)
+    except NameError:
+      # row can be not defined if feed is empty
+      pass