sitemon SR (ping+dns+http)

software/sitemon/instance.cfg

+[buildout]
+
+parts =
+  directories
+  publish-connection-informations
+  dump-instance-parameters
+  sitemon-python
+  sitemon-bin
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+
+[directories]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin/
+etc = $${buildout:directory}/etc/
+var = $${buildout:directory}/var/
+log = $${:var}/log/
+services = $${:etc}/service/
+promises = $${:etc}/promise/
+
+[sitemon-python]
+recipe = zc.recipe.egg
+eggs =
+    PyRSS2Gen
+    requests
+interpreter = py-sitemon
+
+
+[sitemon-bin]
+recipe = cns.recipe.symlink
+symlink = ${sitemon-py:location}/${sitemon-py:filename} = $${directories:bin}/sitemon
+
+
+[sitemon-py]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/$${:filename}
+#md5sum = 
+download-only = True
+filename = sitemon.py
+mode = 0644
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+
+
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+#test = test
+
+
+[dump-instance-parameters]
+recipe = slapos.cookbook:slapconfiguration.jsondump
+json-output = $${buildout:directory}/parameters.json
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}

software/sitemon/sitemon.py

+#!/usr/bin/env python
+
+import collections
+import csv
+import json
+import logging
+import re
+import socket
+import subprocess
+import time
+from collections import namedtuple
+
+import PyRSS2Gen
+import requests
+
+log = logging.getLogger()
+ch = logging.StreamHandler()
+ch.setLevel(logging.DEBUG)
+log.addHandler(ch)
+
+
+Row = namedtuple('Row', ['type', 'address', 'expected', 'timeout', 'status', 'elapsed_msec', 'error'])
+
+
+# avoid writing a new RSS if same as precedent in the last 10 minutes
+
+# TODO csv rotation
+# TODO dns timeout
+# TODO check correctness of parameters (i.e. timeout must be a number, etc) and report as error item, to avoid exceptions
+
+
+
+
+def empty_row(testparameters):
+    return {
+            'type': testparameters['type'],
+            'address': testparameters.get('address', ''),
+            'expected': testparameters.get('expected', ''),
+            'timeout': testparameters.get('timeout', ''),
+            'status': '',
+            'elapsed_msec': '',
+            'error': '',
+            }
+
+
+def run_web(testparameters):
+    url = testparameters['address']
+    expected = testparameters['expected']
+    timeout = float(testparameters.get('timeout', 10))
+
+    try:
+        r = requests.get(url, timeout=timeout)
+    except requests.exceptions.ConnectTimeout:
+        return dict(empty_row(testparameters),
+                    error = 'Timeout while connecting to url')
+    except requests.exceptions.ReadTimeout:
+        return dict(empty_row(testparameters),
+                    error = 'Timeout while reading data')
+
+    r.raise_for_status()
+    text = r.text
+
+    if expected and expected not in text:
+        return dict(empty_row(testparameters),
+                    status = r.status_code,
+                    error = 'expected string not in page')
+
+    return dict(empty_row(testparameters),
+                status = r.status_code)
+
+
+
+
+# rtt min/avg/max/mdev = 1.102/1.493/2.203/0.438 ms
+ping_re = re.compile(
+    ".*"
+    "(?P<min>[\d\.]+)/"
+    "(?P<max>[\d\.]+)/"
+    "(?P<avg>[\d\.]+)/"
+    "(?P<mdev>[\d\.]+) ms"
+    )
+
+
+def run_ping(testparameters):
+    host = testparameters['address']
+    timeout = float(testparameters.get('timeout', 10))
+    proc = subprocess.Popen(['ping', '-c', '4', '-w', str(timeout), host],
+                            stdout=subprocess.PIPE)
+    out, err = proc.communicate()
+    summary_line = (out.splitlines() or [''])[-1]
+    m = ping_re.match(summary_line)
+    if m:
+        return dict(empty_row(testparameters),
+                    status = 'min %(min)s max %(max)s avg %(avg)s' % m.groupdict())
+    else:
+        return dict(empty_row(testparameters),
+                    status = 'FAILED',
+                    error = 'Cannot ping host')
+
+
+
+
+def run_dns(testparameters):
+    name = testparameters['address']
+    try:
+        socket.gethostbyname(name)
+        return dict(empty_row(testparameters),
+                    status = 'OK')
+    except socket.gaierror:
+        return dict(empty_row(testparameters),
+                    status = 'FAILED',
+                    error='Cannot resolve hostname')
+ 
+
+
+
+def get_elapsed_msec(t):
+    return (time.time() * 1e6 - t) / 1000
+
+
+
+def run_single(testparameters):
+    typ = testparameters['type']
+    t = time.time() * 1e6
+    if typ == 'http':
+        return run_web(dict(testparameters, elapsed_msec=get_elapsed_msec(t)))
+    elif typ == 'ping':
+        return run_ping(dict(testparameters, elapsed_msec=get_elapsed_msec(t)))
+    elif typ == 'dns':
+        return run_dns(dict(testparameters, elapsed_msec=get_elapsed_msec(t)))
+    else:
+        return dict(empty_row(testparameters),
+                    error = 'Unknown test type')
+
+
+
+def write_csv_row(csv_path, seq):
+    # open the file for each line because it could be rotated anytime.
+    with open(csv_path, 'a+') as fout:
+        writer = csv.writer(fout)
+        fout.seek(0, 2)
+        if fout.tell() == 0:
+            writer.writerow(Row._fields)
+        writer.writerow(seq)
+
+
+def grouped_tests(parameters):
+    ret = collections.defaultdict(dict)
+    for key in parameters:
+        testname, parname = key.split('-')
+        ret[testname][parname] = parameters[key]
+    return ret
+
+
+
+def run_all(parameters, csv_path, feed_path):
+    errors = []
+    for testname, testparameters in grouped_tests(parameters).iteritems():
+        result = Row(**run_single(testparameters))
+        if result:
+            write_csv_row(csv_path, result)
+            if result.error:
+                errors.append(result)
+
+    if errors:
+        items = []
+        for err_result in errors:
+            content = '\n'.join('%s: %s' % (col, err_result[idx]) for idx, col in enumerate(Row._fields))
+            items.append(PyRSS2Gen.RSSItem(
+                title=err_result.address,
+                link='XXX',
+                description=content
+                ))
+
+        rss = PyRSS2Gen.RSS2(
+                title='Site Monitor',
+                link='XXX',
+                description='XXX',
+                items=items)
+
+        rss.write_xml(open(feed_path, 'w'))
+
+
+
+def main(parameters_path, csv_path, feed_path):
+    with open(parameters_path, 'r') as fin:
+        parameters = json.load(fin)
+
+    run_all(parameters, csv_path=csv_path, feed_path=feed_path)
+
+
+
+if __name__ == '__main__':
+    main(parameters_path='parameters.json',
+         csv_path='log.csv',
+         feed_path='feed.xml')
+
+

software/sitemon/software.cfg

+[buildout]
+
+extends =
+  ../../stack/slapos.cfg
+
+parts = 
+  slapos-cookbook
+  eggs
+  instance
+  sitemon-py
+  
+[eggs]
+recipe = z3c.recipe.scripts
+eggs = 
+  slapos.cookbook
+  requests
+  PyRSS2Gen
+  cns.recipe.symlink
+
+[instance]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg
+output = ${buildout:directory}/instance.cfg
+#md5sum = 
+mode = 0644
+
+
+[sitemon-py]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/sitemon.py
+#md5sum = 
+download-only = True
+filename = sitemon.py
+mode = 0755
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+
+
+  
+[versions]
+slapos.cookbook = 0.89
+slapos.recipe.download = 1.0.dev-r4053
+slapos.recipe.template = 2.5
+z3c.recipe.scripts = 1.0.1
+PyRSS2Gen = 1.1
+cns.recipe.symlink = 0.2.3
+
+
+# Required by:
+# slapos.cookbook==0.89
+jsonschema = 2.4.0
+