X watch-rtcache: Updates

- more info about invalid route (also shows 'after' entry)
- notify when a cache entry gets recovered
- cosmetics

draft/watch-rtcache

@@ -19,6 +19,9 @@ def ip6route(*args):
   r = {}
   for route in subprocess.check_output(('ip', '-6', '-o', 'r') + args
                                        ).splitlines():
+    # 2001:67c:1254:e:e::/80 via fe80::64ec:caff:fe4f:4a5b dev re6stnet7  proto babel  src 2001:67c:1254:c0::1  metric 1024  pref medium
+    # unreachable 2001:67c:1254:c0::/64 dev lo  proto kernel  metric 256  error -101 pref medium
+    # unreachable 2001:67c:1254::/48 dev lo  metric 1024  error -113 pref medium
     route = route.split()
     unreachable = route[0] == 'unreachable'
     dst = route[unreachable]
@@ -27,9 +30,11 @@ def ip6route(*args):
         dst, n = dst.split('/')
       except ValueError:
         n = 128
+      # None (unreachable)
+      # fe80::64ec:caff:fe4f:4a5b, re6stnet7
       r[binFromIp(dst)[:int(n)]] = None \
         if unreachable else (route[2], route[4])
-  return sorted(r.iteritems())
+  return sorted(r.iteritems())  # (dst, (via, dev))
 
 def lookup(r, dst):
   i = bisect.bisect(r, (dst,))
@@ -49,13 +54,14 @@ class Invalid(dict):
     after = ip6route()
     for dst, via in cache:
       expected = lookup(before, dst)
-      if expected != via != lookup(after, dst):
-        self[ipFromBin(dst)] = expected, via
+      after_   = lookup(after, dst)
+      if expected != via != after_:
+        self[ipFromBin(dst)] = expected, via, after_
 
   def __str__(self):
     return "\n".join(
-      "%s: expected %s, got %s" % (ip, via_str(expected), via_str(via))
-      for ip, (expected, via) in sorted(self.iteritems()))
+      "%s: expected %s, got %s  (after: %s)" % (ip, via_str(expected), via_str(via), via_str(after))
+      for ip, (expected, via, after) in sorted(self.iteritems()))
 
 if __name__ == "__main__":
   if len(sys.argv) > 1:
@@ -64,6 +70,25 @@ if __name__ == "__main__":
     db = sqlite3.connect(db)
     db.execute("CREATE TABLE IF NOT EXISTS invalid (ip, start, end)")
     host = socket.gethostname()
+
+    def sendmail(subject, text):
+      text = str(text)
+      msg = MIMEText(text)
+      msg['Subject'] = subject
+      msg['From'] = host
+      msg['Date'] = formatdate(now)
+      print msg['Date'], subject
+      print text
+
+      try:
+        s = smtplib.SMTP(smtp)
+        s.sendmail(None, to, msg.as_string())
+        s.close()
+      except socket.error:
+        print 'W: sendmail failed (ignore)'
+        pass
+
+
     t = 0
     while 1:
       now = time()
@@ -74,27 +99,22 @@ if __name__ == "__main__":
       else:
         t = now
       invalid = Invalid()
+      recovered = set() # of ip now having valid routes in cachec
       with db:
         for ip, in db.execute("SELECT ip FROM invalid WHERE end IS NULL"):
           try:
             del invalid[ip]
           except KeyError:
             db.execute("UPDATE invalid SET end=? WHERE ip=?", (int(now), ip))
+            recovered.add(ip)
+        if recovered:
+            sendmail("recovered: %s invalid routes in cache" % len(recovered),
+                     "\n".join(sorted(recovered)))
         if not invalid:
           continue
         db.executemany("INSERT INTO invalid VALUES (?, ?, NULL)", (
           (ip, int(now)) for ip in invalid))
-      msg = MIMEText(str(invalid))
-      msg['Subject'] =  "%s invalid routes in cache" % len(invalid)
-      msg['From'] = host
-      msg['Date'] = formatdate(now)
-      print msg['Date'], invalid
-      try:
-        s = smtplib.SMTP(smtp)
-        s.sendmail(None, to, msg.as_string())
-        s.close()
-      except socket.error:
-        pass
+      sendmail("%s new invalid routes in cache" % len(invalid), invalid)
 
   else:
     invalid = Invalid()