zopefds: Program to monitor number of file descriptors used by Zopes

Committing quick & dirty hack I did some time ago found in snippets around.

zopefds

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2017  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+"""monitor #fd used by zopes over time"""
+# FIXME quick hack & dirty
+
+from __future__ import division
+
+from subprocess import check_output, PIPE
+import os, os.path
+import sys
+import time
+
+# get list of opened files by pid
+def opened_files(pid):
+    fddir = "/proc/%s/fd/" % pid
+    fdv = os.listdir(fddir)
+    fdv = [os.readlink(fddir + _) for _ in fdv]         # readlink *
+    return fdv
+
+# current time formatted as in zope logs
+def ztime():
+    t = time.gmtime(time.time())
+    return time.strftime("%Y-%m-%d %H:%M:%S", t)
+
+
+def dump_zopesfd():
+    T = ztime()
+    zopeps = check_output("xslapos node |grep 8:zope |grep RUNNING", shell=True)
+    for l in zopeps.splitlines():
+        # slappart6:zope-0-on-watch RUNNING pid 19610, uptime 1 day, 20:54:16
+        name, _, _, pid, _ = l.split(None, 4)
+        pid = pid.rstrip(',')
+        #print name, pid
+        fdv = opened_files(pid)
+
+        # leave only class part for interesting things
+        fdv = [os.path.basename(_) for _ in fdv]    # basename *
+        fdv2 = []
+        for fd in fdv:
+            if fd.startswith("pipe:["):
+                fd = "pipe"
+            elif fd.startswith("socket:["):
+                fd = "socket"
+            elif fd.startswith("ramh."):
+                fd = "ramh"
+
+            fdv2.append(fd)
+
+        fdv = fdv2
+
+        fdtotal = len(fdv)
+
+        # sort | uniq -c | sort -nr
+        fdcount = {}   # fd -> nentries
+        for fd in fdv:
+            try:
+                fdcount[fd] += 1
+            except KeyError:
+                fdcount[fd] = 1
+
+        fdv = fdcount.keys()
+        fdv.sort(key=lambda fd: fdcount[fd], reverse=True)
+
+        # dump
+        showv = []
+        fdprinted = 0
+        tailprinted = False
+        for fd in fdv:
+            if fdprinted / fdtotal < 0.97:
+                showv.append("%s:%i" % (fd, fdcount[fd]))
+                fdprinted += fdcount[fd]
+
+            elif not tailprinted:
+                showv.append("...")
+                tailprinted = True
+
+        print T, name, '\t'.join(showv)
+
+
+def main():
+    while 1:
+        dump_zopesfd()
+        sys.stdout.flush()
+        time.sleep(30)
+
+
+if __name__ == '__main__':
+    main()

[Kirill Smelkov]

/cc @klaus, @Tyagov, @jm, @rafael