Commit 8c76eae2 authored by Jérome Perrin's avatar Jérome Perrin

Fix zodb analyze with empty reports

Fix for this kind of errors:

```
(env)$ zodb analyze demo.fs ffffffffffffffff..
# ø
Processed 0 records in 0 transactions
Traceback (most recent call last):
  File "/srv/slapgrid/slappart8/srv/runner/project/zodbtools/env/bin/zodb", line 11, in <module>
    load_entry_point('zodbtools', 'console_scripts', 'zodb')()
  File "/srv/slapgrid/slappart8/srv/runner/project/zodbtools/zodbtools/zodb.py", line 130, in main
    return command_module.main(argv)
  File "/srv/slapgrid/slappart8/srv/runner/project/zodbtools/zodbtools/zodbanalyze.py", line 305, in main
    report(analyze(path, use_dbm, delta_fs, tidmin, tidmax), csv)
  File "/srv/slapgrid/slappart8/srv/runner/project/zodbtools/zodbtools/zodbanalyze.py", line 102, in report
    print "Average record size is %7.2f bytes" % (rep.DBYTES * 1.0 / rep.OIDS)
ZeroDivisionError: float division by zero
```

and also small fixes for python3 compatibility

/reviewed-on !9
parents 7c5bb0b5 d37746c6
# -*- coding: utf-8 -*-
# Copyright (C) 2019 Nexedi SA and Contributors.
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
from zodbtools.zodbanalyze import analyze, report
import os.path
def test_zodbanalyze(capsys):
for use_dbm in (False, True):
report(
analyze(
os.path.join(os.path.dirname(__file__), "testdata", "1.fs"),
use_dbm=use_dbm,
delta_fs=False,
tidmin=None,
tidmax=None,
),
csv=False,
)
captured = capsys.readouterr()
assert "Processed 68 records in 59 transactions" in captured.out
assert captured.err == ""
# csv output
report(
analyze(
os.path.join(os.path.dirname(__file__), "testdata", "1.fs"),
use_dbm=False,
delta_fs=False,
tidmin=None,
tidmax=None,
),
csv=True,
)
captured = capsys.readouterr()
assert (
"""Class Name,T.Count,T.Bytes,Pct,AvgSize,C.Count,C.Bytes,O.Count,O.Bytes
persistent.mapping.PersistentMapping,10,1578,45.633314%,157.800000,1,213,9,1365
__main__.Object,56,1880,54.366686%,33.571429,9,303,47,1577
"""
== captured.out
)
assert captured.err == ""
# empty range
report(
analyze(
os.path.join(os.path.dirname(__file__), "testdata", "1.fs"),
use_dbm=False,
delta_fs=False,
tidmin="ffffffffffffffff",
tidmax=None,
),
csv=False,
)
captured = capsys.readouterr()
assert "# ø\nNo transactions processed\n" == captured.out.encode('utf-8')
assert captured.err == ""
...@@ -4,10 +4,12 @@ ...@@ -4,10 +4,12 @@
# Based on a transaction analyzer by Matt Kromer. # Based on a transaction analyzer by Matt Kromer.
from __future__ import print_function
import sys import sys
import os import os
import getopt import getopt
import anydbm as dbm from six.moves import dbm_gnu as dbm
import tempfile import tempfile
import shutil import shutil
from ZODB.FileStorage import FileIterator, packed_version from ZODB.FileStorage import FileIterator, packed_version
...@@ -29,7 +31,7 @@ class DeltaFileStorage( ...@@ -29,7 +31,7 @@ class DeltaFileStorage(
pass pass
class DeltaFileIterator(FileIterator): class DeltaFileIterator(FileIterator):
def __init__(self, filename, start=None, stop=None, pos=0L): def __init__(self, filename, start=None, stop=None, pos=0):
assert isinstance(filename, str) assert isinstance(filename, str)
file = open(filename, 'rb') file = open(filename, 'rb')
self._file = file self._file = file
...@@ -95,15 +97,17 @@ def report(rep, csv=False): ...@@ -95,15 +97,17 @@ def report(rep, csv=False):
delta_fs = rep.delta_fs delta_fs = rep.delta_fs
if not csv: if not csv:
if rep.TIDS == 0: if rep.TIDS == 0:
print "# ø" print ("# ø")
else: print ("No transactions processed")
print "# %s..%s" % (ashex(rep.tidmin), ashex(rep.tidmax)) return
print "Processed %d records in %d transactions" % (rep.OIDS, rep.TIDS)
print "Average record size is %7.2f bytes" % (rep.DBYTES * 1.0 / rep.OIDS) print ("# %s..%s" % (ashex(rep.tidmin), ashex(rep.tidmax)))
print ("Processed %d records in %d transactions" % (rep.OIDS, rep.TIDS))
print ("Average record size is %7.2f bytes" % (rep.DBYTES * 1.0 / rep.OIDS))
print ("Average transaction size is %7.2f bytes" % print ("Average transaction size is %7.2f bytes" %
(rep.DBYTES * 1.0 / rep.TIDS)) (rep.DBYTES * 1.0 / rep.TIDS))
print "Types used:" print ("Types used:")
if delta_fs: if delta_fs:
if csv: if csv:
fmt = "%s,%s,%s,%s,%s" fmt = "%s,%s,%s,%s,%s"
...@@ -111,9 +115,9 @@ def report(rep, csv=False): ...@@ -111,9 +115,9 @@ def report(rep, csv=False):
else: else:
fmt = "%-46s %7s %9s %6s %7s" fmt = "%-46s %7s %9s %6s %7s"
fmtp = "%-46s %7d %9d %5.1f%% %7.2f" # per-class format fmtp = "%-46s %7d %9d %5.1f%% %7.2f" # per-class format
print fmt % ("Class Name", "T.Count", "T.Bytes", "Pct", "AvgSize") print (fmt % ("Class Name", "T.Count", "T.Bytes", "Pct", "AvgSize"))
if not csv: if not csv:
print fmt % ('-'*46, '-'*7, '-'*9, '-'*5, '-'*7) print (fmt % ('-'*46, '-'*7, '-'*9, '-'*5, '-'*7))
else: else:
if csv: if csv:
fmt = "%s,%s,%s,%s,%s,%s,%s,%s,%s" fmt = "%s,%s,%s,%s,%s,%s,%s,%s,%s"
...@@ -121,15 +125,13 @@ def report(rep, csv=False): ...@@ -121,15 +125,13 @@ def report(rep, csv=False):
else: else:
fmt = "%-46s %7s %9s %6s %7s %7s %9s %7s %9s" fmt = "%-46s %7s %9s %6s %7s %7s %9s %7s %9s"
fmtp = "%-46s %7d %9d %5.1f%% %7.2f %7d %9d %7d %9d" # per-class format fmtp = "%-46s %7d %9d %5.1f%% %7.2f %7d %9d %7d %9d" # per-class format
print fmt % ("Class Name", "T.Count", "T.Bytes", "Pct", "AvgSize", print (fmt % ("Class Name", "T.Count", "T.Bytes", "Pct", "AvgSize",
"C.Count", "C.Bytes", "O.Count", "O.Bytes") "C.Count", "C.Bytes", "O.Count", "O.Bytes"))
if not csv: if not csv:
print fmt % ('-'*46, '-'*7, '-'*9, '-'*5, '-'*7, '-'*7, '-'*9, '-'*7, '-'*9) print (fmt % ('-'*46, '-'*7, '-'*9, '-'*5, '-'*7, '-'*7, '-'*9, '-'*7, '-'*9))
fmts = "%46s %7d %8dk %5.1f%% %7.2f" # summary format fmts = "%46s %7d %8dk %5.1f%% %7.2f" # summary format
typemap = rep.TYPEMAP.keys()
typemap.sort(key=lambda a:rep.TYPESIZE[a])
cumpct = 0.0 cumpct = 0.0
for t in typemap: for t in sorted(rep.TYPEMAP.keys(), key=lambda a:rep.TYPESIZE[a]):
pct = rep.TYPESIZE[t] * 100.0 / rep.DBYTES pct = rep.TYPESIZE[t] * 100.0 / rep.DBYTES
cumpct += pct cumpct += pct
if csv: if csv:
...@@ -137,37 +139,37 @@ def report(rep, csv=False): ...@@ -137,37 +139,37 @@ def report(rep, csv=False):
else: else:
t_display = shorten(t, 46) t_display = shorten(t, 46)
if delta_fs: if delta_fs:
print fmtp % (t_display, rep.TYPEMAP[t], rep.TYPESIZE[t], print (fmtp % (t_display, rep.TYPEMAP[t], rep.TYPESIZE[t],
pct, rep.TYPESIZE[t] * 1.0 / rep.TYPEMAP[t]) pct, rep.TYPESIZE[t] * 1.0 / rep.TYPEMAP[t]))
else: else:
print fmtp % (t_display, rep.TYPEMAP[t], rep.TYPESIZE[t], print (fmtp % (t_display, rep.TYPEMAP[t], rep.TYPESIZE[t],
pct, rep.TYPESIZE[t] * 1.0 / rep.TYPEMAP[t], pct, rep.TYPESIZE[t] * 1.0 / rep.TYPEMAP[t],
rep.COIDSMAP[t], rep.CBYTESMAP[t], rep.COIDSMAP[t], rep.CBYTESMAP[t],
rep.FOIDSMAP.get(t, 0), rep.FBYTESMAP.get(t, 0)) rep.FOIDSMAP.get(t, 0), rep.FBYTESMAP.get(t, 0)))
if csv: if csv:
return return
if delta_fs: if delta_fs:
print fmt % ('='*46, '='*7, '='*9, '='*5, '='*7) print (fmt % ('='*46, '='*7, '='*9, '='*5, '='*7))
print "%46s %7d %9s %6s %6.2f" % ('Total Transactions', rep.TIDS, ' ', print ("%46s %7d %9s %6s %6.2f" % ('Total Transactions', rep.TIDS, ' ',
' ', rep.DBYTES * 1.0 / rep.TIDS) ' ', rep.DBYTES * 1.0 / rep.TIDS))
print fmts % ('Total Records', rep.OIDS, rep.DBYTES, cumpct, print (fmts % ('Total Records', rep.OIDS, rep.DBYTES, cumpct,
rep.DBYTES * 1.0 / rep.OIDS) rep.DBYTES * 1.0 / rep.OIDS))
else: else:
print fmt % ('='*46, '='*7, '='*9, '='*5, '='*7, '='*7, '='*9, '='*7, '='*9) print (fmt % ('='*46, '='*7, '='*9, '='*5, '='*7, '='*7, '='*9, '='*7, '='*9))
print "%46s %7d %9s %6s %6.2fk" % ('Total Transactions', rep.TIDS, ' ', print ("%46s %7d %9s %6s %6.2fk" % ('Total Transactions', rep.TIDS, ' ',
' ', rep.DBYTES * 1.0 / rep.TIDS / 1024.0) ' ', rep.DBYTES * 1.0 / rep.TIDS / 1024.0))
print fmts % ('Total Records', rep.OIDS, rep.DBYTES / 1024.0, cumpct, print (fmts % ('Total Records', rep.OIDS, rep.DBYTES / 1024.0, cumpct,
rep.DBYTES * 1.0 / rep.OIDS) rep.DBYTES * 1.0 / rep.OIDS))
print fmts % ('Current Objects', rep.COIDS, rep.CBYTES / 1024.0, print (fmts % ('Current Objects', rep.COIDS, rep.CBYTES / 1024.0,
rep.CBYTES * 100.0 / rep.DBYTES, rep.CBYTES * 100.0 / rep.DBYTES,
rep.CBYTES * 1.0 / rep.COIDS) rep.CBYTES * 1.0 / rep.COIDS))
if rep.FOIDS: if rep.FOIDS:
print fmts % ('Old Objects', rep.FOIDS, rep.FBYTES / 1024.0, print (fmts % ('Old Objects', rep.FOIDS, rep.FBYTES / 1024.0,
rep.FBYTES * 100.0 / rep.DBYTES, rep.FBYTES * 100.0 / rep.DBYTES,
rep.FBYTES * 1.0 / rep.FOIDS) rep.FBYTES * 1.0 / rep.FOIDS))
@func @func
def analyze(path, use_dbm, delta_fs, tidmin, tidmax): def analyze(path, use_dbm, delta_fs, tidmin, tidmax):
...@@ -238,8 +240,8 @@ def analyze_rec(report, record): ...@@ -238,8 +240,8 @@ def analyze_rec(report, record):
report.CBYTESMAP[type] = report.CBYTESMAP.get(type, 0) + size - fsize report.CBYTESMAP[type] = report.CBYTESMAP.get(type, 0) + size - fsize
report.TYPEMAP[type] = report.TYPEMAP.get(type, 0) + 1 report.TYPEMAP[type] = report.TYPEMAP.get(type, 0) + 1
report.TYPESIZE[type] = report.TYPESIZE.get(type, 0) + size report.TYPESIZE[type] = report.TYPESIZE.get(type, 0) + size
except Exception, err: except Exception as err:
print err print (err, file=sys.stderr)
__doc__ = """%(program)s: Analyzer for ZODB data or repozo deltafs __doc__ = """%(program)s: Analyzer for ZODB data or repozo deltafs
...@@ -261,9 +263,9 @@ summary = "analyze ZODB database or repozo deltafs usage" ...@@ -261,9 +263,9 @@ summary = "analyze ZODB database or repozo deltafs usage"
def usage(stream, msg=None): def usage(stream, msg=None):
if msg: if msg:
print >>stream, msg print (msg, file=stream)
print >>stream print (file=stream)
print >>stream, __doc__ % {"program": "zodb analyze"} print (__doc__ % {"program": "zodb analyze"}, file=stream)
def main(argv): def main(argv):
...@@ -271,7 +273,7 @@ def main(argv): ...@@ -271,7 +273,7 @@ def main(argv):
opts, args = getopt.getopt(argv[1:], opts, args = getopt.getopt(argv[1:],
'hcd', ['help', 'csv', 'dbm']) 'hcd', ['help', 'csv', 'dbm'])
path = args[0] path = args[0]
except (getopt.GetoptError, IndexError), msg: except (getopt.GetoptError, IndexError) as msg:
usage(sys.stderr, msg) usage(sys.stderr, msg)
sys.exit(2) sys.exit(2)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment