Commit a59746d2 authored by Tim Peters's avatar Tim Peters

stats.py: cut the # of file reads by 1/3, and the # of

struct.unpack() calls by 1/2.

Various bugfixes elsewhere.
parent 94a7da6f
...@@ -368,7 +368,7 @@ class ClientCache(object): ...@@ -368,7 +368,7 @@ class ClientCache(object):
assert o.end_tid is None # i.e., o was current assert o.end_tid is None # i.e., o was current
if o is None: if o is None:
# TODO: Since we asserted o is not None above, this block # TODO: Since we asserted o is not None above, this block
# should be removing; waiting on time to prove it can't happen. # should be removed; waiting on time to prove it can't happen.
return return
o.end_tid = tid o.end_tid = tid
self.fc.update(o) # record the new end_tid on disk self.fc.update(o) # record the new end_tid on disk
......
#! /usr/bin/env python #! /usr/bin/env python
############################################################################## ##############################################################################
# #
# Copyright (c) 2001, 2002 Zope Corporation and Contributors. # Copyright (c) 2001-2005 Zope Corporation and Contributors.
# All Rights Reserved. # All Rights Reserved.
# #
# This software is subject to the provisions of the Zope Public License, # This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
...@@ -55,32 +55,34 @@ def main(): ...@@ -55,32 +55,34 @@ def main():
for o, a in opts: for o, a in opts:
if o == '-b': if o == '-b':
simclass = BuddyCacheSimulation simclass = BuddyCacheSimulation
if o == '-f': elif o == '-f':
simclass = SimpleCacheSimulation simclass = SimpleCacheSimulation
if o == '-l': elif o == '-l':
simclass = LRUCacheSimulation simclass = LRUCacheSimulation
if o == '-y': elif o == '-y':
simclass = AltZEOCacheSimulation simclass = AltZEOCacheSimulation
if o == '-z': elif o == '-z':
simclass = ZEOCacheSimulation simclass = ZEOCacheSimulation
if o == '-s': elif o == '-s':
cachelimit = int(float(a)*MB) cachelimit = int(float(a)*MB)
if o == '-2': elif o == '-2':
simclass = TwoQSimluation simclass = TwoQSimluation
if o == '-c': elif o == '-c':
simclass = CircularCacheSimulation simclass = CircularCacheSimulation
if o == '-o': elif o == '-o':
omicron = float(a) omicron = float(a)
if o == '-t': elif o == '-t':
theta = float(a) theta = float(a)
if o == '-O': elif o == '-O':
simclass = OracleSimulation simclass = OracleSimulation
if o == '-a': elif o == '-a':
simclass = ARCCacheSimulation simclass = ARCCacheSimulation
if o == '-T': elif o == '-T':
simclass = ThorSimulation simclass = ThorSimulation
if o == '-U': elif o == '-U':
simclass = UnboundedSimulation simclass = UnboundedSimulation
else:
assert False, (o, a)
if len(args) != 1: if len(args) != 1:
usage("exactly one file argument required") usage("exactly one file argument required")
...@@ -97,12 +99,12 @@ def main(): ...@@ -97,12 +99,12 @@ def main():
try: try:
import gzip import gzip
except ImportError: except ImportError:
print >>sys.stderr, "can't read gzipped files (no module gzip)" print >> sys.stderr, "can't read gzipped files (no module gzip)"
return 1 return 1
try: try:
f = gzip.open(filename, "rb") f = gzip.open(filename, "rb")
except IOError, msg: except IOError, msg:
print >>sys.stderr, "can't open %s: %s" % (filename, msg) print >> sys.stderr, "can't open %s: %s" % (filename, msg)
return 1 return 1
elif filename == "-": elif filename == "-":
# Read from stdin # Read from stdin
...@@ -112,7 +114,7 @@ def main(): ...@@ -112,7 +114,7 @@ def main():
try: try:
f = open(filename, "rb") f = open(filename, "rb")
except IOError, msg: except IOError, msg:
print >>sys.stderr, "can't open %s: %s" % (filename, msg) print >> sys.stderr, "can't open %s: %s" % (filename, msg)
return 1 return 1
# Create simulation object # Create simulation object
...@@ -127,7 +129,6 @@ def main(): ...@@ -127,7 +129,6 @@ def main():
sim.printheader() sim.printheader()
# Read trace file, simulating cache behavior # Read trace file, simulating cache behavior
offset = 0
records = 0 records = 0
f_read = f.read f_read = f.read
struct_unpack = struct.unpack struct_unpack = struct.unpack
...@@ -136,16 +137,13 @@ def main(): ...@@ -136,16 +137,13 @@ def main():
r = f_read(8) r = f_read(8)
if len(r) < 8: if len(r) < 8:
break break
offset += 8
ts, code = struct_unpack(">ii", r) ts, code = struct_unpack(">ii", r)
if ts == 0: if ts == 0:
# Must be a misaligned record caused by a crash # Must be a misaligned record caused by a crash
##print "Skipping 8 bytes at offset", offset-8
continue continue
r = f_read(16) r = f_read(16)
if len(r) < 16: if len(r) < 16:
break break
offset += 16
records += 1 records += 1
oid, serial = struct_unpack(">8s8s", r) oid, serial = struct_unpack(">8s8s", r)
# Decode the code # Decode the code
...@@ -288,12 +286,10 @@ class Simulation: ...@@ -288,12 +286,10 @@ class Simulation:
print (self.format + " OVERALL") % args print (self.format + " OVERALL") % args
class ZEOCacheSimulation(Simulation): class ZEOCacheSimulation(Simulation):
"""Simulate the ZEO 1.0 and 2.0cache behavior.
"""Simulate the current (ZEO 1.0 and 2.0) ZEO cache behavior.
This assumes the cache is not persistent (we don't know how to This assumes the cache is not persistent (we don't know how to
simulate cache validation.) simulate cache validation.)
""" """
extraname = "flips" extraname = "flips"
...@@ -348,7 +344,6 @@ class ZEOCacheSimulation(Simulation): ...@@ -348,7 +344,6 @@ class ZEOCacheSimulation(Simulation):
del self.fileoids[1 - self.current][oid] del self.fileoids[1 - self.current][oid]
class AltZEOCacheSimulation(ZEOCacheSimulation): class AltZEOCacheSimulation(ZEOCacheSimulation):
"""A variation of the ZEO cache that copies to the current file. """A variation of the ZEO cache that copies to the current file.
When a hit is found in the non-current cache file, it is copied to When a hit is found in the non-current cache file, it is copied to
...@@ -436,16 +431,13 @@ class LRUCacheSimulation(Simulation): ...@@ -436,16 +431,13 @@ class LRUCacheSimulation(Simulation):
self.size -= node.size self.size -= node.size
assert self.size >= 0 assert self.size >= 0
class Node: class Node(object):
"""Node in a doubly-linked list, storing oid and size as payload. """Node in a doubly-linked list, storing oid and size as payload.
A node can be linked or unlinked; in the latter case, next and A node can be linked or unlinked; in the latter case, next and
prev are None. Initially a node is unlinked. prev are None. Initially a node is unlinked.
""" """
# Make it a new-style class in Python 2.2 and up; no effect in 2.1
__metaclass__ = type
__slots__ = ['prev', 'next', 'oid', 'size'] __slots__ = ['prev', 'next', 'oid', 'size']
def __init__(self, oid, size): def __init__(self, oid, size):
...@@ -495,7 +487,6 @@ class Node2Q(Node): ...@@ -495,7 +487,6 @@ class Node2Q(Node):
Node.linkbefore(self, next) Node.linkbefore(self, next)
class TwoQSimluation(Simulation): class TwoQSimluation(Simulation):
# The original 2Q algorithm is page based and the authors offer # The original 2Q algorithm is page based and the authors offer
# tuning guidlines based on a page-based cache. Our cache is # tuning guidlines based on a page-based cache. Our cache is
# object based, so, for example, it's hard to compute the number # object based, so, for example, it's hard to compute the number
...@@ -922,7 +913,6 @@ class ARCCacheSimulation(Simulation): ...@@ -922,7 +913,6 @@ class ARCCacheSimulation(Simulation):
pass pass
class OracleSimulation(LRUCacheSimulation): class OracleSimulation(LRUCacheSimulation):
# Not sure how to implement this yet. This is a cache where I # Not sure how to implement this yet. This is a cache where I
# cheat to see how good we could actually do. The cache # cheat to see how good we could actually do. The cache
# replacement problem for multi-size caches is NP-hard, so we're # replacement problem for multi-size caches is NP-hard, so we're
...@@ -987,7 +977,6 @@ class OracleSimulation(LRUCacheSimulation): ...@@ -987,7 +977,6 @@ class OracleSimulation(LRUCacheSimulation):
all, len(self.count)) all, len(self.count))
class CircularCacheSimulation(Simulation): class CircularCacheSimulation(Simulation):
# The cache is managed as a single file with a pointer that # The cache is managed as a single file with a pointer that
# goes around the file, circularly, forever. New objects # goes around the file, circularly, forever. New objects
# are written at the current pointer, evicting whatever was # are written at the current pointer, evicting whatever was
...@@ -1481,7 +1470,6 @@ def hitrate(loads, hits): ...@@ -1481,7 +1470,6 @@ def hitrate(loads, hits):
return "%5.1f%%" % (100.0 * hits / max(1, loads)) return "%5.1f%%" % (100.0 * hits / max(1, loads))
def duration(secs): def duration(secs):
mm, ss = divmod(secs, 60) mm, ss = divmod(secs, 60)
hh, mm = divmod(mm, 60) hh, mm = divmod(mm, 60)
if hh: if hh:
...@@ -1509,7 +1497,7 @@ def maybe(f, p=0.5): ...@@ -1509,7 +1497,7 @@ def maybe(f, p=0.5):
############################################################################# #############################################################################
# Thor-like eviction scheme. # Thor-like eviction scheme.
# #
# The cache keeps such a list of all objects, and uses a travelling pointer # The cache keeps a list of all objects, and uses a travelling pointer
# to decay the worth of objects over time. # to decay the worth of objects over time.
class ThorNode(Node): class ThorNode(Node):
......
...@@ -149,23 +149,20 @@ def main(): ...@@ -149,23 +149,20 @@ def main():
he = None # timestamp at end of current interval he = None # timestamp at end of current interval
thisinterval = None # generally te//interval thisinterval = None # generally te//interval
f_read = f.read f_read = f.read
struct_unpack = struct.unpack unpack = struct.unpack
# Read file, gathering statistics, and printing each record if verbose. # Read file, gathering statistics, and printing each record if verbose.
try: try:
while 1: while 1:
r = f_read(8) # timestamp:4 code:4 r = f_read(26)
if len(r) < 8: if len(r) < 26:
break break
ts, code = struct_unpack(">ii", r) ts, code, oidlen, start_tid, end_tid = unpack(">iiH8s8s", r)
if ts == 0: if ts == 0:
# Must be a misaligned record caused by a crash. # Must be a misaligned record caused by a crash.
if not quiet: if not quiet:
print "Skipping 8 bytes at offset", f.tell() - 8 print "Skipping 8 bytes at offset", f.tell() - 26
f.seek(f.tell() - 18)
continue continue
r = f_read(18) # oidlen:2 starttid:8 endtid:8
if len(r) < 18:
break
oidlen, start_tid, end_tid = struct_unpack(">H8s8s", r)
oid = f_read(oidlen) oid = f_read(oidlen)
if len(oid) < oidlen: if len(oid) < oidlen:
break break
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment