Skip to content

Z
ZODB
Commit 94a7da6f authored by Tim Peters's avatar Tim Peters
Browse files
Options

Many little bugfixes and improvements in stats.py. 

This has survived several 100 MB of trace files
I generated over the last few days, so it's solid
now if not necessarily perfect.

Replaced simul.py with the much broader-ranging code
Jeremy and I were working on a couple years ago,
although it can't work with the current trace file
format (no real loss there -- the simul.py it's
replacing can't work with the current format either).
parent 2f0cf584
Hide whitespace changes
Inline Side-by-side
Showing with 1037 additions and 86 deletions
doc/ZEO/trace.txt
View file @ 94a7da6f
......@@ -24,7 +24,7 @@ application server) must be restarted.
The trace file can grow pretty quickly; on a moderately loaded server, we
observed it growing by 5 MB per hour. The file consists of binary records,
each 26 bytes long if 8-byte oids are in use; a detailed description of the
each 34 bytes long if 8-byte oids are in use; a detailed description of the
record lay-out is given in stats.py. No sensitive data is logged: data
record sizes and binary object and transaction ids are logged, but no
information about object types or names, user names, version names,
......
src/ZEO/simul.py
View file @ 94a7da6f
#! /usr/bin/env python
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
......@@ -13,7 +14,7 @@
##############################################################################
"""Cache simulation.
Usage: simul.py [-bflyz] [-X] [-s size] tracefile
Usage: simul.py [-bflyz] [-s size] tracefile
Use one of -b, -f, -l, -y or -z select the cache simulator:
-b: buddy system allocator
......@@ -24,8 +25,6 @@ Use one of -b, -f, -l, -y or -z select the cache simulator:
Options:
-s size: cache size in MB (default 20 MB)
-X: enable heuristic checking for misaligned records: oids > 2**32
will be rejected; this requires the tracefile to be seekable
Note: the buddy system allocator rounds the cache size up to a power of 2
"""
......@@ -34,6 +33,9 @@ import sys
import time
import getopt
import struct
import math
from sets import Set
def usage(msg):
print >>sys.stderr, msg
......@@ -44,9 +46,9 @@ def main():
MB = 1000*1000
cachelimit = 20*MB
simclass = ZEOCacheSimulation
heuristic = 0
theta = omicron = None
try:
opts, args = getopt.getopt(sys.argv[1:], "bflyzs:X")
opts, args = getopt.getopt(sys.argv[1:], "bflyz2cOaTUs:o:t:")
except getopt.error, msg:
usage(msg)
return 2
......@@ -63,13 +65,32 @@ def main():
simclass = ZEOCacheSimulation
if o == '-s':
cachelimit = int(float(a)*MB)
if o == '-X':
heuristic = 1
if o == '-2':
simclass = TwoQSimluation
if o == '-c':
simclass = CircularCacheSimulation
if o == '-o':
omicron = float(a)
if o == '-t':
theta = float(a)
if o == '-O':
simclass = OracleSimulation
if o == '-a':
simclass = ARCCacheSimulation
if o == '-T':
simclass = ThorSimulation
if o == '-U':
simclass = UnboundedSimulation
if len(args) != 1:
usage("exactly one file argument required")
return 2
filename = args[0]
if omicron is not None and simclass != CircularCacheSimulation:
usage("-o flag only useful with -c (CircularCacheSimulation)")
return 2
# Open file
if filename.endswith(".gz"):
# Open gzipped file
......@@ -95,7 +116,12 @@ def main():
return 1
# Create simulation object
sim = simclass(cachelimit)
if omicron is not None or theta is not None:
sim = simclass(cachelimit, omicron, theta)
elif simclass is OracleSimulation:
sim = simclass(cachelimit, filename)
else:
sim = simclass(cachelimit)
# Print output header
sim.printheader()
......@@ -107,21 +133,21 @@ def main():
struct_unpack = struct.unpack
while 1:
# Read a record and decode it
r = f_read(10)
if len(r) < 10:
r = f_read(8)
if len(r) < 8:
break
offset += 10
ts, code, lenoid = struct_unpack(">iiH", r)
offset += 8
ts, code = struct_unpack(">ii", r)
if ts == 0:
# Must be a misaligned record caused by a crash
##print "Skipping 8 bytes at offset", offset-8
continue
r = f_read(8 + lenoid)
if len(r) < 8 + lenoid:
r = f_read(16)
if len(r) < 16:
break
offset += 8 + lenoid
offset += 16
records += 1
serial, oid = struct_unpack(">8s%ds" % lenoid, r)
oid, serial = struct_unpack(">8s8s", r)
# Decode the code
dlen, version, code, current = (code & 0x7fffff00,
code & 0x80,
......@@ -153,17 +179,20 @@ class Simulation:
# Initialize global statistics
self.epoch = None
self.total_loads = 0
self.total_hits = 0 # Subclass must increment
self.total_invals = 0
self.total_hits = 0 # subclass must increment
self.total_invals = 0 # subclass must increment
self.total_writes = 0
if not hasattr(self, "extras"):
self.extras = (self.extraname,)
self.format = self.format + " %7s" * len(self.extras)
# Reset per-run statistics and set up simulation data
self.restart()
def restart(self):
# Reset per-run statistics
self.loads = 0
self.hits = 0 # Subclass must increment
self.invals = 0
self.hits = 0 # subclass must increment
self.invals = 0 # subclass must increment
self.writes = 0
self.ts0 = None
......@@ -201,12 +230,14 @@ class Simulation:
pass
def load(self, oid, size):
# Must increment .hits and .total_hits as appropriate.
pass
def inval(self, oid):
# Must increment .invals and .total_invals as appropriate.
pass
format = "%12s %9s %8s %8s %6s %6s %6s %6s"
format = "%12s %9s %8s %8s %6s %6s %7s"
# Subclass should override extraname to name known instance variables;
# if extraname is 'foo', both self.foo and self.total_foo must exist:
......@@ -215,34 +246,46 @@ class Simulation:
def printheader(self):
print "%s, cache size %s bytes" % (self.__class__.__name__,
addcommas(self.cachelimit))
print self.format % (
"START TIME", "DURATION", "LOADS", "HITS",
"INVALS", "WRITES", self.extraname.upper(), "HITRATE")
self.extraheader()
extranames = tuple([s.upper() for s in self.extras])
args = ("START TIME", "DURATION", "LOADS", "HITS",
"INVALS", "WRITES", "HITRATE") + extranames
print self.format % args
def extraheader(self):
pass
nreports = 0
def report(self):
def report(self, extratext=''):
if self.loads:
self.nreports += 1
print self.format % (
time.ctime(self.ts0)[4:-8],
duration(self.ts1 - self.ts0),
self.loads, self.hits, self.invals, self.writes,
getattr(self, self.extraname),
hitrate(self.loads, self.hits))
args = (time.ctime(self.ts0)[4:-8],
duration(self.ts1 - self.ts0),
self.loads, self.hits, self.invals, self.writes,
hitrate(self.loads, self.hits))
args += tuple([getattr(self, name) for name in self.extras])
print self.format % args, extratext
def finish(self):
self.report()
if self.nreports > 1:
print (self.format + " OVERALL") % (
# Make sure that the last line of output ends with "OVERALL". This
# makes it much easier for another program parsing the output to
# find summary statistics.
if self.nreports < 2:
self.report('OVERALL')
else:
self.report()
args = (
time.ctime(self.epoch)[4:-8],
duration(self.ts1 - self.epoch),
self.total_loads,
self.total_hits,
self.total_invals,
self.total_writes,
getattr(self, "total_" + self.extraname),
hitrate(self.total_loads, self.total_hits))
args += tuple([getattr(self, "total_" + name)
for name in self.extras])
print (self.format + " OVERALL") % args
class ZEOCacheSimulation(Simulation):
......@@ -433,6 +476,710 @@ class Node:
self.next = next
prev.next = next.prev = self
am = object()
a1in = object()
a1out = object()
class Node2Q(Node):
__slots__ = ["kind", "hits"]
def __init__(self, oid, size, kind=None):
Node.__init__(self, oid, size)
self.kind = kind
self.hits = 0
def linkbefore(self, next):
if next.kind != self.kind:
self.kind = next.kind
Node.linkbefore(self, next)
class TwoQSimluation(Simulation):
# The original 2Q algorithm is page based and the authors offer
# tuning guidlines based on a page-based cache. Our cache is
# object based, so, for example, it's hard to compute the number
# of oids to store in a1out based on the size of a1in.
extras = "evicts", "hothit", "am_add"
NodeClass = Node2Q
def __init__(self, cachelimit, outlen=10000, threshold=0):
Simulation.__init__(self, cachelimit)
# The promotion threshold: If a hit occurs in a1out, it is
# promoted to am if the number of hits on the object while it
# was in a1in is at least threshold. The standard 2Q scheme
# uses a threshold of 0.
self.threshold = threshold
self.am_limit = 3 * self.cachelimit / 4
self.a1in_limit = self.cachelimit / 4
self.cache = {}
self.am_size = 0
self.a1in_size = 0
self.a1out_size = 0
self.total_evicts = 0
self.total_hothit = 0
self.total_am_add = 0
self.a1out_limit = outlen
# An LRU queue of hot objects
self.am = self.NodeClass(None, None, am)
self.am.linkbefore(self.am)
# A FIFO queue of recently referenced objects. It's purpose
# is to absorb references to objects that are accessed a few
# times in short order, then forgotten about.
self.a1in = self.NodeClass(None, None, a1in)
self.a1in.linkbefore(self.a1in)
# A FIFO queue of recently reference oids.
# This queue only stores the oids, not any data. If we get a
# hit in this queue, promote the object to am.
self.a1out = self.NodeClass(None, None, a1out)
self.a1out.linkbefore(self.a1out)
def makespace(self, size):
for space in 0, size:
if self.enoughspace(size):
return
self.evict_a1in(space)
if self.enoughspace(size):
return
self.evict_am(space)
def enoughspace(self, size):
totalsize = self.a1in_size + self.am_size
return totalsize + size < self.cachelimit
def evict_a1in(self, extra):
while self.a1in_size + extra > self.a1in_limit:
if self.a1in.next is self.a1in:
return
assert self.a1in.next is not None
node = self.a1in.next
self.evicts += 1
self.total_evicts += 1
node.linkbefore(self.a1out)
self.a1out_size += 1
self.a1in_size -= node.size
if self.a1out_size > self.a1out_limit:
assert self.a1out.next is not None
node = self.a1out.next
node.unlink()
del self.cache[node.oid]
self.a1out_size -= 1
def evict_am(self, extra):
while self.am_size + extra > self.am_limit:
if self.am.next is self.am:
return
assert self.am.next is not None
node = self.am.next
self.evicts += 1
self.total_evicts += 1
# This node hasn't been accessed in a while, so just
# forget about it.
node.unlink()
del self.cache[node.oid]
self.am_size -= node.size
def write(self, oid, size):
# A write always follows a read (ZODB doesn't allow blind writes).
# So this write must have followed a recent read of the object.
# Don't change it's position, but do update the size.
# XXX For now, don't evict pages if the new version of the object
# is big enough to require eviction.
node = self.cache.get(oid)
if node is None or node.kind is a1out:
return
if node.kind is am:
self.am_size = self.am_size - node.size + size
node.size = size
else:
self.a1in_size = self.a1in_size - node.size + size
node.size = size
def load(self, oid, size):
node = self.cache.get(oid)
if node is not None:
if node.kind is am:
self.hits += 1
self.total_hits += 1
self.hothit += 1
self.total_hothit += 1
node.hits += 1
node.linkbefore(self.am)
elif node.kind is a1in:
self.hits += 1
self.total_hits += 1
node.hits += 1
elif node.kind is a1out:
self.a1out_size -= 1
if node.hits >= self.threshold:
self.makespace(node.size)
self.am_size += node.size
node.linkbefore(self.am)
self.cache[oid] = node
self.am_add += 1
self.total_am_add += 1
else:
node.unlink()
self.insert(oid, size)
else:
self.insert(oid, size)
def insert(self, oid, size):
# New objects enter the cache via a1in. If they
# are frequently used over a long enough time, they
# will be promoted to am -- but only via a1out.
self.makespace(size)
node = self.NodeClass(oid, size, a1in)
node.linkbefore(self.a1in)
self.cache[oid] = node
self.a1in_size += node.size
def inval(self, oid):
# The original 2Q algorithm didn't have to deal with
# invalidations. My own solution: Move it to the head of
# a1out.
node = self.cache.get(oid)
if node is None:
return
self.invals += 1
self.total_invals += 1
# XXX Should an invalidation to a1out count?
if node.kind is a1out:
return
node.linkbefore(self.a1out)
if node.kind is am:
self.am_size -= node.size
else:
self.a1in_size -= node.size
def restart(self):
Simulation.restart(self)
self.evicts = 0
self.hothit = 0
self.am_add = 0
lruT = object()
lruB = object()
fifoT = object()
fifoB = object()
class ARCCacheSimulation(Simulation):
# Based on the paper ARC: A Self-Tuning, Low Overhead Replacement
# Cache by Nimrod Megiddo and Dharmendra S. Modha, USENIX FAST
# 2003. The paper describes a block-based cache. A lot of the
# details need to be fiddled to work with an object-based cache.
# For size issues, the key insight ended up being conditions
# A.1-A.5 rather than the details of the algorithm in Fig. 4.
extras = "lruThits", "evicts", "p"
def __init__(self, cachelimit):
Simulation.__init__(self, cachelimit)
# There are two pairs of linked lists. Each pair has a top and
# bottom half. The bottom half contains metadata, but not actual
# objects.
# LRU list of frequently used objects
self.lruT = Node2Q(None, None, lruT)
self.lruT.linkbefore(self.lruT)
self.lruT_len = 0
self.lruT_size = 0
self.lruB = Node2Q(None, None, lruB)
self.lruB.linkbefore(self.lruB)
self.lruB_len = 0
self.lruB_size = 0
# FIFO list of objects seen once
self.fifoT = Node2Q(None, None, fifoT)
self.fifoT.linkbefore(self.fifoT)
self.fifoT_len = 0
self.fifoT_size = 0
self.fifoB = Node2Q(None, None, fifoB)
self.fifoB.linkbefore(self.fifoB)
self.fifoB_len = 0
self.fifoB_size = 0
# maps oid to node
self.cache = {}
# The paper says that p should be adjust be 1 as the minimum:
# "The compound effect of such small increments and decrements
# to p s quite profound as we will demonstrated in the next
# section." Not really, as far as I can tell. In my traces
# with a very small cache, it was taking far too long to adjust
# towards favoring some FIFO component. I would guess that the
# chief difference is that our caches are much bigger than the
# ones they experimented with. Their biggest cache had 512K
# entries, while our smallest cache will have 40 times that many
# entries.
self.p = 0
# XXX multiply computed adjustments to p by walk_factor
self.walk_factor = 500
# statistics
self.total_hits = 0
self.total_lruThits = 0
self.total_fifoThits = 0
self.total_evicts = 0
def restart(self):
Simulation.restart(self)
self.hits = 0
self.lruThits = 0
self.fifoThits = 0
self.evicts = 0
def write(self, oid, size):
pass
def replace(self, lruB=False):
self.evicts += 1
self.total_evicts += 1
if self.fifoT_size > self.p or (lruB and self.fifoT_size == self.p):
node = self.fifoT.next
if node is self.fifoT:
return 0
assert node is not self.fifoT, self.stats()
node.linkbefore(self.fifoB)
self.fifoT_len -= 1
self.fifoT_size -= node.size
self.fifoB_len += 1
self.fifoB_size += node.size
else:
node = self.lruT.next
if node is self.lruT:
return 0
assert node is not self.lruT, self.stats()
node.linkbefore(self.lruB)
self.lruT_len -= 1
self.lruT_size -= node.size
self.lruB_len += 1
self.lruB_size += node.size
return node.size
def stats(self):
self.totalsize = self.lruT_size + self.fifoT_size
self.allsize = self.totalsize + self.lruB_size + self.fifoB_size
print "cachelimit = %s totalsize=%s allsize=%s" % (
addcommas(self.cachelimit),
addcommas(self.totalsize),
addcommas(self.allsize))
fmt = (
"p=%(p)d\n"
"lruT = %(lruT_len)5d / %(lruT_size)8d / %(lruThits)d\n"
"fifoT = %(fifoT_len)5d / %(fifoT_size)8d / %(fifoThits)d\n"
"lruB = %(lruB_len)5d / %(lruB_size)8d\n"
"fifoB = %(fifoB_len)5d / %(fifoB_size)8d\n"
"loads=%(loads)d hits=%(hits)d evicts=%(evicts)d\n"
)
print fmt % self.__dict__
def report(self):
self.total_p = self.p
Simulation.report(self)
## self.stats()
def load(self, oid, size):
## maybe(self.stats, p=0.002)
node = self.cache.get(oid)
if node is None:
# cache miss: We're going to insert a new object in fifoT.
# If fifo is full, we'll need to evict something to make
# room for it.
prev = need = size
while need > 0:
if size + self.fifoT_size + self.fifoB_size >= self.cachelimit:
if need + self.fifoT_size >= self.cachelimit:
node = self.fifoB.next
assert node is not self.fifoB, self.stats()
node.unlink()
del self.cache[node.oid]
self.fifoB_size -= node.size
self.fifoB_len -= 1
self.evicts += 1
self.total_evicts += 1
else:
node = self.fifoB.next
assert node is not self.fifoB, self.stats()
node.unlink()
del self.cache[node.oid]
self.fifoB_size -= node.size
self.fifoB_len -= 1
if self.fifoT_size + self.lruT_size > self.cachelimit:
need -= self.replace()
else:
incache_size = self.fifoT_size + self.lruT_size + need
total_size = (incache_size + self.fifoB_size
+ self.lruB_size)
if total_size >= self.cachelimit * 2:
node = self.lruB.next
if node is self.lruB:
break
assert node is not self.lruB
node.unlink()
del self.cache[node.oid]
self.lruB_size -= node.size
self.lruB_len -= 1
elif incache_size > self.cachelimit:
need -= self.replace()
else:
break
if need == prev:
# XXX hack, apparently we can't get rid of anything else
break
prev = need
node = Node2Q(oid, size)
node.linkbefore(self.fifoT)
self.fifoT_len += 1
self.fifoT_size += size
self.cache[oid] = node
else:
# a cache hit, but possibly in a bottom list that doesn't
# actually hold the object
if node.kind is lruT:
node.linkbefore(self.lruT)
self.hits += 1
self.total_hits += 1
self.lruThits += 1
self.total_lruThits += 1
elif node.kind is fifoT:
node.linkbefore(self.lruT)
self.fifoT_len -= 1
self.lruT_len += 1
self.fifoT_size -= node.size
self.lruT_size += node.size
self.hits += 1
self.total_hits += 1
self.fifoThits += 1
self.total_fifoThits += 1
elif node.kind is fifoB:
node.linkbefore(self.lruT)
self.fifoB_len -= 1
self.lruT_len += 1
self.fifoB_size -= node.size
self.lruT_size += node.size
# XXX need a better min than 1?
## print "adapt+", max(1, self.lruB_size // self.fifoB_size)
delta = max(1, self.lruB_size / max(1, self.fifoB_size))
self.p += delta * self.walk_factor
if self.p > self.cachelimit:
self.p = self.cachelimit
need = node.size
if self.lruT_size + self.fifoT_size + need > self.cachelimit:
while need > 0:
r = self.replace()
if not r:
break
need -= r
elif node.kind is lruB:
node.linkbefore(self.lruT)
self.lruB_len -= 1
self.lruT_len += 1
self.lruB_size -= node.size
self.lruT_size += node.size
# XXX need a better min than 1?
## print "adapt-", max(1, self.fifoB_size // self.lruB_size)
delta = max(1, self.fifoB_size / max(1, self.lruB_size))
self.p -= delta * self.walk_factor
if self.p < 0:
self.p = 0
need = node.size
if self.lruT_size + self.fifoT_size + need > self.cachelimit:
while need > 0:
r = self.replace(lruB=True)
if not r:
break
need -= r
def inval(self, oid):
pass
def extraheader(self):
pass
class OracleSimulation(LRUCacheSimulation):
# Not sure how to implement this yet. This is a cache where I
# cheat to see how good we could actually do. The cache
# replacement problem for multi-size caches is NP-hard, so we're
# not going to have an optimal solution.
# At the moment, the oracle is mostly blind. It knows which
# objects will be referenced more than once, so that it can
# ignore objects referenced only once. In most traces, these
# objects account for about 20% of references.
def __init__(self, cachelimit, filename):
LRUCacheSimulation.__init__(self, cachelimit)
self.count = {}
self.scan(filename)
def load(self, oid, size):
node = self.cache.get(oid)
if node is not None:
self.hits += 1
self.total_hits += 1
node.linkbefore(self.head)
else:
if oid in self.count:
self.write(oid, size)
def scan(self, filename):
# scan the file in advance to figure out which objects will
# be referenced more than once.
f = open(filename, "rb")
struct_unpack = struct.unpack
f_read = f.read
offset = 0
while 1:
# Read a record and decode it
r = f_read(8)
if len(r) < 8:
break
offset += 8
ts, code = struct_unpack(">ii", r)
if ts == 0:
# Must be a misaligned record caused by a crash
##print "Skipping 8 bytes at offset", offset-8
continue
r = f_read(16)
if len(r) < 16:
break
offset += 16
oid, serial = struct_unpack(">8s8s", r)
if code & 0x70 == 0x20:
# only look at loads
self.count[oid] = self.count.get(oid, 0) + 1
all = len(self.count)
# Now remove everything with count == 1
once = [oid for oid, count in self.count.iteritems()
if count == 1]
for oid in once:
del self.count[oid]
print "Scanned file, %d unique oids, %d repeats" % (
all, len(self.count))
class CircularCacheSimulation(Simulation):
# The cache is managed as a single file with a pointer that
# goes around the file, circularly, forever. New objects
# are written at the current pointer, evicting whatever was
# there previously.
# For each cache hit, there is some distance between the current
# pointer offset and the offset of the cached data record. The
# cache can be configured to copy objects to the current offset
# depending on how far away they are now. The omicron parameter
# specifies a percentage
extras = "evicts", "copies", "inuse", "skips"
def __init__(self, cachelimit, omicron=None, skip=None):
Simulation.__init__(self, cachelimit)
self.omicron = omicron or 0
self.skip = skip or 0
self.total_evicts = 0
self.total_copies = 0
self.total_skips = 0
# Current offset in file
self.offset = 0
# Map offset in file to tuple of size, oid
self.filemap = {0: (self.cachelimit, None)}
# Map oid to offset, node
self.cache = {}
# LRU list of oids
self.head = Node(None, None)
self.head.linkbefore(self.head)
def extraheader(self):
print "omicron = %s, theta = %s" % (self.omicron, self.skip)
def restart(self):
Simulation.restart(self)
self.evicts = 0
self.copies = 0
self.skips = 0
def load(self, oid, size):
p = self.cache.get(oid)
if p is None:
self.add(oid, size)
else:
pos, node = p
self.hits += 1
self.total_hits += 1
node.linkbefore(self.head)
self.copy(oid, size, pos)
def check(self):
d = dict(self.filemap)
done = {}
while d:
pos, (size, oid) = d.popitem()
next = pos + size
if not (next in d or next in done or next == self.cachelimit):
print "check", next, pos, size, repr(oid)
self.dump()
raise RuntimeError
done[pos] = pos
def dump(self):
print len(self.filemap)
L = list(self.filemap)
L.sort()
for k in L:
v = self.filemap[k]
print k, v[0], repr(v[1])
def add(self, oid, size):
avail = self.makeroom(size)
assert oid not in self.cache
self.filemap[self.offset] = size, oid
node = Node(oid, size)
node.linkbefore(self.head)
self.cache[oid] = self.offset, node
self.offset += size
# All the space made available must be accounted for in filemap.
excess = avail - size
if excess:
self.filemap[self.offset] = excess, None
def makeroom(self, need):
if self.offset + need > self.cachelimit:
self.offset = 0
pos = self.offset
# Evict enough objects to make the necessary space available.
self.compute_closeenough()
evicted = False
while need > 0:
if pos == self.cachelimit:
print "wrap makeroom", need
pos = 0
try:
size, oid = self.filemap[pos]
except:
self.dump()
raise
if not evicted and self.skip and oid and self.closeenough(oid):
self.skips += 1
self.total_skips += 1
self.offset += size
pos += size
continue
evicted = True
del self.filemap[pos]
if oid is not None:
self.evicts += 1
self.total_evicts += 1
pos, node = self.cache.pop(oid)
node.unlink()
need -= size
pos += size
return pos - self.offset
def compute_closeenough(self):
self.lru = {}
n = int(len(self.cache) * self.skip)
node = self.head.prev
while n > 0:
self.lru[node.oid] = True
node = node.prev
n -= 1
def closeenough(self, oid):
# If oid is in the top portion of the most recently used
# elements, skip it.
return oid in self.lru
def copy(self, oid, size, pos):
# Copy only if the distance is greater than omicron.
dist = self.offset - pos
if dist < 0:
dist += self.cachelimit
if dist < self.omicron * self.cachelimit:
self.copies += 1
self.total_copies += 1
self.filemap[pos] = size, None
pos, node = self.cache.pop(oid)
node.unlink()
self.add(oid, size)
def inval(self, oid):
p = self.cache.get(oid)
if p is None:
return
pos, node = p
self.invals += 1
self.total_invals += 1
size, _oid = self.filemap[pos]
assert oid == _oid
self.filemap[pos] = size, None
pos, node = self.cache.pop(oid)
node.unlink()
def write(self, oid, size):
p = self.cache.get(oid)
if p is None:
return
pos, node = p
oldsize, _oid = self.filemap[pos]
assert oid == _oid
if size == oldsize:
return
if size < oldsize:
excess = oldsize - size
self.filemap[pos] = size, oid
self.filemap[pos + size] = excess, None
else:
self.filemap[pos] = oldsize, None
pos, node = self.cache.pop(oid)
node.unlink()
self.add(oid, size)
def report(self):
free = used = total = 0
for size, oid in self.filemap.itervalues():
total += size
if oid:
used += size
else:
free += size
self.inuse = round(100.0 * used / total, 1)
self.total_inuse = self.inuse
Simulation.report(self)
class BuddyCacheSimulation(LRUCacheSimulation):
def __init__(self, cachelimit):
......@@ -753,5 +1500,218 @@ def addcommas(n):
i -= 3
return sign + s
import random
def maybe(f, p=0.5):
if random.random() < p:
f()
#############################################################################
# Thor-like eviction scheme.
#
# The cache keeps such a list of all objects, and uses a travelling pointer
# to decay the worth of objects over time.
class ThorNode(Node):
__slots__ = ['worth']
def __init__(self, oid, size, worth=None):
Node.__init__(self, oid, size)
self.worth = worth
class ThorListHead(Node):
def __init__(self):
Node.__init__(self, 0, 0)
self.next = self.prev = self
class ThorSimulation(Simulation):
extras = "evicts", "trips"
def __init__(self, cachelimit):
Simulation.__init__(self, cachelimit)
# Maximum total of object sizes we keep in cache.
self.maxsize = cachelimit
# Current total of object sizes in cache.
self.currentsize = 0
# A worth byte maps to a set of all objects with that worth.
# This is cheap to keep updated, and makes finding low-worth
# objects for eviction trivial (just march over the worthsets
# list, in order).
self.worthsets = [Set() for dummy in range(256)]
# We keep a circular list of all objects in cache. currentobj
# walks around it forever. Each time _tick() is called, the
# worth of currentobj is decreased, basically by shifting
# right 1, and currentobj moves on to the next object. When
# an object is first inserted, it enters the list right before
# currentobj. When an object is accessed, its worth is
# increased by or'ing in 0x80. This scheme comes from the
# Thor system, and is an inexpensive way to account for both
# recency and frequency of access: recency is reflected in
# the leftmost bit set, and frequency by how many bits are
# set.
#
# Note: because evictions are interleaved with ticks,
# unlinking an object is tricky, lest we evict currentobj. The
# class _unlink method takes care of this properly.
self.listhead = ThorListHead()
self.currentobj = self.listhead
# Map an object.oid to its ThorNode.
self.oid2object = {}
self.total_evicts = self.total_trips = 0
# Unlink object from the circular list, taking care not to lose
# track of the current object. Always call this instead of
# invoking obj.unlink() directly.
def _unlink(self, obj):
assert obj is not self.listhead
if obj is self.currentobj:
self.currentobj = obj.next
obj.unlink()
# Change obj.worth to newworth, maintaining invariants.
def _change_worth(self, obj, newworth):
if obj.worth != newworth:
self.worthsets[obj.worth].remove(obj)
obj.worth = newworth
self.worthsets[newworth].add(obj)
def add(self, object):
assert object.oid not in self.oid2object
self.oid2object[object.oid] = object
newsize = self.currentsize + object.size
if newsize > self.maxsize:
self._evictbytes(newsize - self.maxsize)
self.currentsize += object.size
object.linkbefore(self.currentobj)
if object.worth is None:
# Give smaller objects higher initial worth. This favors kicking
# out unreferenced large objects before kicking out unreferenced
# small objects. On real life traces, this is a significant
# win for the hit rate.
object.worth = 32 - int(round(math.log(object.size, 2)))
self.worthsets[object.worth].add(object)
# Decrease the worth of the current object, and advance the
# current object.
def _tick(self):
c = self.currentobj
if c is self.listhead:
c = c.next
if c is self.listhead: # list is empty
return
self.total_trips += 1
self.trips += 1
self._change_worth(c, (c.worth + 1) >> 1)
self.currentobj = c.next
def access(self, oid):
self._tick()
obj = self.oid2object.get(oid)
if obj is None:
return None
self._change_worth(obj, obj.worth | 0x80)
return obj
# Evict objects of least worth first, until at least nbytes bytes
# have been freed.
def _evictbytes(self, nbytes):
for s in self.worthsets:
while s:
if nbytes <= 0:
return
obj = s.pop()
nbytes -= obj.size
self._evictobj(obj)
def _evictobj(self, obj):
self.currentsize -= obj.size
self.worthsets[obj.worth].discard(obj)
del self.oid2object[obj.oid]
self._unlink(obj)
self.evicts += 1
self.total_evicts += 1
def _evict_without_bumping_evict_stats(self, obj):
self._evictobj(obj)
self.evicts -= 1
self.total_evicts -= 1
# Simulator overrides from here on.
def restart(self):
# Reset base class
Simulation.restart(self)
# Reset additional per-run statistics
self.evicts = self.trips = 0
def write(self, oid, size):
obj = self.oid2object.get(oid)
worth = None
if obj is not None:
worth = obj.worth
self._evict_without_bumping_evict_stats(obj)
self.add(ThorNode(oid, size, worth))
def load(self, oid, size):
if self.access(oid) is not None:
self.hits += 1
self.total_hits += 1
else:
self.write(oid, size)
def inval(self, oid):
obj = self.oid2object.get(oid)
if obj is not None:
self.invals += 1
self.total_invals += 1
self._evict_without_bumping_evict_stats(obj)
# Take the "x" off to see additional stats after each restart period.
def xreport(self):
Simulation.report(self)
print 'non-empty worth sets', sum(map(bool, self.worthsets)),
print 'objects', len(self.oid2object),
print 'size', self.currentsize
#############################################################################
# Perfection: What if the cache were unbounded, and never forgot anything?
# This simulator answers that question directly; the cache size parameter
# isn't used.
class UnboundedSimulation(Simulation):
extraname = 'evicts' # for some reason we *have* to define >= 1 extra
def __init__(self, cachelimit):
Simulation.__init__(self, cachelimit)
self.oids = Set()
self.evicts = self.total_evicts = 0
def write(self, oid, size):
self.oids.add(oid)
def load(self, oid, size):
if oid in self.oids:
self.hits += 1
self.total_hits += 1
else:
self.oids.add(oid)
def inval(self, oid):
if oid in self.oids:
self.invals += 1
self.total_invals += 1
self.oids.remove(oid)
if __name__ == "__main__":
sys.exit(main())
src/ZEO/stats.py
View file @ 94a7da6f
......@@ -26,7 +26,7 @@ Usage: stats.py [-h] [-i interval] [-q] [-s] [-S] [-v] [-X] tracefile
"""File format:
Each record is 18 bytes, plus a variable number of bytes to store an oid,
Each record is 26 bytes, plus a variable number of bytes to store an oid,
with the following layout. Numbers are big-endian integers.
Offset Size Contents
......@@ -35,8 +35,9 @@ Offset Size Contents
4 3 data size, in 256-byte increments, rounded up
7 1 code (see below)
8 2 object id length
10 8 serial number
18 variable object id
10 8 start tid
18 8 end tid
26 variable object id
The code at offset 7 packs three fields:
......@@ -131,74 +132,66 @@ def main():
print >> sys.stderr, "can't open %s: %s" % (filename, msg)
return 1
# Read file, gathering statistics, and printing each record if verbose
rt0 = time.time()
# bycode -- map code to count of occurrences
bycode = {}
# records -- number of records
records = 0
# version -- number of records with versions
versions = 0
t0 = te = None
# datarecords -- number of records with dlen set
datarecords = 0
datasize = 0L
# oids -- maps oid to number of times it was loaded
oids = {}
# bysize -- maps data size to number of loads
bysize = {}
# bysize -- maps data size to number of writes
bysizew = {}
bycode = {} # map code to count of occurrences
byinterval = {} # map code to count in current interval
records = 0 # number of trace records read
versions = 0 # number of trace records with versions
datarecords = 0 # number of records with dlen set
datasize = 0L # sum of dlen across records with dlen set
oids = {} # map oid to number of times it was loaded
bysize = {} # map data size to number of loads
bysizew = {} # map data size to number of writes
total_loads = 0
byinterval = {}
thisinterval = None
h0 = he = None
offset = 0
t0 = None # first timestamp seen
te = None # most recent timestamp seen
h0 = None # timestamp at start of current interval
he = None # timestamp at end of current interval
thisinterval = None # generally te//interval
f_read = f.read
struct_unpack = struct.unpack
# Read file, gathering statistics, and printing each record if verbose.
try:
while 1:
r = f_read(8)
r = f_read(8) # timestamp:4 code:4
if len(r) < 8:
break
offset += 8
ts, code = struct_unpack(">ii", r)
if ts == 0:
# Must be a misaligned record caused by a crash
# Must be a misaligned record caused by a crash.
if not quiet:
print "Skipping 8 bytes at offset", offset-8
print "Skipping 8 bytes at offset", f.tell() - 8
continue
r = f_read(18)
if len(r) < 10:
r = f_read(18) # oidlen:2 starttid:8 endtid:8
if len(r) < 18:
break
offset += 10
records += 1
oidlen, start_tid, end_tid = struct_unpack(">H8s8s", r)
oid = f_read(oidlen)
if len(oid) != oidlen:
if len(oid) < oidlen:
break
offset += oidlen
records += 1
if t0 is None:
t0 = ts
thisinterval = t0 / interval
thisinterval = t0 // interval
h0 = he = ts
te = ts
if ts / interval != thisinterval:
if ts // interval != thisinterval:
if not quiet:
dumpbyinterval(byinterval, h0, he)
byinterval = {}
thisinterval = ts / interval
thisinterval = ts // interval
h0 = ts
he = ts
dlen, code = code & 0x7fffff00, code & 0xff
if dlen:
datarecords += 1
datasize += dlen
version = '-'
if code & 0x80:
version = 'V'
versions += 1
code = code & 0x7e
else:
version = '-'
code &= 0x7e
bycode[code] = bycode.get(code, 0) + 1
byinterval[code] = byinterval.get(code, 0) + 1
if dlen:
......@@ -220,11 +213,11 @@ def main():
if code & 0x70 == 0x20:
oids[oid] = oids.get(oid, 0) + 1
total_loads += 1
if code == 0x00:
if code == 0x00: # restart
if not quiet:
dumpbyinterval(byinterval, h0, he)
byinterval = {}
thisinterval = ts / interval
thisinterval = ts // interval
h0 = he = ts
if not quiet:
print time.ctime(ts)[4:-5],
......@@ -232,6 +225,7 @@ def main():
except KeyboardInterrupt:
print "\nInterrupted. Stats so far:\n"
end_pos = f.tell()
f.close()
rte = time.time()
if not quiet:
......@@ -245,8 +239,8 @@ def main():
# Print statistics
if dostats:
print
print "Read %s records (%s bytes) in %.1f seconds" % (
addcommas(records), addcommas(records*24), rte-rt0)
print "Read %s trace records (%s bytes) in %.1f seconds" % (
addcommas(records), addcommas(end_pos), rte-rt0)
print "Versions: %s records used a version" % addcommas(versions)
print "First time: %s" % time.ctime(t0)
print "Last time: %s" % time.ctime(te)
......@@ -309,9 +303,8 @@ def dumpbysize(bysize, how, how2):
loads)
def dumpbyinterval(byinterval, h0, he):
loads = 0
hits = 0
for code in byinterval.keys():
loads = hits = 0
for code in byinterval:
if code & 0x70 == 0x20:
n = byinterval[code]
loads += n
......@@ -328,8 +321,7 @@ def dumpbyinterval(byinterval, h0, he):
addcommas(loads), addcommas(hits), hr)
def hitrate(bycode):
loads = 0
hits = 0
loads = hits = 0
for code in bycode:
if code & 0x70 == 0x20:
n = bycode[code]
......@@ -389,7 +381,6 @@ explain = {
0x50: "store (version)",
0x52: "store (current, non-version)",
0x54: "store (non-current)",
}
if __name__ == "__main__":
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7