Skip to content

Z
ZODB
Commit 27b61d35 authored by Tim Peters's avatar Tim Peters
Browse files
Options

Merge rev 37389 from 3.4 branch. 

Lots of fiddling to work w/ current trace file format.

Output isn't obviously wholly insane, but isn't a reasonably
close enough match to stats.py output either yet.
parent fb8f5ad2
Hide whitespace changes
Inline Side-by-side
Showing with 90 additions and 179 deletions
src/ZEO/simul.py
View file @ 27b61d35
...@@ -42,13 +42,12 @@ def usage(msg): ...@@ -42,13 +42,12 @@ def usage(msg):
print >>sys.stderr, __doc__ print >>sys.stderr, __doc__
def main(): def main():
# Parse options # Parse options.
MB = 1000*1000 MB = 1024**2
cachelimit = 20*MB cachelimit = 20*MB
simclass = ZEOCacheSimulation simclass = ZEOCacheSimulation
theta = omicron = None
try: try:
opts, args = getopt.getopt(sys.argv[1:], "bflyz2cOaTUs:o:t:") opts, args = getopt.getopt(sys.argv[1:], "bflyz2cOaTUs:")
except getopt.error, msg: except getopt.error, msg:
usage(msg) usage(msg)
return 2 return 2
...@@ -69,10 +68,6 @@ def main(): ...@@ -69,10 +68,6 @@ def main():
simclass = TwoQSimluation simclass = TwoQSimluation
elif o == '-c': elif o == '-c':
simclass = CircularCacheSimulation simclass = CircularCacheSimulation
elif o == '-o':
omicron = float(a)
elif o == '-t':
theta = float(a)
elif o == '-O': elif o == '-O':
simclass = OracleSimulation simclass = OracleSimulation
elif o == '-a': elif o == '-a':
...@@ -89,13 +84,9 @@ def main(): ...@@ -89,13 +84,9 @@ def main():
return 2 return 2
filename = args[0] filename = args[0]
if omicron is not None and simclass != CircularCacheSimulation: # Open file.
usage("-o flag only useful with -c (CircularCacheSimulation)")
return 2
# Open file
if filename.endswith(".gz"): if filename.endswith(".gz"):
# Open gzipped file # Open gzipped file.
try: try:
import gzip import gzip
except ImportError: except ImportError:
...@@ -107,57 +98,55 @@ def main(): ...@@ -107,57 +98,55 @@ def main():
print >> sys.stderr, "can't open %s: %s" % (filename, msg) print >> sys.stderr, "can't open %s: %s" % (filename, msg)
return 1 return 1
elif filename == "-": elif filename == "-":
# Read from stdin # Read from stdin.
f = sys.stdin f = sys.stdin
else: else:
# Open regular file # Open regular file.
try: try:
f = open(filename, "rb") f = open(filename, "rb")
except IOError, msg: except IOError, msg:
print >> sys.stderr, "can't open %s: %s" % (filename, msg) print >> sys.stderr, "can't open %s: %s" % (filename, msg)
return 1 return 1
# Create simulation object # Create simulation object.
if omicron is not None or theta is not None: if simclass is OracleSimulation:
sim = simclass(cachelimit, omicron, theta)
elif simclass is OracleSimulation:
sim = simclass(cachelimit, filename) sim = simclass(cachelimit, filename)
else: else:
sim = simclass(cachelimit) sim = simclass(cachelimit)
# Print output header # Print output header.
sim.printheader() sim.printheader()
# Read trace file, simulating cache behavior # Read trace file, simulating cache behavior.
records = 0
f_read = f.read f_read = f.read
struct_unpack = struct.unpack unpack = struct.unpack
while 1: while 1:
# Read a record and decode it # Read a record and decode it.
r = f_read(8) r = f_read(26)
if len(r) < 8: if len(r) < 26:
break break
ts, code = struct_unpack(">ii", r) ts, code, oidlen, start_tid, end_tid = unpack(">iiH8s8s", r)
if ts == 0: if ts == 0:
# Must be a misaligned record caused by a crash # Must be a misaligned record caused by a crash; skip 8 bytes
# and try again. Why 8? Lost in the mist of history.
f.seek(f.tell() - 18)
continue continue
r = f_read(16) oid = f_read(oidlen)
if len(r) < 16: if len(oid) < oidlen:
break break
records += 1 # Decode the code.
oid, serial = struct_unpack(">8s8s", r)
# Decode the code
dlen, version, code, current = (code & 0x7fffff00, dlen, version, code, current = (code & 0x7fffff00,
code & 0x80, code & 0x80,
code & 0x7e, code & 0x7e,
code & 0x01) code & 0x01)
# And pass it to the simulation # And pass it to the simulation.
sim.event(ts, dlen, version, code, current, oid, serial) sim.event(ts, dlen, version, code, current, oid, start_tid)
# Finish simulation f.close()
# Finish simulation.
sim.finish() sim.finish()
# Exit code from main() # Exit code from main().
return 0 return 0
class Simulation: class Simulation:
...@@ -195,32 +184,33 @@ class Simulation: ...@@ -195,32 +184,33 @@ class Simulation:
self.ts0 = None self.ts0 = None
def event(self, ts, dlen, _version, code, _current, oid, _serial): def event(self, ts, dlen, _version, code, _current, oid, _serial):
# Record first and last timestamp seen # Record first and last timestamp seen.
if self.ts0 is None: if self.ts0 is None:
self.ts0 = ts self.ts0 = ts
if self.epoch is None: if self.epoch is None:
self.epoch = ts self.epoch = ts
self.ts1 = ts self.ts1 = ts
# Simulate cache behavior. Use load hits, updates and stores # Simulate cache behavior.
# only (each load miss is followed immediately by a store action = code & 0x70 # ignore high bit (version flag)
# unless the object in fact did not exist). Updates always write. if dlen:
if dlen and code & 0x70 in (0x20, 0x30, 0x50): if action == 0x20:
if code == 0x3A: # Load.
# Update self.loads += 1
self.total_loads += 1
self.load(oid, dlen)
elif action == 0x50:
# Store.
self.writes += 1 self.writes += 1
self.total_writes += 1 self.total_writes += 1
self.write(oid, dlen) self.write(oid, dlen)
else: else:
# Load hit or store -- these are really the load requests assert False, (hex(code), dlen)
self.loads += 1 elif action == 0x10:
self.total_loads += 1 # Invalidate.
self.load(oid, dlen)
elif code & 0x70 == 0x10:
# Invalidate
self.inval(oid) self.inval(oid)
elif code == 0x00: elif action == 0x00:
# Restart # Restart.
self.report() self.report()
self.restart() self.restart()
...@@ -982,78 +972,34 @@ class CircularCacheSimulation(Simulation): ...@@ -982,78 +972,34 @@ class CircularCacheSimulation(Simulation):
# are written at the current pointer, evicting whatever was # are written at the current pointer, evicting whatever was
# there previously. # there previously.
# For each cache hit, there is some distance between the current extras = "evicts", "inuse"
# pointer offset and the offset of the cached data record. The
# cache can be configured to copy objects to the current offset
# depending on how far away they are now. The omicron parameter
# specifies a percentage
extras = "evicts", "copies", "inuse", "skips"
def __init__(self, cachelimit, omicron=None, skip=None): def __init__(self, cachelimit):
Simulation.__init__(self, cachelimit) Simulation.__init__(self, cachelimit)
self.omicron = omicron or 0
self.skip = skip or 0
self.total_evicts = 0 self.total_evicts = 0
self.total_copies = 0 # Current offset in file.
self.total_skips = 0
# Current offset in file
self.offset = 0 self.offset = 0
# Map offset in file to tuple of size, oid # Map offset in file to (size, oid) pair.
self.filemap = {0: (self.cachelimit, None)} self.filemap = {0: (self.cachelimit, None)}
# Map oid to offset, node # Map oid to file offset.
self.cache = {} self.oid2ofs = {}
# LRU list of oids
self.head = Node(None, None)
self.head.linkbefore(self.head)
def extraheader(self):
print "omicron = %s, theta = %s" % (self.omicron, self.skip)
def restart(self): def restart(self):
Simulation.restart(self) Simulation.restart(self)
self.evicts = 0 self.evicts = 0
self.copies = 0
self.skips = 0
def load(self, oid, size): def load(self, oid, size):
p = self.cache.get(oid) if oid in self.oid2ofs:
if p is None:
self.add(oid, size)
else:
pos, node = p
self.hits += 1 self.hits += 1
self.total_hits += 1 self.total_hits += 1
node.linkbefore(self.head) else:
self.copy(oid, size, pos) self.add(oid, size)
def check(self):
d = dict(self.filemap)
done = {}
while d:
pos, (size, oid) = d.popitem()
next = pos + size
if not (next in d or next in done or next == self.cachelimit):
print "check", next, pos, size, repr(oid)
self.dump()
raise RuntimeError
done[pos] = pos
def dump(self):
print len(self.filemap)
L = list(self.filemap)
L.sort()
for k in L:
v = self.filemap[k]
print k, v[0], repr(v[1])
def add(self, oid, size): def add(self, oid, size):
avail = self.makeroom(size) avail = self.makeroom(size)
assert oid not in self.cache assert oid not in self.oid2ofs
self.filemap[self.offset] = size, oid self.filemap[self.offset] = size, oid
node = Node(oid, size) self.oid2ofs[oid] = self.offset
node.linkbefore(self.head)
self.cache[oid] = self.offset, node
self.offset += size self.offset += size
# All the space made available must be accounted for in filemap. # All the space made available must be accounted for in filemap.
excess = avail - size excess = avail - size
...@@ -1061,102 +1007,48 @@ class CircularCacheSimulation(Simulation): ...@@ -1061,102 +1007,48 @@ class CircularCacheSimulation(Simulation):
self.filemap[self.offset] = excess, None self.filemap[self.offset] = excess, None
def makeroom(self, need): def makeroom(self, need):
# Evict enough objects to make the necessary space available.
if self.offset + need > self.cachelimit: if self.offset + need > self.cachelimit:
self.offset = 0 self.offset = 0
pos = self.offset pos = self.offset
# Evict enough objects to make the necessary space available.
self.compute_closeenough()
evicted = False
while need > 0: while need > 0:
if pos == self.cachelimit: assert pos < self.cachelimit
print "wrap makeroom", need
pos = 0
try: try:
size, oid = self.filemap[pos] size, oid = self.filemap[pos]
except: except KeyError:
self.dump() self.dump()
raise raise
if not evicted and self.skip and oid and self.closeenough(oid):
self.skips += 1
self.total_skips += 1
self.offset += size
pos += size
continue
evicted = True
del self.filemap[pos] del self.filemap[pos]
if oid:
if oid is not None:
self.evicts += 1 self.evicts += 1
self.total_evicts += 1 self.total_evicts += 1
pos, node = self.cache.pop(oid) _pos = self.oid2ofs.pop(oid)
node.unlink() assert pos == _pos
need -= size need -= size
pos += size pos += size
return pos - self.offset # total number of bytes freed
return pos - self.offset
def compute_closeenough(self):
self.lru = {}
n = int(len(self.cache) * self.skip)
node = self.head.prev
while n > 0:
self.lru[node.oid] = True
node = node.prev
n -= 1
def closeenough(self, oid):
# If oid is in the top portion of the most recently used
# elements, skip it.
return oid in self.lru
def copy(self, oid, size, pos):
# Copy only if the distance is greater than omicron.
dist = self.offset - pos
if dist < 0:
dist += self.cachelimit
if dist < self.omicron * self.cachelimit:
self.copies += 1
self.total_copies += 1
self.filemap[pos] = size, None
pos, node = self.cache.pop(oid)
node.unlink()
self.add(oid, size)
def inval(self, oid): def inval(self, oid):
p = self.cache.get(oid) pos = self.oid2ofs.pop(oid, None)
if p is None: if pos is None:
return return
pos, node = p
self.invals += 1 self.invals += 1
self.total_invals += 1 self.total_invals += 1
size, _oid = self.filemap[pos] size, _oid = self.filemap[pos]
assert oid == _oid assert oid == _oid
self.filemap[pos] = size, None self.filemap[pos] = size, None
pos, node = self.cache.pop(oid)
node.unlink()
def write(self, oid, size): def write(self, oid, size):
p = self.cache.get(oid) if oid in self.oid2ofs:
if p is None: # Delete the current record.
return pos = self.oid2ofs.pop(oid)
pos, node = p size, _oid = self.filemap[pos]
oldsize, _oid = self.filemap[pos] assert oid == _oid
assert oid == _oid self.filemap[pos] = size, None
if size == oldsize: self.add(oid, size)
return
if size < oldsize:
excess = oldsize - size
self.filemap[pos] = size, oid
self.filemap[pos + size] = excess, None
else:
self.filemap[pos] = oldsize, None
pos, node = self.cache.pop(oid)
node.unlink()
self.add(oid, size)
def report(self): def report(self):
self.check()
free = used = total = 0 free = used = total = 0
for size, oid in self.filemap.itervalues(): for size, oid in self.filemap.itervalues():
total += size total += size
...@@ -1169,6 +1061,25 @@ class CircularCacheSimulation(Simulation): ...@@ -1169,6 +1061,25 @@ class CircularCacheSimulation(Simulation):
self.total_inuse = self.inuse self.total_inuse = self.inuse
Simulation.report(self) Simulation.report(self)
def check(self):
pos = oidcount = 0
while pos < self.cachelimit:
size, oid = self.filemap[pos]
if oid:
oidcount += 1
assert self.oid2ofs[oid] == pos
pos += size
assert oidcount == len(self.oid2ofs)
assert pos == self.cachelimit
def dump(self):
print len(self.filemap)
L = list(self.filemap)
L.sort()
for k in L:
v = self.filemap[k]
print k, v[0], repr(v[1])
class BuddyCacheSimulation(LRUCacheSimulation): class BuddyCacheSimulation(LRUCacheSimulation):
def __init__(self, cachelimit): def __init__(self, cachelimit):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7