Skip to content

Z
ZODB
Commit e9ed8b1f authored by Jim Fulton's avatar Jim Fulton
Browse files
Options

Updated simulation to match cache improvements.

parent 022a5d45
Hide whitespace changes
Inline Side-by-side
Showing with 99 additions and 73 deletions
src/ZEO/scripts/cache_simul.py
View file @ e9ed8b1f
...@@ -19,6 +19,7 @@ Usage: simul.py [-s size] tracefile ...@@ -19,6 +19,7 @@ Usage: simul.py [-s size] tracefile
Options: Options:
-s size: cache size in MB (default 20 MB) -s size: cache size in MB (default 20 MB)
-i: summarizing interval in minutes (default 15; max 60) -i: summarizing interval in minutes (default 15; max 60)
-r: rearrange factor
Note: Note:
...@@ -54,18 +55,22 @@ def main(args=None): ...@@ -54,18 +55,22 @@ def main(args=None):
# Parse options. # Parse options.
MB = 1<<20 MB = 1<<20
cachelimit = 20*MB cachelimit = 20*MB
rearrange = 0.8
simclass = CircularCacheSimulation simclass = CircularCacheSimulation
interval_step = 15 interval_step = 15
try: try:
opts, args = getopt.getopt(args, "s:i:") opts, args = getopt.getopt(args, "s:i:r:")
except getopt.error, msg: except getopt.error, msg:
usage(msg) usage(msg)
return 2 return 2
for o, a in opts: for o, a in opts:
if o == '-s': if o == '-s':
cachelimit = int(float(a)*MB) cachelimit = int(float(a)*MB)
elif o == '-i': elif o == '-i':
interval_step = int(a) interval_step = int(a)
elif o == '-r':
rearrange = float(a)
else: else:
assert False, (o, a) assert False, (o, a)
...@@ -105,8 +110,8 @@ def main(args=None): ...@@ -105,8 +110,8 @@ def main(args=None):
return 1 return 1
# Create simulation object. # Create simulation object.
sim = simclass(cachelimit) sim = simclass(cachelimit, rearrange)
interval_sim = simclass(cachelimit) interval_sim = simclass(cachelimit, rearrange)
# Print output header. # Print output header.
sim.printheader() sim.printheader()
...@@ -143,6 +148,8 @@ def main(args=None): ...@@ -143,6 +148,8 @@ def main(args=None):
if last_interval is not None: if last_interval is not None:
interval_sim.report() interval_sim.report()
interval_sim.restart() interval_sim.restart()
if not interval_sim.warm:
sim.restart()
last_interval = this_interval last_interval = this_interval
sim.event(ts, dlen, version, code, oid, start_tid, end_tid) sim.event(ts, dlen, version, code, oid, start_tid, end_tid)
interval_sim.event(ts, dlen, version, code, oid, start_tid, end_tid) interval_sim.event(ts, dlen, version, code, oid, start_tid, end_tid)
...@@ -162,10 +169,12 @@ class Simulation(object): ...@@ -162,10 +169,12 @@ class Simulation(object):
finish() method also calls report(). finish() method also calls report().
""" """
def __init__(self, cachelimit): def __init__(self, cachelimit, rearrange):
self.cachelimit = cachelimit self.cachelimit = cachelimit
self.rearrange = rearrange
# Initialize global statistics. # Initialize global statistics.
self.epoch = None self.epoch = None
self.warm = False
self.total_loads = 0 self.total_loads = 0
self.total_hits = 0 # subclass must increment self.total_hits = 0 # subclass must increment
self.total_invals = 0 # subclass must increment self.total_invals = 0 # subclass must increment
...@@ -286,20 +295,19 @@ class Simulation(object): ...@@ -286,20 +295,19 @@ class Simulation(object):
# For use in CircularCacheSimulation. # For use in CircularCacheSimulation.
class CircularCacheEntry(object): class CircularCacheEntry(object):
__slots__ = (# object key: an (oid, start_tid) pair, where __slots__ = (
# start_tid is the tid of the transaction that created # object key: an (oid, start_tid) pair, where start_tid is the
# this revision of oid # tid of the transaction that created this revision of oid
'key', 'key',
# tid of transaction that created the next revision; # tid of transaction that created the next revision; z64 iff
# z64 iff this is the current revision # this is the current revision
'end_tid', 'end_tid',
# Offset from start of file to the object's data # Offset from start of file to the object's data record; this
# record; this includes all overhead bytes (status # includes all overhead bytes (status byte, size bytes, etc).
# byte, size bytes, etc). 'offset',
'offset', )
)
def __init__(self, key, end_tid, offset): def __init__(self, key, end_tid, offset):
self.key = key self.key = key
...@@ -318,10 +326,12 @@ class CircularCacheSimulation(Simulation): ...@@ -318,10 +326,12 @@ class CircularCacheSimulation(Simulation):
extras = "evicts", "inuse" extras = "evicts", "inuse"
def __init__(self, cachelimit): evicts = 0
def __init__(self, cachelimit, rearrange):
from ZEO import cache from ZEO import cache
Simulation.__init__(self, cachelimit) Simulation.__init__(self, cachelimit, rearrange)
self.total_evicts = 0 # number of cache evictions self.total_evicts = 0 # number of cache evictions
# Current offset in file. # Current offset in file.
...@@ -350,6 +360,8 @@ class CircularCacheSimulation(Simulation): ...@@ -350,6 +360,8 @@ class CircularCacheSimulation(Simulation):
def restart(self): def restart(self):
Simulation.restart(self) Simulation.restart(self)
if self.evicts:
self.warm = True
self.evicts = 0 self.evicts = 0
self.evicted_hit = self.evicted_miss = 0 self.evicted_hit = self.evicted_miss = 0
...@@ -360,6 +372,20 @@ class CircularCacheSimulation(Simulation): ...@@ -360,6 +372,20 @@ class CircularCacheSimulation(Simulation):
if oid in self.current: # else it's a cache miss if oid in self.current: # else it's a cache miss
self.hits += 1 self.hits += 1
self.total_hits += 1 self.total_hits += 1
tid = self.current[oid]
entry = self.key2entry[(oid, tid)]
offset_offset = self.offset - entry.offset
if offset_offset < 0:
offset_offset += self.cachelimit
assert offset_offset >= 0
if offset_offset > self.rearrange * self.cachelimit:
# we haven't accessed it in a while. Move it forward
size = self.filemap[entry.offset][0]
self._remove(*entry.key)
self.add(oid, size, tid)
elif oid in self.evicted: elif oid in self.evicted:
size, e = self.evicted[oid] size, e = self.evicted[oid]
self.write(oid, size, e.key[1], z64, 1) self.write(oid, size, e.key[1], z64, 1)
......
src/ZEO/tests/test_cache.py
View file @ e9ed8b1f
...@@ -871,39 +871,39 @@ Check to make sure the cache analysis scripts work. ...@@ -871,39 +871,39 @@ Check to make sure the cache analysis scripts work.
Jul 11 12:30 4:59 272 60 8 240 22.1% 0 67.1 Jul 11 12:30 4:59 272 60 8 240 22.1% 0 67.1
Jul 11 12:35 4:59 273 68 6 232 24.9% 0 79.8 Jul 11 12:35 4:59 273 68 6 232 24.9% 0 79.8
Jul 11 12:40 4:59 273 85 8 215 31.1% 0 91.4 Jul 11 12:40 4:59 273 85 8 215 31.1% 0 91.4
Jul 11 12:45 4:59 273 85 7 215 31.1% 61 100.0 Jul 11 12:45 4:59 273 84 6 216 30.8% 77 99.1
Jul 11 12:50 4:59 272 105 9 195 38.6% 193 100.0 Jul 11 12:50 4:59 272 104 9 196 38.2% 196 98.9
Jul 11 12:55 4:59 273 107 4 193 39.2% 184 100.0 Jul 11 12:55 4:59 273 104 4 196 38.1% 188 99.1
Jul 11 13:00 4:59 273 93 12 207 34.1% 215 99.9 Jul 11 13:00 4:59 273 92 12 208 33.7% 213 99.3
Jul 11 13:05 4:59 273 103 7 197 37.7% 191 99.9 Jul 11 13:05 4:59 273 103 8 197 37.7% 190 99.0
Jul 11 13:10 4:59 272 101 16 199 37.1% 191 99.9 Jul 11 13:10 4:59 272 100 16 200 36.8% 203 99.2
Jul 11 13:15 4:59 273 93 9 207 34.1% 227 99.9 Jul 11 13:15 4:59 273 91 11 209 33.3% 222 98.7
Jul 11 13:20 4:59 273 94 9 206 34.4% 211 99.9 Jul 11 13:20 4:59 273 96 9 204 35.2% 210 99.2
Jul 11 13:25 4:59 272 93 11 207 34.2% 208 99.9 Jul 11 13:25 4:59 272 89 11 211 32.7% 212 99.1
Jul 11 13:30 4:59 273 84 14 216 30.8% 221 99.9 Jul 11 13:30 4:59 273 82 14 218 30.0% 220 99.1
Jul 11 13:35 4:59 273 102 6 198 37.4% 201 99.9 Jul 11 13:35 4:59 273 101 9 199 37.0% 191 99.5
Jul 11 13:40 4:59 273 88 6 212 32.2% 215 100.0 Jul 11 13:40 4:59 273 92 6 208 33.7% 214 99.4
Jul 11 13:45 4:59 272 81 5 219 29.8% 209 100.0 Jul 11 13:45 4:59 272 80 6 220 29.4% 217 99.3
Jul 11 13:50 4:59 273 87 8 213 31.9% 223 100.0 Jul 11 13:50 4:59 273 81 8 219 29.7% 214 99.2
Jul 11 13:55 4:59 273 86 10 214 31.5% 196 99.9 Jul 11 13:55 4:59 273 86 11 214 31.5% 208 98.8
Jul 11 14:00 4:59 273 96 12 204 35.2% 195 100.0 Jul 11 14:00 4:59 273 95 11 205 34.8% 188 99.3
Jul 11 14:05 4:59 272 95 11 205 34.9% 201 99.9 Jul 11 14:05 4:59 272 93 10 207 34.2% 207 99.3
Jul 11 14:10 4:59 273 110 9 190 40.3% 189 99.9 Jul 11 14:10 4:59 273 110 6 190 40.3% 198 98.8
Jul 11 14:15 4:59 273 94 9 206 34.4% 210 99.9 Jul 11 14:15 4:59 273 91 9 209 33.3% 209 99.1
Jul 11 14:20 4:59 272 87 17 213 32.0% 207 100.0 Jul 11 14:20 4:59 272 85 16 215 31.2% 210 99.3
Jul 11 14:25 4:59 273 91 10 209 33.3% 214 99.9 Jul 11 14:25 4:59 273 89 8 211 32.6% 226 99.3
Jul 11 14:30 4:59 273 106 13 194 38.8% 210 100.0 Jul 11 14:30 4:59 273 96 12 204 35.2% 214 99.3
Jul 11 14:35 4:59 273 90 13 210 33.0% 206 100.0 Jul 11 14:35 4:59 273 90 10 210 33.0% 213 99.3
Jul 11 14:40 4:59 272 113 11 187 41.5% 180 100.0 Jul 11 14:40 4:59 272 106 10 194 39.0% 196 98.8
Jul 11 14:45 4:59 273 83 10 217 30.4% 230 99.9 Jul 11 14:45 4:59 273 80 8 220 29.3% 230 99.0
Jul 11 14:50 4:59 273 101 9 199 37.0% 204 100.0 Jul 11 14:50 4:59 273 99 8 201 36.3% 202 99.0
Jul 11 14:55 4:59 273 87 7 213 31.9% 223 100.0 Jul 11 14:55 4:59 273 87 8 213 31.9% 205 99.4
Jul 11 15:00 4:59 272 99 6 201 36.4% 192 99.9 Jul 11 15:00 4:59 272 98 8 202 36.0% 211 99.3
Jul 11 15:05 4:59 273 96 12 204 35.2% 203 99.9 Jul 11 15:05 4:59 273 93 11 207 34.1% 198 99.2
Jul 11 15:10 4:59 273 98 12 202 35.9% 182 99.9 Jul 11 15:10 4:59 273 96 11 204 35.2% 184 99.2
Jul 11 15:15 1 2 1 0 1 50.0% 1 99.9 Jul 11 15:15 1 2 1 0 1 50.0% 1 99.2
-------------------------------------------------------------------------- --------------------------------------------------------------------------
Jul 11 12:11 3:03:19 10000 3170 327 7830 31.7% 5993 99.9 Jul 11 12:45 2:30:01 8184 2794 286 6208 34.1% 6067 99.2
>>> cache_run('cache4', 4) >>> cache_run('cache4', 4)
...@@ -951,16 +951,16 @@ Check to make sure the cache analysis scripts work. ...@@ -951,16 +951,16 @@ Check to make sure the cache analysis scripts work.
Jul 11 12:45 14:59 818 322 23 578 39.4% 0 61.4 Jul 11 12:45 14:59 818 322 23 578 39.4% 0 61.4
Jul 11 13:00 14:59 818 381 43 519 46.6% 0 75.8 Jul 11 13:00 14:59 818 381 43 519 46.6% 0 75.8
Jul 11 13:15 14:59 818 450 44 450 55.0% 0 88.2 Jul 11 13:15 14:59 818 450 44 450 55.0% 0 88.2
Jul 11 13:30 14:59 819 503 47 397 61.4% 0 98.9 Jul 11 13:30 14:59 819 503 47 397 61.4% 36 98.2
Jul 11 13:45 14:59 818 501 51 399 61.2% 368 100.0 Jul 11 13:45 14:59 818 496 49 404 60.6% 388 98.5
Jul 11 14:00 14:59 818 523 50 377 63.9% 372 100.0 Jul 11 14:00 14:59 818 515 48 385 63.0% 376 98.3
Jul 11 14:15 14:59 818 528 61 372 64.5% 361 100.0 Jul 11 14:15 14:59 818 529 58 371 64.7% 391 98.1
Jul 11 14:30 14:59 818 507 49 393 62.0% 404 100.0 Jul 11 14:30 14:59 818 511 51 389 62.5% 376 98.5
Jul 11 14:45 14:59 819 523 51 377 63.9% 387 100.0 Jul 11 14:45 14:59 819 529 53 371 64.6% 410 97.9
Jul 11 15:00 14:59 818 503 52 397 61.5% 384 100.0 Jul 11 15:00 14:59 818 512 49 388 62.6% 379 97.7
Jul 11 15:15 1 2 2 0 0 100.0% 0 100.0 Jul 11 15:15 1 2 2 0 0 100.0% 0 97.7
-------------------------------------------------------------------------- --------------------------------------------------------------------------
Jul 11 12:11 3:03:19 10000 5064 504 5936 50.6% 2276 100.0 Jul 11 13:30 1:45:01 5730 3597 355 2705 62.8% 2356 97.7
>>> cache_run('cache1', 1) >>> cache_run('cache1', 1)
...@@ -1003,21 +1003,21 @@ Check to make sure the cache analysis scripts work. ...@@ -1003,21 +1003,21 @@ Check to make sure the cache analysis scripts work.
CircularCacheSimulation, cache size 1,048,576 bytes CircularCacheSimulation, cache size 1,048,576 bytes
START TIME DUR. LOADS HITS INVALS WRITES HITRATE EVICTS INUSE START TIME DUR. LOADS HITS INVALS WRITES HITRATE EVICTS INUSE
Jul 11 12:11 3:17 180 1 2 197 0.6% 0 21.5 Jul 11 12:11 3:17 180 1 2 197 0.6% 0 21.5
Jul 11 12:15 14:59 818 107 9 793 13.1% 94 99.9 Jul 11 12:15 14:59 818 107 9 793 13.1% 96 99.6
Jul 11 12:30 14:59 818 159 16 741 19.4% 722 99.9 Jul 11 12:30 14:59 818 160 16 740 19.6% 724 99.6
Jul 11 12:45 14:59 818 156 8 744 19.1% 743 99.9 Jul 11 12:45 14:59 818 158 8 742 19.3% 741 99.2
Jul 11 13:00 14:59 818 141 21 759 17.2% 767 99.9 Jul 11 13:00 14:59 818 140 21 760 17.1% 771 99.5
Jul 11 13:15 14:59 818 126 16 774 15.4% 786 99.9 Jul 11 13:15 14:59 818 125 17 775 15.3% 781 99.6
Jul 11 13:30 14:59 819 148 12 752 18.1% 736 100.0 Jul 11 13:30 14:59 819 147 13 753 17.9% 748 99.5
Jul 11 13:45 14:59 818 123 17 777 15.0% 764 99.9 Jul 11 13:45 14:59 818 120 17 780 14.7% 763 99.5
Jul 11 14:00 14:59 818 158 18 742 19.3% 739 100.0 Jul 11 14:00 14:59 818 159 17 741 19.4% 728 99.4
Jul 11 14:15 14:59 818 148 14 752 18.1% 776 99.9 Jul 11 14:15 14:59 818 141 13 759 17.2% 787 99.6
Jul 11 14:30 14:59 818 156 15 744 19.1% 745 99.9 Jul 11 14:30 14:59 818 150 15 750 18.3% 755 99.2
Jul 11 14:45 14:59 819 142 15 758 17.3% 749 99.9 Jul 11 14:45 14:59 819 132 13 768 16.1% 771 99.5
Jul 11 15:00 14:59 818 150 10 750 18.3% 740 99.8 Jul 11 15:00 14:59 818 154 10 746 18.8% 723 99.2
Jul 11 15:15 1 2 1 0 1 50.0% 0 99.9 Jul 11 15:15 1 2 1 0 1 50.0% 0 99.3
-------------------------------------------------------------------------- --------------------------------------------------------------------------
Jul 11 12:11 3:03:19 10000 1716 173 9284 17.2% 8361 99.9 Jul 11 12:15 3:00:01 9820 1694 169 9108 17.3% 8388 99.3
Cleanup: Cleanup:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7