Commit 84ae8cea authored by Hanno Schlichting's avatar Hanno Schlichting

Use some named tuples to avoid hard to understand v[2][1][0] constructs

parent 41935174
...@@ -12,6 +12,7 @@ ...@@ -12,6 +12,7 @@
############################################################################## ##############################################################################
import time import time
from collections import namedtuple
from thread import allocate_lock from thread import allocate_lock
from Acquisition import aq_base from Acquisition import aq_base
...@@ -105,6 +106,12 @@ def make_key(catalog, query): ...@@ -105,6 +106,12 @@ def make_key(catalog, query):
return key return key
Duration = namedtuple('Duration', ['start', 'end'])
IndexMeasurement = namedtuple('IndexMeasurement',
['name', 'duration', 'num'])
Benchmark = namedtuple('Benchmark', ['num', 'duration', 'hits'])
class CatalogPlan(object): class CatalogPlan(object):
"""Catalog plan class to measure and identify catalog queries and plan """Catalog plan class to measure and identify catalog queries and plan
their execution. their execution.
...@@ -153,42 +160,44 @@ class CatalogPlan(object): ...@@ -153,42 +160,44 @@ class CatalogPlan(object):
if not benchmark: if not benchmark:
return None return None
# sort indexes on (mean hits, mean search time) # sort indexes on (mean result length, mean search time)
ranking = [((v[0], v[1]), k) for k, v in benchmark.items()] ranking = [((value.num, value.duration), name)
for name, value in benchmark.items()]
ranking.sort() ranking.sort()
return [i[1] for i in ranking] return [r[1] for r in ranking]
def start(self): def start(self):
self.init_timer() self.init_timer()
self.start_time = time.time() self.start_time = time.time()
def start_split(self, label, result=None): def start_split(self, name, result=None):
self.interim[label] = (time.time(), None) self.interim[name] = Duration(time.time(), None)
def stop_split(self, name, result=None): def stop_split(self, name, result=None):
current = time.time() current = time.time()
start_time, stop_time = self.interim.get(name, (None, None)) start_time, stop_time = self.interim.get(name, Duration(None, None))
length = 0 length = 0
if result is not None: if result is not None:
# TODO: calculating the length can be expensive # TODO: calculating the length can be expensive
length = len(result) length = len(result)
self.interim[name] = (start_time, current) self.interim[name] = Duration(start_time, current)
dt = current - start_time dt = current - start_time
self.res.append((name, current - start_time, length)) self.res.append(IndexMeasurement(
name=name, duration=current - start_time, num=length))
# remember index's hits, search time and calls # remember index's hits, search time and calls
benchmark = self.benchmark() benchmark = self.benchmark()
if name not in benchmark: if name not in benchmark:
benchmark[name] = (length, dt, 1) benchmark[name] = Benchmark(num=length, duration=dt, hits=1)
else: else:
n, t, c = benchmark[name] num, duration, hits = benchmark[name]
n = int(((n*c) + length) / float(c + 1)) num = int(((num * hits) + length) / float(hits + 1))
t = ((t*c) + dt) / float(c + 1) duration = ((duration * hits) + dt) / float(hits + 1)
# reset adaption # reset adaption
if c % REFRESH_RATE == 0: if hits % REFRESH_RATE == 0:
c = 0 hits = 0
c += 1 hits += 1
benchmark[name] = (n, t, c) benchmark[name] = Benchmark(num, duration, hits)
def stop(self): def stop(self):
self.end_time = time.time() self.end_time = time.time()
...@@ -206,25 +215,24 @@ class CatalogPlan(object): ...@@ -206,25 +215,24 @@ class CatalogPlan(object):
if key not in stats: if key not in stats:
mt = self.duration mt = self.duration
c = 1 hits = 1
else: else:
mt, c = stats[key] mt, hits = stats[key]
mt = ((mt * c) + self.duration) / float(c + 1) mt = ((mt * hits) + self.duration) / float(hits + 1)
c += 1 hits += 1
stats[key] = (mt, c) stats[key] = (mt, hits)
self.log() self.log()
def result(self):
return (self.duration, tuple(self.res))
def log(self): def log(self):
# result of stopwatch # result of stopwatch
res = self.result() total = self.duration
if res[0] < self.threshold: if total < self.threshold:
return return
key = self.key key = self.key
res = (total, self.res)
reports_lock.acquire() reports_lock.acquire()
try: try:
if self.cid not in reports: if self.cid not in reports:
...@@ -233,11 +241,10 @@ class CatalogPlan(object): ...@@ -233,11 +241,10 @@ class CatalogPlan(object):
previous = reports[self.cid].get(key) previous = reports[self.cid].get(key)
if previous: if previous:
counter, mean, last = previous counter, mean, last = previous
mean = (mean * counter + res[0]) / float(counter + 1) mean = (mean * counter + total) / float(counter + 1)
reports[self.cid][key] = (counter + 1, mean, res) reports[self.cid][key] = (counter + 1, mean, res)
else: else:
reports[self.cid][key] = (1, res[0], res) reports[self.cid][key] = (1, total, res)
finally: finally:
reports_lock.release() reports_lock.release()
...@@ -264,13 +271,6 @@ class CatalogPlan(object): ...@@ -264,13 +271,6 @@ class CatalogPlan(object):
'details': <duration of single indexes>, 'details': <duration of single indexes>,
} }
<duration of single indexes> := [{'id': <index_name1>,
'duration': <duration>,
'length': <resultset length>,
},
...
]
The duration is provided in millisecond. The duration is provided in millisecond.
""" """
rval = [] rval = []
...@@ -280,9 +280,9 @@ class CatalogPlan(object): ...@@ -280,9 +280,9 @@ class CatalogPlan(object):
'counter': v[0], 'counter': v[0],
'duration': v[1] * 1000, 'duration': v[1] * 1000,
'last': {'duration': v[2][0] * 1000, 'last': {'duration': v[2][0] * 1000,
'details': [dict(id=i[0], 'details': [dict(id=i.name,
duration=i[1]*1000, duration=i.duration * 1000,
length=i[2]) length=i.num)
for i in v[2][1]], for i in v[2][1]],
}, },
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment