Skip to content

Z
Zope
Commit 4938ef3f authored by Hanno Schlichting's avatar Hanno Schlichting
Browse files
Options

Actually expose the query plan TTW

parent bb5b83eb
Hide whitespace changes
Inline Side-by-side
Showing with 177 additions and 133 deletions
src/Products/ZCatalog/ZCatalog.py
View file @ 4938ef3f
......@@ -48,6 +48,7 @@ from Products.ZCatalog.Catalog import Catalog, CatalogError
from Products.ZCatalog.interfaces import IZCatalog
from Products.ZCatalog.ProgressHandler import ZLogHandler
from Products.ZCatalog.ZCatalogIndexes import ZCatalogIndexes
from .plan import PriorityMap
LOG = logging.getLogger('Zope.ZCatalog')
......@@ -103,7 +104,8 @@ class ZCatalog(Folder, Persistent, Implicit):
{'label': 'Metadata', 'action': 'manage_catalogSchema'},
{'label': 'Find Objects', 'action': 'manage_catalogFind'},
{'label': 'Advanced', 'action': 'manage_catalogAdvanced'},
{'label': 'Query Report', 'action': 'manage_catalogPlan'},
{'label': 'Query Report', 'action': 'manage_catalogReport'},
{'label': 'Query Plan', 'action': 'manage_catalogPlan'},
{'label': 'Undo', 'action': 'manage_UndoForm'},
{'label': 'Security', 'action': 'manage_access'},
{'label': 'Ownership', 'action': 'manage_owner'},
......@@ -127,6 +129,9 @@ class ZCatalog(Folder, Persistent, Implicit):
'manage_catalogAdvanced')
manage_catalogAdvanced = DTMLFile('dtml/catalogAdvanced', globals())
security.declareProtected(manage_zcatalog_entries, 'manage_catalogReport')
manage_catalogReport = DTMLFile('dtml/catalogReport', globals())
security.declareProtected(manage_zcatalog_entries, 'manage_catalogPlan')
manage_catalogPlan = DTMLFile('dtml/catalogPlan', globals())
......@@ -877,24 +882,39 @@ class ZCatalog(Folder, Persistent, Implicit):
security.declareProtected(manage_zcatalog_entries, 'getCatalogPlan')
def getCatalogPlan(self):
"""Query time reporting and planning."""
"""Get a string representation of a query plan"""
plan = PriorityMap.get_plan()
output = []
output.append('queryplan = {')
for querykey, details in sorted(plan.items()):
output.append(' %s: {' % repr(querykey))
for indexname, benchmark in sorted(details.items()):
tuplebench = repr(tuple(benchmark))
output.append(' %r:\n %s,' % (indexname, tuplebench))
output.append(' },')
output.append('}')
return '\n'.join(output)
security.declareProtected(manage_zcatalog_entries, 'getCatalogReport')
def getCatalogReport(self):
"""Query time reporting."""
rval = self._catalog.getCatalogPlan().report()
rval.sort(key=operator.itemgetter('duration'), reverse=True)
return rval
security.declareProtected(manage_zcatalog_entries,
'manage_resetCatalogPlan')
def manage_resetCatalogPlan(self, REQUEST=None):
"""Resets the catalog plan."""
'manage_resetCatalogReport')
def manage_resetCatalogReport(self, REQUEST=None):
"""Resets the catalog report."""
self._catalog.getCatalogPlan().reset()
if REQUEST is not None:
REQUEST.response.redirect(REQUEST.URL1 +
'/manage_catalogPlan?manage_tabs_message=Plan%20cleared')
'/manage_catalogReport?manage_tabs_message=Report%20cleared')
security.declareProtected(manage_zcatalog_entries,
'manage_editCatalogPlan')
def manage_editCatalogPlan(self, long_query_time=0.1, REQUEST=None):
'manage_editCatalogReport')
def manage_editCatalogReport(self, long_query_time=0.1, REQUEST=None):
"""Edit the long query time."""
if not isinstance(long_query_time, float):
long_query_time = float(long_query_time)
......@@ -902,7 +922,7 @@ class ZCatalog(Folder, Persistent, Implicit):
if REQUEST is not None:
REQUEST.response.redirect(REQUEST.URL1 +
'/manage_catalogPlan?manage_tabs_message=' +
'/manage_catalogReport?manage_tabs_message=' +
'Long%20query%20time%20changed')
InitializeClass(ZCatalog)
......
src/Products/ZCatalog/__init__.py
View file @ 4938ef3f
......@@ -17,6 +17,10 @@ import ZCatalog
def initialize(context):
# Load a default map
from Products.ZCatalog.plan import PriorityMap
PriorityMap.load_default()
context.registerClass(
ZCatalog.ZCatalog,
permission='Add ZCatalogs',
......
src/Products/ZCatalog/dtml/catalogPlan.dtml
View file @ 4938ef3f
......@@ -2,117 +2,12 @@
<dtml-var manage_tabs>
<p class="form-help">
The <strong>query report</strong> shows catalog queries that
perform slowly.
The <strong>query plan</strong> shows the actual query plan of the
current process.
</p>
<table width="100%" cellspacing="0" cellpadding="2" border="0">
<tr class="list-header" >
<td align="left" valign="top">
<div class="list-nav">
Mean duration&nbsp;[ms]
</div>
</td>
<td align="left" valign="top">
<div class="list-nav">
Hits
</div>
</td>
<td align="left" valign="top">
<div class="list-nav">
Query key
</div>
</td>
<td align="left" valign="top">
<div class="list-nav">
Recent
</div>
</td>
</tr>
<dtml-if getCatalogPlan>
<dtml-in getCatalogPlan mapping>
<dtml-if sequence-odd>
<tr class="row-normal">
<dtml-else>
<tr class="row-hilite">
</dtml-if>
<td align="left" valign="top">
<div class="list-item">
<dtml-var expr="'%3.2f' % duration">
</div>
</td>
<td align="left" valign="top">
<div class="list-item">
&dtml-counter;
</div>
</td>
<td align="left" valign="top">
<div class="list-item">
&dtml-query;
</div>
</td>
<td align="left" valign="top">
<div class="list-item">
<dtml-var expr="'%3.2f' % last['duration']">ms
[<dtml-in expr="last['details']" sort mapping>
&dtml-id;:
<dtml-var expr="'%3.2f' % duration">ms /
&dtml-length; objects,
</dtml-in>]
</div>
</td>
</tr>
</dtml-in>
<tr>
<td colspan="2" align="left" valign="top">
<p class="form-help">Resetting the catalog report will reinitialize the report log.</p>
</td>
<td colspan="2" align="right" valign="top">
<form action="manage_resetCatalogPlan" method=POST>
<div class="form-element">
<input class="form-element" type="submit" value="Reset Report">
</div>
</form>
</td>
</tr>
<dtml-else>
<tr>
<td colspan="4" >
<div class="list-item">
Report is empty.
</div>
</td>
</tr>
</dtml-if>
</table>
<form action="manage_editCatalogPlan" method="post">
<table width="100%" style="padding-top:1em;" cellspacing="0" cellpadding="2" border="0">
<tr class="section-bar">
<td colspan="3" align="left">
<div class="form-label">
Settings
</div>
</td>
</tr>
<tr>
<td align="right" valign="middle">
<div class="list-item">
Threshold (in seconds)
</div>
</td>
<td align="left" valign="middle">
<div class="form-element">
<input name="long_query_time:float" value="&dtml-long_query_time;" />
</div>
</td>
<td align="left" valign="middle">
<p class="form-help">Only queries whose execution
takes longer than the configured threshold are considered
being slow. (Default 0.1 seconds).</p>
</tr>
</table>
<input class="form-element" type="submit" value="Apply settings">
</form>
<textarea name="queryplan" cols="70" rows="25" readonly="readonly">
&dtml-getCatalogPlan;
</textarea>
<dtml-var manage_page_footer>
src/Products/ZCatalog/dtml/catalogReport.dtml 0 → 100644
View file @ 4938ef3f
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<p class="form-help">
The <strong>query report</strong> shows catalog queries that
perform slowly.
</p>
<table width="100%" cellspacing="0" cellpadding="2" border="0">
<tr class="list-header" >
<td align="left" valign="top">
<div class="list-nav">
Mean duration&nbsp;[ms]
</div>
</td>
<td align="left" valign="top">
<div class="list-nav">
Hits
</div>
</td>
<td align="left" valign="top">
<div class="list-nav">
Query key
</div>
</td>
<td align="left" valign="top">
<div class="list-nav">
Recent
</div>
</td>
</tr>
<dtml-if getCatalogReport>
<dtml-in getCatalogReport mapping>
<dtml-if sequence-odd>
<tr class="row-normal">
<dtml-else>
<tr class="row-hilite">
</dtml-if>
<td align="left" valign="top">
<div class="list-item">
<dtml-var expr="'%3.2f' % duration">
</div>
</td>
<td align="left" valign="top">
<div class="list-item">
&dtml-counter;
</div>
</td>
<td align="left" valign="top">
<div class="list-item">
&dtml-query;
</div>
</td>
<td align="left" valign="top">
<div class="list-item">
<dtml-var expr="'%3.2f' % last['duration']">ms
[<dtml-in expr="last['details']" sort mapping>
&dtml-id;:
<dtml-var expr="'%3.2f' % duration">ms /
&dtml-length; objects,
</dtml-in>]
</div>
</td>
</tr>
</dtml-in>
<tr>
<td colspan="2" align="left" valign="top">
<p class="form-help">Resetting the catalog report will reinitialize the report log.</p>
</td>
<td colspan="2" align="right" valign="top">
<form action="manage_resetCatalogReport" method=POST>
<div class="form-element">
<input class="form-element" type="submit" value="Reset Report">
</div>
</form>
</td>
</tr>
<dtml-else>
<tr>
<td colspan="4" >
<div class="list-item">
Report is empty.
</div>
</td>
</tr>
</dtml-if>
</table>
<form action="manage_editCatalogReport" method="post">
<table width="100%" style="padding-top:1em;" cellspacing="0" cellpadding="2" border="0">
<tr class="section-bar">
<td colspan="3" align="left">
<div class="form-label">
Settings
</div>
</td>
</tr>
<tr>
<td align="right" valign="middle">
<div class="list-item">
Threshold (in seconds)
</div>
</td>
<td align="left" valign="middle">
<div class="form-element">
<input name="long_query_time:float" value="&dtml-long_query_time;" />
</div>
</td>
<td align="left" valign="middle">
<p class="form-help">Only queries whose execution
takes longer than the configured threshold are considered
being slow. (Default 0.1 seconds).</p>
</tr>
</table>
<input class="form-element" type="submit" value="Apply settings">
</form>
<dtml-var manage_page_footer>
src/Products/ZCatalog/plan.py
View file @ 4938ef3f
......@@ -41,6 +41,10 @@ class PriorityMap(object):
lock = allocate_lock()
value = {}
@classmethod
def get_plan(cls):
return cls.value.copy()
@classmethod
def get(cls, key):
return cls.value.get(key, None)
......@@ -63,14 +67,17 @@ class PriorityMap(object):
pmap = resolve(location)
logger.info('loaded priority %d map(s) from %s',
len(pmap), location)
# Convert simple benchmark tuples to namedtuples
new_plan = {}
for querykey, details in pmap.items():
new_plan[querykey] = {}
for indexname, benchmark in details.items():
new_plan[querykey][indexname] = Benchmark(*benchmark)
with cls.lock:
cls.value = pmap.copy()
cls.value = new_plan
except ImportError:
logger.warning('could not load priority map from %s', location)
# Load a default map
PriorityMap.load_default()
class Reports(object):
"""This holds a structure of nested dicts.
......
src/Products/ZCatalog/tests/test_plan.py
View file @ 4938ef3f
......@@ -46,7 +46,7 @@ class TestCatalogPlan(unittest.TestCase):
def test_ReportLength(self):
""" tests the report aggregation """
self.zcat.manage_resetCatalogPlan()
self.zcat.manage_resetCatalogReport()
self.zcat.searchResults(numbers=4, sort_on='num')
self.zcat.searchResults(numbers=1, sort_on='num')
......@@ -58,49 +58,49 @@ class TestCatalogPlan(unittest.TestCase):
self.zcat.searchResults(num=[5, 4, 3], sort_on='num')
self.zcat.searchResults(num=(3, 4, 5), sort_on='num')
self.assertEqual(4, len(self.zcat.getCatalogPlan()))
self.assertEqual(4, len(self.zcat.getCatalogReport()))
def test_ReportCounter(self):
""" tests the counter of equal queries """
self.zcat.manage_resetCatalogPlan()
self.zcat.manage_resetCatalogReport()
self.zcat.searchResults(numbers=5, sort_on='num')
self.zcat.searchResults(numbers=6, sort_on='num')
self.zcat.searchResults(numbers=8, sort_on='num')
r = self.zcat.getCatalogPlan()[0]
r = self.zcat.getCatalogReport()[0]
self.assertEqual(r['counter'], 3)
def test_ReportKey(self):
""" tests the query keys for uniqueness """
# query key 1
key = ('sort_on', ('big', 'True'))
self.zcat.manage_resetCatalogPlan()
self.zcat.manage_resetCatalogReport()
self.zcat.searchResults(big=True, sort_on='num')
self.zcat.searchResults(big=True, sort_on='num')
r = self.zcat.getCatalogPlan()[0]
r = self.zcat.getCatalogReport()[0]
self.assertEqual(r['query'], key)
self.assertEqual(r['counter'], 2)
# query key 2
key = ('sort_on', ('big', 'False'))
self.zcat.manage_resetCatalogPlan()
self.zcat.manage_resetCatalogReport()
self.zcat.searchResults(big=False, sort_on='num')
r = self.zcat.getCatalogPlan()[0]
r = self.zcat.getCatalogReport()[0]
self.assertEqual(r['query'], key)
self.assertEqual(r['counter'], 1)
# query key 3
key = ('sort_on', ('num', '[3, 4, 5]'))
self.zcat.manage_resetCatalogPlan()
self.zcat.manage_resetCatalogReport()
self.zcat.searchResults(num=[5, 4, 3], sort_on='num')
self.zcat.searchResults(num=(3, 4, 5), sort_on='num')
r = self.zcat.getCatalogPlan()[0]
r = self.zcat.getCatalogReport()[0]
self.assertEqual(r['query'], key)
self.assertEqual(r['counter'], 2)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7