Skip to content

erp5
erp5
Commit 72891a4b authored by Jean-Paul Smets's avatar Jean-Paul Smets
Browse files
Options

provides from_table_list to optimise queries and uses where_expression 


git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@1360 20353a03-c40f-0410-a6d1-a30d3c3de9de
parent 3a892089
Show whitespace changes
Inline Side-by-side
Showing with 79 additions and 45 deletions
product/ZSQLCatalog/SQLCatalog.py
View file @ 72891a4b
...@@ -35,7 +35,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -35,7 +35,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
An Object Catalog maintains a table of object metadata, and a An Object Catalog maintains a table of object metadata, and a
series of manageable indexes to quickly search for objects series of manageable indexes to quickly search for objects
(references in the metadata) that satisfy a search query. (references in the metadata) that satisfy a search where_expression.
This class is not Zope specific, and can be used in any python This class is not Zope specific, and can be used in any python
program to build catalogs of objects. Note that it does require program to build catalogs of objects. Note that it does require
...@@ -116,6 +116,8 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -116,6 +116,8 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
""" """
Calls the show column method and returns dictionnary of Calls the show column method and returns dictionnary of
Field Ids Field Ids
XXX This should be cached
""" """
method_name = self.sql_catalog_schema method_name = self.sql_catalog_schema
keys = {} keys = {}
...@@ -132,6 +134,30 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -132,6 +134,30 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
keys.sort() keys.sort()
return keys return keys
def getColumnMap(self):
"""
Calls the show column method and returns dictionnary of
Field Ids
XXX This should be cached
"""
method_name = self.sql_catalog_schema
keys = {}
for table in self.getCatalogSearchTableIds():
try:
method = getattr(self, method_name)
search_result = method(table=table)
for c in search_result:
key = c.Field
if not keys.has_key(key): keys[c.Field] = []
keys[key].append(table)
key = '%s.%s' % (table, c.Field)
if not keys.has_key(key): keys[key] = []
keys[key].append(table) # Is this inconsistent ?
except:
pass
return keys
def getResultColumnIds(self): def getResultColumnIds(self):
""" """
Calls the show column method and returns dictionnary of Calls the show column method and returns dictionnary of
...@@ -222,7 +248,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -222,7 +248,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
'uid' is the unique Catalog identifier for this object 'uid' is the unique Catalog identifier for this object
""" """
LOG('Catalog object:',0,str(path)) #LOG('Catalog object:',0,str(path))
# Prepare the dictionnary of values # Prepare the dictionnary of values
kw = {} kw = {}
...@@ -243,7 +269,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -243,7 +269,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
if (uid != index): if (uid != index):
# Update uid attribute of object # Update uid attribute of object
uid = int(index) uid = int(index)
LOG("Write Uid",0, "uid %s index %s" % (uid, index)) #LOG("Write Uid",0, "uid %s index %s" % (uid, index))
object.uid = uid object.uid = uid
# We will check if there is an filter on this # We will check if there is an filter on this
# method, if so we may not call this zsqlMethod # method, if so we may not call this zsqlMethod
...@@ -253,7 +279,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -253,7 +279,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
if self.filter_dict.has_key(method_name): if self.filter_dict.has_key(method_name):
portal_type = object.getPortalType() portal_type = object.getPortalType()
if portal_type not in (self.filter_dict[method_name]['type']): if portal_type not in (self.filter_dict[method_name]['type']):
LOG('catalog_object',0,'XX1 this method is broken because not in types: %s' % method_name) #LOG('catalog_object',0,'XX1 this method is broken because not in types: %s' % method_name)
continue continue
else: else:
expression = self.filter_dict[method_name]['expression_instance'] expression = self.filter_dict[method_name]['expression_instance']
...@@ -263,7 +289,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -263,7 +289,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
if not result: if not result:
#LOG('catalog_object',0,'XX2 this method is broken because expression: %s' % method_name) #LOG('catalog_object',0,'XX2 this method is broken because expression: %s' % method_name)
continue continue
LOG('catalog_object',0,'this method is not broken: %s' % method_name) #LOG('catalog_object',0,'this method is not broken: %s' % method_name)
# Get the appropriate SQL Method # Get the appropriate SQL Method
# Lookup by path is required because of OFS Semantics # Lookup by path is required because of OFS Semantics
method = getattr(self, method_name) method = getattr(self, method_name)
...@@ -284,11 +310,11 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -284,11 +310,11 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
kw['path'] = path kw['path'] = path
kw['uid'] = int(index) kw['uid'] = int(index)
kw['insert_catalog_line'] = 0 kw['insert_catalog_line'] = 0
LOG("SQLCatalog Warning: insert_catalog_line, case1 value",0,0) #LOG("SQLCatalog Warning: insert_catalog_line, case1 value",0,0)
# LOG # LOG
# LOG("Call SQL Method %s with args:" % method_name,0, str(kw)) # LOG("Call SQL Method %s with args:" % method_name,0, str(kw))
# Alter row # Alter row
LOG("Call SQL Method %s with args:" % method_name,0, str(kw)) #LOG("Call SQL Method %s with args:" % method_name,0, str(kw))
method(**kw) method(**kw)
else: else:
# Get the appropriate SQL Method # Get the appropriate SQL Method
...@@ -301,22 +327,22 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -301,22 +327,22 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
if catalog_path == "reserved": if catalog_path == "reserved":
# Reserved line in catalog table # Reserved line in catalog table
insert_catalog_line = 0 insert_catalog_line = 0
LOG("SQLCatalog Warning: insert_catalog_line, case2",0,insert_catalog_line) #LOG("SQLCatalog Warning: insert_catalog_line, case2",0,insert_catalog_line)
elif catalog_path is None: elif catalog_path is None:
# No line in catalog table # No line in catalog table
insert_catalog_line = 1 insert_catalog_line = 1
LOG("SQLCatalog Warning: insert_catalog_line, case3",0,insert_catalog_line) #LOG("SQLCatalog Warning: insert_catalog_line, case3",0,insert_catalog_line)
else: else:
LOG('SQLCatalog WARNING',0,'assigning new uid to already catalogued object %s' % path) #LOG('SQLCatalog WARNING',0,'assigning new uid to already catalogued object %s' % path)
uid = 0 uid = 0
insert_catalog_line = 0 insert_catalog_line = 0
LOG("SQLCatalog Warning: insert_catalog_line, case4",0,insert_catalog_line) #LOG("SQLCatalog Warning: insert_catalog_line, case4",0,insert_catalog_line)
if not uid: if not uid:
# Generate UID # Generate UID
index = self.newUid() index = self.newUid()
object.uid = index object.uid = index
insert_catalog_line = 0 insert_catalog_line = 0
LOG("SQLCatalog Warning: insert_catalog_line, case5",0,insert_catalog_line) #LOG("SQLCatalog Warning: insert_catalog_line, case5",0,insert_catalog_line)
else: else:
index = uid index = uid
for method_name in self.sql_catalog_object: for method_name in self.sql_catalog_object:
...@@ -327,7 +353,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -327,7 +353,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
if self.filter_dict.has_key(method_name): if self.filter_dict.has_key(method_name):
portal_type = object.getPortalType() portal_type = object.getPortalType()
if portal_type not in (self.filter_dict[method_name]['type']): if portal_type not in (self.filter_dict[method_name]['type']):
LOG('catalog_object',0,'XX1 this method is broken because not in types: %s' % method_name) #LOG('catalog_object',0,'XX1 this method is broken because not in types: %s' % method_name)
continue continue
else: else:
expression = self.filter_dict[method_name]['expression_instance'] expression = self.filter_dict[method_name]['expression_instance']
...@@ -335,9 +361,9 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -335,9 +361,9 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
econtext = self.getExpressionContext(object) econtext = self.getExpressionContext(object)
result = expression(econtext) result = expression(econtext)
if not result: if not result:
LOG('catalog_object',0,'XX2 this method is broken because expression: %s' % method_name) #LOG('catalog_object',0,'XX2 this method is broken because expression: %s' % method_name)
continue continue
LOG('catalog_object',0,'this method is not broken: %s' % method_name) #LOG('catalog_object',0,'this method is not broken: %s' % method_name)
method = getattr(self, method_name) method = getattr(self, method_name)
if method.meta_type == "Z SQL Method": if method.meta_type == "Z SQL Method":
...@@ -350,7 +376,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -350,7 +376,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
value = value() value = value()
kw[arg] = value kw[arg] = value
except: except:
LOG("SQLCatalog Warning: Callable value could not be called",0,str((path, arg, method_name))) #LOG("SQLCatalog Warning: Callable value could not be called",0,str((path, arg, method_name)))
kw[arg] = None kw[arg] = None
method = aq_base(method).__of__(object.__of__(self)) # Use method in the context of object method = aq_base(method).__of__(object.__of__(self)) # Use method in the context of object
# Generate UID # Generate UID
...@@ -377,7 +403,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -377,7 +403,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
XXX Add filter of methods XXX Add filter of methods
""" """
LOG('Uncatalog object:',0,str(path)) #LOG('Uncatalog object:',0,str(path))
uid = self.getUidForPath(path) uid = self.getUidForPath(path)
methods = self.sql_uncatalog_object methods = self.sql_uncatalog_object
...@@ -487,7 +513,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -487,7 +513,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
""" """
def queryResults(self, sql_method, REQUEST=None, used=None, **kw): def queryResults(self, sql_method, REQUEST=None, used=None, **kw):
""" Builds a complex SQL query to simulate ZCalatog behaviour """ """ Builds a complex SQL where_expression to simulate ZCalatog behaviour """
""" Returns a list of brains from a set of constraints on variables """ """ Returns a list of brains from a set of constraints on variables """
# Get search arguments: # Get search arguments:
...@@ -501,28 +527,32 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -501,28 +527,32 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
if kw is None or kw == {}: if kw is None or kw == {}:
kw = REQUEST kw = REQUEST
acceptable_key_map = self.getColumnMap()
acceptable_keys = acceptable_key_map.keys()
full_text_search_keys = self.sql_catalog_full_text_search_keys
keyword_search_keys = self.sql_catalog_keyword_search_keys
# We take additional parameters from the REQUEST # We take additional parameters from the REQUEST
# and give priority to the REQUEST # and give priority to the REQUEST
if REQUEST is not None: if REQUEST is not None:
acceptable_keys = self.getColumnIds()
for key in acceptable_keys: for key in acceptable_keys:
if REQUEST.has_key(key): if REQUEST.has_key(key):
# Only copy a few keys from the REQUEST # Only copy a few keys from the REQUEST
if key in self.sql_catalog_request_keys: if key in self.sql_catalog_request_keys:
kw[key] = REQUEST[key] kw[key] = REQUEST[key]
# Let us start building the query # Let us start building the where_expression
if kw: if kw:
query = [] where_expression = []
acceptable_keys = self.getColumnIds() from_table_dict = {'catalog': 1} # Always include catalog table
full_text_search_keys = self.sql_catalog_full_text_search_keys
keyword_search_keys = self.sql_catalog_keyword_search_keys
for key, value in kw.items(): for key, value in kw.items():
if key not in ('query', 'sort-on', 'sort_on', 'sort-order', 'sort_order'): if key not in ('where_expression', 'sort-on', 'sort_on', 'sort-order', 'sort_order'):
# Make sure key belongs to schema # Make sure key belongs to schema
if key in acceptable_keys: if key in acceptable_keys:
# uid is always ambiguous so we can only change it here # uid is always ambiguous so we can only change it here
if key == 'uid': key = 'catalog.uid' if key == 'uid': key = 'catalog.uid'
# Add table to table dict
from_table_dict[acceptable_key_map[key][0]] = 1 # We use catalog by default
# Default case: variable equality # Default case: variable equality
if type(value) is type(''): if type(value) is type(''):
if value != '': if value != '':
...@@ -531,19 +561,19 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -531,19 +561,19 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
# But we consider the sign = as empty string # But we consider the sign = as empty string
value='' value=''
if '%' in value: if '%' in value:
query += ["%s LIKE '%s'" % (key, value)] where_expression += ["%s LIKE '%s'" % (key, value)]
elif value[0] == '>': elif value[0] == '>':
query += ["%s > '%s'" % (key, value[1:])] where_expression += ["%s > '%s'" % (key, value[1:])]
elif value[0] == '<': elif value[0] == '<':
query += ["%s < '%s'" % (key, value[1:])] where_expression += ["%s < '%s'" % (key, value[1:])]
elif key in keyword_search_keys: elif key in keyword_search_keys:
# We must add % in the request to simulate the catalog # We must add % in the request to simulate the catalog
query += ["%s LIKE '%%%s%%'" % (key, value)] where_expression += ["%s LIKE '%%%s%%'" % (key, value)]
elif key in full_text_search_keys: elif key in full_text_search_keys:
# We must add % in the request to simulate the catalog # We must add % in the request to simulate the catalog
query += ["MATCH %s AGAINST ('%s')" % (key, value)] where_expression += ["MATCH %s AGAINST ('%s')" % (key, value)]
else: else:
query += ["%s = '%s'" % (key, value)] where_expression += ["%s = '%s'" % (key, value)]
elif type(value) is type([]) or type(value) is type(()): elif type(value) is type([]) or type(value) is type(()):
# We have to create an OR from tuple or list # We have to create an OR from tuple or list
query_item = [] query_item = []
...@@ -565,17 +595,17 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -565,17 +595,17 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
else: else:
query_item += ["%s = '%s'" % (key, str(value_item))] query_item += ["%s = '%s'" % (key, str(value_item))]
if len(query_item) > 0: if len(query_item) > 0:
query += ['(%s)' % join(query_item, ' OR ')] where_expression += ['(%s)' % join(query_item, ' OR ')]
else: else:
query += ["%s = %s" % (key, value)] where_expression += ["%s = %s" % (key, value)]
elif key is 'query': elif key == 'where_expression':
# Not implemented yet # Not implemented yet
pass pass
if kw.has_key('query'): if kw.has_key('where_expression'):
if len(query) > 0: if len(where_expression) > 0:
kw['query'] = "(%s) AND (%s)" % (kw['query'], join(query, ' AND ') ) kw['where_expression'] = "(%s) AND (%s)" % (kw['where_expression'], join(where_expression, ' AND ') )
else: else:
kw['query'] = join(query, ' AND ') kw['where_expression'] = join(where_expression, ' AND ')
#LOG("Search Query Args:",0,str(kw)) #LOG("Search Query Args:",0,str(kw))
...@@ -624,11 +654,15 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -624,11 +654,15 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
pass pass
# Return the result # Return the result
# LOG('queryResults',0,'kw: %s' % str(kw))
return sql_method(**kw) #LOG('acceptable_keys',0,'acceptable_keys: %s' % str(acceptable_keys))
#LOG('acceptable_key_map',0,'acceptable_key_map: %s' % str(acceptable_key_map))
#LOG('queryResults',0,'kw: %s' % str(kw))
#LOG('queryResults',0,'from_table_list: %s' % str(from_table_dict.keys()))
return sql_method(from_table_list = from_table_dict.keys(), **kw)
def searchResults(self, REQUEST=None, used=None, **kw): def searchResults(self, REQUEST=None, used=None, **kw):
""" Builds a complex SQL query to simulate ZCalatog behaviour """ """ Builds a complex SQL where_expression to simulate ZCalatog behaviour """
""" Returns a list of brains from a set of constraints on variables """ """ Returns a list of brains from a set of constraints on variables """
# The used argument is deprecated and is ignored # The used argument is deprecated and is ignored
try: try:
...@@ -646,8 +680,8 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -646,8 +680,8 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
__call__ = searchResults __call__ = searchResults
def countResults(self, REQUEST=None, used=None, **kw): def countResults(self, REQUEST=None, used=None, **kw):
""" Builds a complex SQL query to simulate ZCalatog behaviour """ """ Builds a complex SQL where_expression to simulate ZCalatog behaviour """
""" Returns the number of items which satisfy the query """ """ Returns the number of items which satisfy the where_expression """
try: try:
# Get the search method # Get the search method
#LOG("countResults: scr:",0,str(self.sql_count_results)) #LOG("countResults: scr:",0,str(self.sql_count_results))
...@@ -660,7 +694,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -660,7 +694,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
kw['REQUEST'] = REQUEST kw['REQUEST'] = REQUEST
return self.queryResults(method, **kw) return self.queryResults(method, **kw)
except: except:
LOG("Warning: could not count catalog",0,str(self.sql_count_results)) LOG("Warning: could not count catalog",0,str(self.sql_count_results), error=sys.exc_info())
return [[0]] return [[0]]
class CatalogError(Exception): pass class CatalogError(Exception): pass
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7