diff --git a/product/ZSQLCatalog/ColumnMap.py b/product/ZSQLCatalog/ColumnMap.py
new file mode 100644
index 0000000000000000000000000000000000000000..dd8fe52a41ded3d42139374b1fe47c57e8626c5b
--- /dev/null
+++ b/product/ZSQLCatalog/ColumnMap.py
@@ -0,0 +1,502 @@
+##############################################################################
+#
+# Copyright (c) 2008-2009 Nexedi SARL and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+import sys
+from zLOG import LOG
+from Interface.IColumnMap import IColumnMap
+from Interface.Verify import verifyClass
+from SQLCatalog import profiler_decorator
+
+DEFAULT_GROUP_ID = None
+
+TRACE = False
+
+# TODO: handle left joins
+# TODO: handle straight joins
+# TODO: make it possible to do: query=ComplexQuery(Query(source_title='foo'), Query(source_title='bar')), sort_on=[('source_title_1', )]
+# currently, it's not possible because related_key_dict is indexed by related key name, which makes 'source_title_1' lookup fail. It should be indexed by group (probably).
+# TODO: rename all "related_key" references into "virtual_column"
+
+class ColumnMap(object):
+
+ __implements__ = IColumnMap
+
+ @profiler_decorator
+ def __init__(self, catalog_table_name=None):
+ self.catalog_table_name = catalog_table_name
+ # Key: group
+ # Value: set of column names
+ self.registry = {}
+ # Key: group
+ # Value: dict
+ # Key: column name
+ # Value: set of SimpleQuery
+ self.simple_query_dict = {}
+ # Key: (group, column name)
+ # Value: table name
+ self.column_map = {}
+ # Key: (group, table name)
+ # Value: table alias
+ self.table_alias_dict = {}
+ # Key: related key name
+ # Value: (group, column name)
+ self.related_key_dict = {}
+ # Key: related_column
+ # Value: last used alias order
+ self.related_key_order_dict = {}
+ # Key: group
+ # Value: relate_key
+ self.related_group_dict = {}
+ # Key: table alias
+ # Value: table name
+ self.table_map = {}
+ # Key: raw column
+ # Value: (function, column)
+ self.raw_column_dict = {}
+ # Entries: column name
+ self.column_ignore_set = set()
+ self.join_table_set = set()
+ self.straight_join_table_list = []
+ self.left_join_table_list = []
+ self.join_query_list = []
+
+ @profiler_decorator
+ def registerColumn(self, raw_column, group=DEFAULT_GROUP_ID, simple_query=None):
+ assert ' as ' not in raw_column.lower()
+ # Sanitize input: extract column from raw column (might contain COUNT, ...).
+ if '(' in raw_column:
+ function, column = raw_column.split('(')
+ column = column.strip()
+ assert column[-1] == ')'
+ column = column[:-1].strip()
+ else:
+ function = None
+ column = raw_column
+ # Remove '`' from column.
+ column = column.replace('`', '')
+ # Extract table name from column, if any.
+ if '.' in column:
+ # Assuming the part before the dot is a real table name, not an alias.
+ table, column = column.split('.')
+ LOG('ColumnMap', 100, 'Column %r is pre-mapped. This use is strongly discouraged.' % (raw_column, ))
+ else:
+ table = None
+
+ self.raw_column_dict[raw_column] = (function, column)
+ self.registry.setdefault(group, set()).add(column)
+ self.simple_query_dict.setdefault(group, {}).setdefault(column, set()).add(simple_query)
+ if table is not None:
+ # Register table alias and mark column as resolved.
+ self.registerTable(table, alias=table, group=group)
+ self.resolveColumn(column, table, group=group)
+ if group is DEFAULT_GROUP_ID and table != self.catalog_table_name:
+ # When a column is registered in default group and is explicitely
+ # mapped to a table, we must mark its table as requiring a join with
+ # catalog table (unless it's the catalog table, of course).
+ self._addJoinTable(table, group)
+
+ def ignoreColumn(self, column):
+ self.column_ignore_set.add(column)
+
+ @profiler_decorator
+ def registerRelatedKey(self, related_column, column):
+ # XXX: should we store the group, or directly the table on which the column is mapped ?
+ # The former avoids duplicating data, but requires one more lookup (group + column -> table)
+ # The latter makes it harder (?) to split the mapping in multiple queries (if splitting by groups turns out to be a good idea)
+ real_related_column = related_column
+ order = self.related_key_order_dict.get(real_related_column, 0) + 1
+ related_column = '%s_%s' % (related_column, order)
+ group = 'related_%s' % (related_column, )
+ assert group not in self.registry
+ assert group not in self.related_group_dict
+ self.related_key_order_dict[real_related_column] = order
+ self.related_key_dict[real_related_column] = (group, column)
+ self.registerColumn(column, group=group)
+ self.related_group_dict[group] = related_column
+ # XXX: hardcoded translation table column names: they are not present in sql_catalog.getColumnMap(), and this table cannot be joined by uid, forbidding implicit join.
+ if column in ('translated_message', 'language', 'message_context', 'original_message'):
+ self.registerTable('translation', alias='translation', group=group)
+ self.resolveColumn(column, 'translation', group=group)
+ # Likewise, for measure table. Moreover, there is a related key named the same way as a column of that table (designed to do the join).
+ elif column in ('metric_type_uid', ):
+ self.registerTable('measure', group=group)
+ self.resolveColumn(column, 'measure', group=group)
+ return group
+
+ @profiler_decorator
+ def registerCatalog(self):
+ assert self.catalog_table_name is not None
+ LOG('ColumnMap', 100, 'Registering implicit catalog. This use is strongly discouraged.')
+ self.registerTable(self.catalog_table_name)
+ self.resolveTable(self.catalog_table_name, self.catalog_table_name)
+
+ @profiler_decorator
+ def registerRelatedKeyColumn(self, related_column, position, group):
+ assert group in self.related_group_dict
+ group = self.getRelatedKeyGroup(position, group)
+ assert group not in self.related_group_dict
+ self.related_group_dict[group] = related_column
+ return group
+
+ def getRelatedKeyGroup(self, position, group):
+ return '%s_column_%s' % (group, position)
+
+ @profiler_decorator
+ def registerTable(self, table_name, alias=None, group=DEFAULT_GROUP_ID):
+ table_alias_dict = self.table_alias_dict
+ table_alias_key = (group, table_name)
+ existing_value = table_alias_dict.get(table_alias_key)
+ # alias = None, existing = None -> store
+ # alias = None, existing ! None -> skip
+ # alias ! None, existing = None -> store & resolve
+ # alias ! None, existing ! None -> skip if alias = existing, raise otherwise
+ if existing_value is None:
+ table_alias_dict[table_alias_key] = alias
+ if alias is not None:
+ self.resolveTable(table_name, alias, group=group)
+ elif alias is not None and alias != existing_value:
+ raise ValueError, 'Table %r for group %r is aliased as %r, can\'t alias it now as %r' % (table_name, group, existing_value, alias)
+
+ @profiler_decorator
+ def _mapColumns(self, column_table_map, table_usage_dict, column_name_set, group, vote_result_dict):
+ mapping_dict = {}
+ catalog_table_name = self.catalog_table_name
+
+ # Map all columns to tables decided by vote.
+ for column_name, candidate_dict in vote_result_dict.iteritems():
+ # candidate_dict is never empty
+ max_score = 0
+ for table_name, score in candidate_dict.iteritems():
+ if score > max_score:
+ max_score = score
+ best_count = 0
+ best_choice = table_name
+ elif score == max_score:
+ best_count += 1
+ if best_count:
+ LOG('ColumnMap', 100, 'Mapping vote led to a tie. Mapping to %r' % (best_choice, ))
+ if TRACE:
+ LOG('ColumnMap', 0, 'Mapping by vote %r to %r' % (column_name, best_choice))
+ mapping_dict[column_name] = best_choice
+ column_name_set.remove(column_name)
+ for table_name, column_set in table_usage_dict.iteritems():
+ if table_name != best_choice:
+ column_set.discard(column_name)
+
+ # Map all remaning columns.
+ def table_weight(a):
+ """
+ Compute each table weight.
+ """
+ if (group, a[0]) in self.table_alias_dict:
+ result = (2, )
+ elif a[0] == catalog_table_name:
+ result = (1, )
+ else:
+ result = (0, len(a[1]))
+ return result
+ # Sort table name list, first has the most required columns
+ weighted_table_list = sorted(table_usage_dict.iteritems(), key=table_weight)
+ while len(weighted_table_list):
+ table_name, column_set = weighted_table_list.pop()
+ if len(column_set):
+ common_column_set = column_name_set.intersection(column_set)
+ if len(common_column_set):
+ # Only allow usage of this table if any of those is true:
+ # - current table is the catalog (if any catalog was provided)
+ # - there are column used on that table which are already mapped
+ # (this does not include columns mapped by this code)
+ # If columns are mapped to this table in current group, then using
+ # it will not require a new join, so it should be allowed.
+ # Note: it would be good to take indexes into account when there
+ # are multiple candidate tables.
+ # - any of those columns belongs exclusively to this table
+ # Although the list of tables those columns belong to is known
+ # earlier (in "build"), mapping them here
+ # - avoids code duplication (registerTable, resolveColumn,
+ # _addJoinTable)
+ # - offers user to vote for an unknown table, overriding this
+ # forced mapping.
+ use_allowed = table_name == catalog_table_name or \
+ len(common_column_set) < len(column_set)
+ if not use_allowed:
+ for column_name in column_set:
+ if len(column_table_map.get(column_name, [])) == 1:
+ # There is no alternative, mark as required
+ use_allowed = True
+ break
+ if use_allowed:
+ for column_name in common_column_set:
+ if TRACE:
+ LOG('ColumnMap', 0, 'Mapping by default %r to %r' % \
+ (column_name, table_name))
+ mapping_dict[column_name] = table_name
+ # This column must not be resolved any longer
+ column_name_set.remove(column_name)
+ # Remove this column from sets containing it. This prevents from
+ # giving a high score to a table which columns would already have
+ # been mapped to another table.
+ for ignored, other_column_set in weighted_table_list:
+ other_column_set.discard(column_name)
+ weighted_table_list.sort(key=table_weight)
+ else:
+ # All column which are mappable on that table are to-be-mapped
+ # columns. This means that this table was not explicitely used, and
+ # as each table contain a different amount of lines, we should not
+ # join with any non-explicit table. Hence, we skip this mapping.
+ LOG('ColumnMap', 0, 'Skipping possible map of %r on %r as that table' \
+ ' is not explicitely used.' % (common_column_set, table_name))
+
+ # Detect incomplete mappings
+ if len(column_name_set):
+ raise ValueError, 'Could not map those columns: %r' % (column_name_set, )
+
+ # Do the actual mapping
+ for column_name, table_name in mapping_dict.iteritems():
+ # Mark this column as resolved
+ if TRACE:
+ LOG('ColumnMap', 0, 'Mapping column %s to table %s' % (column_name, table_name))
+ self.registerTable(table_name, group=group)
+ self.resolveColumn(column_name, table_name, group=group)
+ if table_name != catalog_table_name:
+ self._addJoinTable(table_name, group)
+
+ @profiler_decorator
+ def build(self, sql_catalog):
+ catalog_table_name = self.catalog_table_name
+ if catalog_table_name is None:
+ return
+
+ column_table_map = sql_catalog.getColumnMap()
+ table_vote_method_list = [getattr(sql_catalog, x) for x in sql_catalog.sql_catalog_table_vote_scripts]
+
+ # Generate missing joins from default group (this is required to allow using related keys outside of queries: order_by, sort_on, ...)
+ column_set = self.registry.get(DEFAULT_GROUP_ID, [])
+ for column_name in column_set:
+ if column_name not in column_table_map and column_name not in self.related_key_dict:
+ related_key_definition = sql_catalog.getRelatedKeyDefinition(column_name)
+ if related_key_definition is not None:
+ join_query = sql_catalog.getSearchKey(column_name, 'RelatedKey').buildQuery(sql_catalog=sql_catalog, related_key_definition=related_key_definition)
+ join_query.registerColumnMap(sql_catalog, self)
+ self._addJoinQuery(join_query)
+
+ # List all possible tables, with all used column for each
+ for group, column_set in self.registry.iteritems():
+ # unique needed column name set
+ column_name_set = set()
+ # table -> column_set, including alternatives
+ table_usage_dict = {}
+
+ for column_name in column_set:
+ if column_name == '*' or column_name in self.column_ignore_set:
+ continue
+ table_name_list = column_table_map.get(column_name, [])
+ if len(table_name_list) == 0:
+ if not(group is DEFAULT_GROUP_ID and column_name in self.related_key_dict):
+ LOG('ColumnMap', 100, 'Not a known column name: %r' % (column_name, ))
+ continue
+ column_map_key = (group, column_name)
+ if column_map_key in self.column_map:
+ # Column is already mapped, so we must count this column as being available only on that table. Its mapping will not change, and it will impact table schema choice.
+ table_name = self.column_map[column_map_key]
+ assert table_name in table_name_list, '%r not in %r' % (table_name, table_name_list)
+ table_name_list = [table_name]
+ else:
+ # Mark this column as requiring to be mapped.
+ column_name_set.add(column_name)
+ for table_name in table_name_list:
+ table_usage_dict.setdefault(table_name, set()).add(column_name)
+ # XXX: mutable datatypes are provided to vote method. if it modifies
+ # them, it can introduce mapping bugs. Copying them might be costly,
+ # especialy if done before each call, since they also contain mutable
+ # types.
+ # XXX: the API of vote methods is not stable yet. Parameters should
+ # always be passed and expected by name, to make it less painful to
+ # change API.
+ # XXX: there is no check that the table voted for contains mapped
+ # column. It is up to the user not to do stupid things.
+ vote_result_dict = {}
+ simple_query_dict = self.simple_query_dict[group]
+ for table_vote_method in table_vote_method_list:
+ vote_dict = table_vote_method(column_name_set=column_name_set,
+ simple_query_dict=simple_query_dict,
+ table_usage_dict=table_usage_dict,
+ group=group)
+ if isinstance(vote_dict, dict):
+ for column, table in vote_dict.iteritems():
+ if column in column_name_set:
+ column_vote_dict = vote_result_dict.setdefault(column, {})
+ column_vote_dict[table] = column_vote_dict.get(table, 0) + 1
+ else:
+ LOG('ColumnMap', 100, 'Vote script %r voted for a ' \
+ 'non-candidate column: %r, candidates are: %r. Ignored.' %
+ (table_vote_method, column, column_name_set))
+ else:
+ LOG('ColumnMap', 100, 'Vote script %r returned invalid data: %r. ' \
+ 'Ignored.' % (table_vote_method, vote_dict))
+ self._mapColumns(column_table_map, table_usage_dict, column_name_set, group, vote_result_dict)
+
+ table_alias_number_dict = {}
+
+ for (group, table_name), alias in self.table_alias_dict.iteritems():
+ if alias is None:
+ if group in self.related_group_dict:
+ alias_table_name = 'related_%s_%s' % (self.related_group_dict[group], table_name)
+ else:
+ alias_table_name = table_name
+ table_alias_number = table_alias_number_dict.get(alias_table_name, 0)
+ while True:
+ if table_alias_number == 0:
+ alias = alias_table_name
+ else:
+ alias = '%s_%s' % (alias_table_name, table_alias_number)
+ table_alias_number += 1
+ if alias not in self.table_map:
+ break
+ table_alias_number_dict[alias_table_name] = table_alias_number
+ self.resolveTable(table_name, alias, group=group)
+
+ if TRACE:
+ # Key: group
+ # Value: 2-tuple
+ # dict
+ # Key: column
+ # Value: table name
+ # dict
+ # Key: table name
+ # Value: table alias
+ summary_dict = {}
+ for (group, column), table_name in self.column_map.iteritems():
+ column_dict = summary_dict.setdefault(group, ({}, {}))[0]
+ assert column not in column_dict, '%r in %r' % (column, column_dict)
+ column_dict[column] = table_name
+ for (group, table_name), table_alias in self.table_alias_dict.iteritems():
+ table_dict = summary_dict.setdefault(group, ({}, {}))[1]
+ assert table_name not in table_dict, '%r in %r' % (table_name, table_dict)
+ table_dict[table_name] = table_alias
+ for group, (column_dict, table_dict) in summary_dict.iteritems():
+ LOG('ColumnMap', 0, 'Group %r:' % (group, ))
+ LOG('ColumnMap', 0, ' Columns:')
+ for column, table_name in column_dict.iteritems():
+ LOG('ColumnMap', 0, ' %r from table %r' % (column, table_name))
+ LOG('ColumnMap', 0, ' Tables:')
+ for table_name, table_alias in table_dict.iteritems():
+ LOG('ColumnMap', 0, ' %r as %r' % (table_name, table_alias))
+
+ def asSQLColumn(self, raw_column, group=DEFAULT_GROUP_ID):
+ if self.catalog_table_name is None or raw_column in self.column_ignore_set or \
+ '.' in raw_column or '*' in raw_column:
+ result = raw_column
+ else:
+ function, column = self.raw_column_dict.get(raw_column, (None, raw_column))
+ if group is DEFAULT_GROUP_ID:
+ group, column = self.related_key_dict.get(column, (group, raw_column))
+ alias = self.table_alias_dict[(group, self.column_map[(group, column)])]
+ result = '`%s`.`%s`' % (alias, column)
+ if function is not None:
+ result = '%s(%s)' % (function, result)
+ return result
+
+ def getCatalogTableAlias(self, group=DEFAULT_GROUP_ID):
+ return self.table_alias_dict[(group, self.catalog_table_name)]
+
+ def getTableAliasDict(self):
+ return self.table_map.copy()
+
+ @profiler_decorator
+ def resolveColumn(self, column, table_name, group=DEFAULT_GROUP_ID):
+ assert group in self.registry
+ assert column in self.registry[group]
+ column_map_key = (group, column)
+ column_map = self.column_map
+ assert (group, table_name) in self.table_alias_dict
+ previous_value = column_map.get(column_map_key)
+ if previous_value is None:
+ column_map[column_map_key] = table_name
+ elif previous_value != table_name:
+ if column == 'uid':
+ LOG('ColumnMap', 100, 'Attempt to remap uid from %r to %r ignored.' % (previous_value, table_name))
+ else:
+ raise ValueError, 'Cannot remap a column to another table. column_map[%r] = %r, new = %r' % (column_map_key, column_map.get(column_map_key), table_name)
+
+ @profiler_decorator
+ def resolveTable(self, table_name, alias, group=DEFAULT_GROUP_ID):
+ table_alias_key = (group, table_name)
+ assert table_alias_key in self.table_alias_dict
+ assert self.table_alias_dict[table_alias_key] in (None, alias)
+ self.table_alias_dict[table_alias_key] = alias
+ assert self.table_map.get(alias) in (None, table_name)
+ self.table_map[alias] = table_name
+
+ def getTableAlias(self, table_name, group=DEFAULT_GROUP_ID):
+ return self.table_alias_dict[(group, table_name)]
+
+ def _addJoinQuery(self, query):
+ self.join_query_list.append(query)
+
+ def addJoinQuery(self, query):
+ LOG('ColumnMap', 0, 'addJoinQuery use is discouraged')
+ self._addJoinQuery(query)
+
+ def iterJoinQueryList(self):
+ return iter(self.join_query_list)
+
+ @profiler_decorator
+ def _addJoinTable(self, table_name, group=DEFAULT_GROUP_ID):
+ """
+ Declare given table as requiring to be joined with catalog table.
+
+ table_name (string)
+ Table name.
+ group (string)
+ Group id of given table.
+ """
+ catalog_table = self.catalog_table_name
+ if catalog_table is not None:
+ # Only join tables when there is a catalog table
+ # Register unconditionaly catalog table
+ self.registerTable(catalog_table)
+ if 'uid' not in self.registry.get(DEFAULT_GROUP_ID, ()):
+ # Register uid column if it is not already
+ self.registerColumn('uid')
+ self.resolveColumn('uid', catalog_table)
+ self.join_table_set.add((group, table_name))
+
+ def getJoinTableAliasList(self):
+ return [self.getTableAlias(table_name, group=group)
+ for (group, table_name) in self.join_table_set]
+
+ def getStraightJoinTableList(self):
+ return self.straight_join_table_list[:]
+
+ def getLeftJoinTableList(self):
+ return self.left_join_table_list[:]
+
+verifyClass(IColumnMap, ColumnMap)
+
diff --git a/product/ZSQLCatalog/Interface/IAbstractSyntaxNode.py b/product/ZSQLCatalog/Interface/IAbstractSyntaxNode.py
new file mode 100644
index 0000000000000000000000000000000000000000..a6310d48ee61e1c0653df44fe465536b5d361a54
--- /dev/null
+++ b/product/ZSQLCatalog/Interface/IAbstractSyntaxNode.py
@@ -0,0 +1,99 @@
+##############################################################################
+#
+# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+from Interface import Interface
+
+class INode(Interface):
+ """
+ Any kind of node in an Abstract Syntax Tree.
+ """
+
+ def isLeaf():
+ """
+ Returns True if current node is a leaf in node tree.
+ Returns False otherwise.
+ """
+
+ def isColumn():
+ """
+ Returns True if current node is a column in node tree.
+ Returns False otherwise.
+ """
+
+class IValueNode(INode):
+ """
+ Value- and comparison-operator-containig node.
+ They are leaf nodes in the syntax tree.
+ """
+
+ def getValue():
+ """
+ Returns node's value.
+ """
+
+ def getComparisonOperator():
+ """
+ Returns node's comparison operator.
+ """
+
+class ILogicalNode(INode):
+ """
+ Logical-operator-containing node.
+ They are internal tree nodes.
+ """
+
+ def getLogicalOperator():
+ """
+ Returns node's logical operator.
+ """
+
+ def getNodeList():
+ """
+ Returns the list of subnodes.
+ """
+
+class IColumnNode(INode):
+ """
+ Column-name-containing node.
+ They are internal tree nodes.
+ Their value applies to any contained ValueNode, except if there is another
+ ColumnNode between current one and a ValueNode, for which the other
+ ColumnNode will take precedence.
+ """
+
+ def getColumnName():
+ """
+ Returns node's column name.
+ """
+
+ def getSubNode():
+ """
+ Returns node's (only) subnode.
+ """
+
diff --git a/product/ZSQLCatalog/Interface/IColumnMap.py b/product/ZSQLCatalog/Interface/IColumnMap.py
new file mode 100644
index 0000000000000000000000000000000000000000..98a3e5991a117bd04a91bf5a6c633a555687fb42
--- /dev/null
+++ b/product/ZSQLCatalog/Interface/IColumnMap.py
@@ -0,0 +1,308 @@
+##############################################################################
+#
+# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+from Interface import Interface
+
+class IColumnMap(Interface):
+ """
+ The role of the column mapper is to make possible to have a "flat"
+ overview of all columns used in a query, to choose how they will be
+ mapped to catalog tables, and how those tables will be aliased in the SQL
+ rendering of that query.
+
+ Typical usage:
+ - Instanciate (with or without a catalog table)
+ - Register all columns the query will use
+ - Build the column map
+ - Fetch SQL representation of each registered column.
+ - Fetch table alias mapping
+ - Fetch the list of table aliases which require a join with catalog
+ - Fetch the list of queries implied by columns. This happens when there
+ are virtual columns registered which were not expanded in the query
+ already. In that case, column map generate queries required to reach
+ the real column behind the virtual one.
+
+ Note that, although it's not enforced, it is meaningless to:
+ - call build more than once
+ - register columns after build happened
+ - fetch SQL representation before build happened (this will most probably
+ lead to KeyErrors anyway)
+
+ Groups.
+ All references to a given table withing a given group will cause all
+ columns referencing that table to be resolved to the same table alias
+ in resulting query.
+ Ex:
+ Note: instead of resolving columns by hand, it is also possible to
+ call "build", but this way it makes .
+ Registration:
+ registerColumn('bar')
+ registerColumn('baz')
+ resolveColumn('bar', 'foo')
+ resolveColumn('baz', 'foo')
+ Result:
+ asSQLColumn('bar') -> 'foo_alias.bar'
+ asSQLColumn('baz') -> 'foo_alias.baz'
+ Complementary, any reference to a given table from one group to
+ another will cause columns to be resolved to distinct table aliases
+ in resulting query.
+ Ex:
+ Registration:
+ registerColumn('bar')
+ registerColumn('baz', group='hoge')
+ resolveColumn('bar', 'foo')
+ resolveColumn('baz', 'foo', group='hoge')
+ Result:
+ asSQLColumn('bar') -> 'foo_alias.bar'
+ asSQLColumn('baz', group='hoge') -> 'other_foo_alias.baz'
+ """
+
+ def __init__(catalog_table_name=None):
+ """
+ Create a column map.
+
+ Initialises internal data structures and set the table name to use as
+ catalog table.
+ """
+
+ def registerColumn(column, group=None, simple_query=None):
+ """
+ Register a column in given group on this column map.
+
+ column (string)
+ Contains the name of the column to register.
+
+ This name might contain a dot, in which case a table name is extracted
+ from it and that column is resolved to that table within its group.
+ Note that this practice is discouraged, but not deprecated.
+ group (string)
+ Group id of given column.
+ simple_query (SimpleQuery)
+ The SimpleQuery doing a comparison on given column. This can be
+ accessed from vote script at ColumnMap build time.
+ """
+
+ def ignoreColumn(column):
+ """
+ Act as if given column was valid and its mapping succeeded.
+ ie: supresses all warnings relative to that column, and all mapping
+ attemps.
+ It is supposed to be used when given value is an SQL alias for a
+ separately registered column.
+
+ column (string)
+ The value to ignore.
+ """
+
+ def registerRelatedKey(related_column, column):
+ """
+ Register a "virtual column".
+ This method is to be called when registering any virtual column.
+ A virtual column makes use internaly of multiple catalog columns.
+ A related key is an example of a virtual column.
+
+ This method internaly generates a new group id that caller must pass to
+ registerRelatedKeyColumn when registering those internal (and real)
+ columns. This is because the same virtual column can be used more than
+ once in a query without interferences from other uses, and hence must be
+ put in separate groups, without having to know that there are other uses
+ for the same virtual column.
+
+ virtual_column
+ Name of the virtual column to register.
+ column
+ Name of the "last" real column behind the virtual column. Comparisons
+ done on the virtual column will happen on that real column instead.
+ For example, it's "title" for "source_title" related key.
+ """
+
+ def registerCatalog():
+ """
+ Register catalog table as being used in default group.
+
+ This is for backward compatibility with text-only related keys, since
+ they often hardcode catalog table name.
+ """
+
+ def registerRelatedKeyColumn(related_column, position, group):
+ """
+ Register given column as part of already registered virtual column.
+
+ related_column (string)
+ Name of the virtual column registered column is a part of.
+ position (int)
+ Unique id of this column in the list of all columns used by the
+ virtual column it is a part of (it might use the same column name
+ while expecting uses to be mapped to different aliases or evne
+ different tables).
+ Typically, this is table position in the parameter list the related
+ key expects.
+ group (string)
+ Group id as returned by registerRelatedKey of given virtual column.
+ """
+
+ def getRelatedKeyGroup(position, group):
+ """
+ For given virtual key position and group, return a group.
+
+ This is here so that all group generation code is inside ColumnMap
+ class.
+ """
+
+ def registerTable(table_name, alias=None, group=None):
+ """
+ Register given table name as being used in given group.
+
+ This method should not be called outside of this class except for
+ backward compatibility purposes.
+
+ It is implicitely done most of the time, and should only be used only
+ when there is no control over chosen table alias or when table
+ registration cannot be done by a Query.
+ There are 2 cases where it cas required:
+ - pre-mapped columns which won't be actually used, but which force table
+ alias choice (see EntireQuery).
+ - related keys, where the Related Query cannot register columns it uses
+ by itself (since its content is not based on Queries but on raw SQL
+ text).
+ """
+
+ def build(sql_catalog):
+ """
+ Resolve all unresolved registered columns (ie, not mapped to tables).
+ Resolve all used tables to unique aliases.
+
+ Chosen table aliases are based on table name and virtual column name the
+ column comes from, if applicable. This is done to make final SQL more
+ reader-friendly.
+ """
+
+ def asSQLColumn(column, group=None):
+ """
+ Return an SQL rendering of given column, with the table alias it has
+ been mapped to.
+
+ column (string)
+ Column name. Can be the name of a virtual column.
+
+ This name might contain a dot, in which case a table name is extracted
+ from it and that column is resolved to that table within its group.
+ Note that this practice is discouraged, but not deprecated.
+ Note also that it does not apply to virtual columns (they cannot
+ contain a dot).
+ group (string)
+ Group id of given column.
+ """
+
+ def getCatalogTableAlias(group=None):
+ """
+ Return the alias of catalog table name given at instantiation.
+ """
+
+ def getTableAliasDict():
+ """
+ Return a copy of table mapping.
+
+ returned value (dict)
+ key (string)
+ Table alias.
+ value (string)
+ Table name.
+ """
+
+ def resolveColumn(column, table_name, group=None):
+ """
+ Map given column to given table within given group.
+
+ column (string)
+ Name of the column to map to a table. This cannot be a virtual column.
+ table_name (string)
+ Name of the table column must be mapped to. This is not an alias.
+ It must be already known to be used within given group.
+ group (string)
+ Name of the group column and table are in.
+ """
+
+ def resolveTable(table_name, alias, group=None):
+ """
+ Resolve given table of given group as given alias.
+
+ table_name (string)
+ Table to alias.
+ alias (string)
+ Table alias. It must be unique at query scope to work as intended.
+ gorup (string)
+ Group the table belongs to.
+ """
+
+ def getTableAlias(table_name, group=None):
+ """
+ Get table alias in given group.
+
+ table_name (string)
+ Table name.
+ group (string)
+ Group id of given table.
+ """
+
+ def addJoinQuery(query):
+ """
+ Adds a query to a list of top-level queries.
+ This method is only made available at interface level for backward
+ compatibility.
+ """
+
+ def iterJoinQueryList():
+ """
+ Get an iterator over queries internally generated when resolving column
+ map. Those queries are generated when a virtual column was registered as
+ a real one. Queries required to map virtual columns to real ones are
+ put in that list by build.
+ """
+
+ def getJoinTableAliasList():
+ """
+ Return a copy of the table alias list for tables requiring a join with
+ catalog table.
+ """
+
+ def getStraightJoinTableList():
+ """
+ Returns the list of tables used this search and which
+ need to be joined with the main table using explicit
+ indices.
+ """
+
+ def getLeftJoinTableList():
+ """
+ Returns the list of tables used this search and which
+ need to be LEFT joined with the main table using explicit
+ indices.
+ """
+
diff --git a/product/ZSQLCatalog/Interface/IEntireQuery.py b/product/ZSQLCatalog/Interface/IEntireQuery.py
new file mode 100644
index 0000000000000000000000000000000000000000..146ab1da2774141008c5fa422ee74c4ae0454e55
--- /dev/null
+++ b/product/ZSQLCatalog/Interface/IEntireQuery.py
@@ -0,0 +1,101 @@
+##############################################################################
+#
+# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+from Interface import Interface
+
+class IEntireQuery(Interface):
+ """
+ A EntireQuery represents an entire SQL expression, where a Query
+ represents only the "WHERE" part of that expression.
+ A EntireQuery contains:
+ - a Query instance
+ - a limit expression
+ - a group-by expression
+ - an order-by expression
+ - a select expression
+ It internaly uses a ColumnMap instance to resolve tables to use to
+ generate a "from" expression.
+ """
+
+ def __init__(query, order_by_list=None, group_by_list=None,
+ select_dict=None, limit=None, catalog_table_name=None,
+ extra_column_list=None, from_expression=None,
+ order_by_override_list=None):
+ """
+ query (Query instance)
+ The root of the Query tree this query will contain.
+ order_by_list (list of 1-tuple, 2-tuple or 3-tuple)
+ The list of columns which will be sorted by SQL.
+ Tuple values are:
+ - mandatory: column name
+ - optionnal: sort order (can be "ASC" or "DESC", "ASC" by default)
+ - optionnal: type cast (no cast by default, see "CAST" SQL method)
+ group_by_list (list of string)
+ The list of columns which will be groupped by value by SQL.
+ select_dict (dict, key: string, value: string, None)
+ Given values describe columns to make available in SQL result.
+ If column is aliased in result set, key is the alias and value is the
+ column.
+ Otherwise, key is the column, and value must be None.
+ limit
+ See SQLExpression.
+ catalog_table_name (string)
+ Name of the table to use as a catalog.
+
+ Deprecated parameters.
+ extra_column_list (list of string)
+ The list of columns to register to column map. They will not be used
+ in final rendering, but are hint on which table are supposed to be
+ used when mapping columns.
+ from_expression
+ See SQLExpression.
+ order_by_override_list (list of string)
+ If a column is in order_by_list, cannot be mapped to a table column
+ but is present in this list, it will be passed through to
+ SQLExpression.
+ """
+
+ def asSQLExpression(sql_catalog, only_group_columns):
+ """
+ Instantiate a column map, process parameters given at instantiation and
+ register them to column map.
+ Register query to column map.
+ Build column map.
+ Generate extra SQLExpressions from column map.
+ Generate SQLExpression instance and return it.
+ """
+
+ def asSearchTextExpression(sql_catalog):
+ """
+ This is just a passthrough to embeded Query's asSearchTextExpression
+ method.
+ This means that only the where expression can be represented as a
+ SearchText, but not sort, limits, ...
+ """
+
diff --git a/product/ZSQLCatalog/Interface/IOperator.py b/product/ZSQLCatalog/Interface/IOperator.py
new file mode 100644
index 0000000000000000000000000000000000000000..459d4f5c4bc77afb9f697437a16d0e9c4c6fa0cd
--- /dev/null
+++ b/product/ZSQLCatalog/Interface/IOperator.py
@@ -0,0 +1,71 @@
+##############################################################################
+#
+# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+from Interface import Interface
+
+class IOperator(Interface):
+ """
+ An operator is responsible for rendering a value and a column name as SQL
+ or Search Text.
+
+ This class is designed to be used as a singleton-per-operator.
+ """
+
+ def __init__(operator):
+ """
+ operator (string)
+ Operator's text representation. It is used both for SQL and SearchText
+ rendering.
+ """
+
+ def asSearchText(value):
+ """
+ Render given value as Search Text
+
+ value (see _renderValue)
+ Value to render as a string for use in a Search Text expression.
+ """
+
+ def asSQLExpression(column, value_list, only_group_columns):
+ """
+ Construct a SQLExpression instance from given column and value, with
+ contained glue text.
+ value_list can be a non-list instance, which must be handled that same
+ way as a list of one item.
+ only_group_columns (bool)
+ If false, the operator can add group columns in the "select_dict" of
+ returned SLQExpression.
+ Otherwise, it must not (SQL would be invalid).
+ """
+
+ def getOperator():
+ """
+ Accessor for operator's text representation.
+ """
+
diff --git a/product/ZSQLCatalog/Interface/IQuery.py b/product/ZSQLCatalog/Interface/IQuery.py
new file mode 100644
index 0000000000000000000000000000000000000000..d84f68226584150b6a5092d1d73612a2a2505cdc
--- /dev/null
+++ b/product/ZSQLCatalog/Interface/IQuery.py
@@ -0,0 +1,69 @@
+##############################################################################
+#
+# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+from Interface import Interface
+
+class IQuery(Interface):
+ """
+ A Query contains:
+ - a value
+ - an operator
+ - a column
+
+ It is the python representation of a predicate, independently of its
+ rendering (SQL or SearchText).
+ For SQL rendering to be possible, it is necesary for some data to be
+ centralized in a data structure known at EntireQuery level (to be able to
+ generate unique table aliases, for exemple). This is the role of
+ ColumnMap, and registerColumnMap method on this interface.
+
+ This interface also offers various rendering methods, one per rendering
+ format.
+ """
+
+ def asSearchTextExpression(sql_catalog, column=None):
+ """
+ Render a query in a user-oriented SearchText.
+ Returns None if there is this query has no SearchText representation,
+ but is SearchText-aware.
+ If column is provided, it must be used instead of local knowledge of
+ column name. It is used to make queries inside a related key render
+ correctly.
+ """
+
+ def asSQLExpression(sql_catalog, column_map, only_group_columns):
+ """
+ Render a query as an SQLExpression instance.
+ """
+
+ def registerColumnMap(sql_catalog, column_map):
+ """
+ Register a query to given column_map.
+ """
+
diff --git a/product/ZSQLCatalog/Interface/IQueryCatalog.py b/product/ZSQLCatalog/Interface/IQueryCatalog.py
new file mode 100644
index 0000000000000000000000000000000000000000..c569909c07b079a14277e179f336d42dfbe3d3ed
--- /dev/null
+++ b/product/ZSQLCatalog/Interface/IQueryCatalog.py
@@ -0,0 +1,209 @@
+##############################################################################
+#
+# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+from Interface import Interface
+
+class ISearchKeyCatalog(Interface):
+
+ def buildQuery(kw, ignore_empty_string=True, operator='and'):
+ """
+ Build a ComplexQuery from kw values.
+
+ kw (dict: string keys, any value)
+ A query will be emited based on its value. Depending on the type of
+ the value it is handled differently. Query values will be passed
+ through to result (key is ignored). For all other types, their key
+ must be either a known column of a sql_search_tables table, or a
+ related key name.
+ - String values will be parsed according to the default SearchKey of
+ their real column (even for related keys). If parsing was
+ successful, Queries will be generated from its output.
+ Otherwise, that value will be taken as such.
+ - Dictionary values can be composed of the following keys:
+ 'query': Their payload value, considered as empty if not given.
+ 'key': The SearchKey to use for this value, overriding default
+ column configuration.
+ (for other possible keys, see SearchKeys)
+ They will be taken as such.
+ - All other types will be taken as such, and no "empty" check will be
+ performed on them.
+ ignore_empty_string (boolean)
+ If True, values from kw which are empty will be skipped.
+ operator (string)
+ Used to explicit the logical relation between kw entries.
+ It must be a valid ComplexQuery logical operator ('and', 'or').
+ """
+
+ def buildSQLQuery(query_table='catalog', REQUEST=None,
+ ignore_empty_string=1, only_group_columns=False,
+ limit=None, extra_column_list=None,
+ **kw):
+ """
+ Construct and return an instance of EntireQuery class from given
+ parameters by calling buildQuery.
+
+ ignore_empty_string (boolean)
+ See buildQuery.
+ limit (1-tuple, 2-tuple)
+ If given, will emit SQL to limit the number of result lines.
+ group_by_list (list of strings)
+ If given, will emit SQL to group found lines on given parameter names
+ (their column if they are column names, corresponding virtual columns
+ otherwise - as for related keys).
+ select_dict (dict, key: string, value: string, None)
+ Given values describe columns to make available in SQL result.
+ If column is aliased in result set, key is the alias and value is the
+ column.
+ Otherwise, key is the column, and value can be None or the same as
+ key.
+ select_list (list of strings)
+ Same as providing select_dict with select_list items as keys, and None
+ values.
+ order_by_list (list of 1-, 2-, or 3-tuples of strings)
+ If given, will emit SQL to sort result lines.
+ Sort will happen with decreasing precedence in list order.
+ Given n-tuples can contain those values, always in this order:
+ - parameter name
+ - sort order (see SQL documentation of 'ORDER BY')
+ - type cast (see SQL documentation of 'CAST')
+ Sort will happen on given parameter name (its column if it's a column
+ name, corresponding virtual column otherwise - as for related keys).
+ Extra parameters are passed through to buildQuery.
+
+ Backward compatibility parameters:
+ Those parameters are deprecated and should not be used. They are present
+ to provide backward compatibility with former ZSQLCatalog version.
+ REQUEST
+ Ignored.
+ extra_column_list (list)
+ query_table (string, None)
+ The table to use as catalog table.
+ If given and None, not catalog table will be used. Use this when you
+ are using SQLCatalog to generate manualy a part of another query.
+ That table has a special position in returned query:
+ - all other tables are joined on this one (when it is required to use
+ other tables)
+ - it is expected to have some columns (uid, path)
+ It is strongly discouraged to use this parameter for any value other
+ than None.
+ group_by
+ Use group_by_list instead.
+ group_by_expression
+ Use group_by_list instead.
+ select_expression
+ Use select_list or select_dict instead.
+ sort_on
+ Use order_by_list instead.
+ sort_order
+ Use order_by_list instead.
+ from_expression
+ This value will be emited in SQL directly in addition to computed
+ value.
+ There is no replacement.
+ where_expression
+ This value will be emited in SQL directly in addition to computed
+ value.
+ Use Query instances instead.
+ select_expression_key
+ This prevents given column from being ignored even if they could not
+ be mapped.
+ There is no replacement.
+ only_group_columns
+ Replaces former stat__ parameter.
+ Used to globally disalow use of non-group columns in SQL.
+ """
+
+ def getSearchKey(column, search_key=None):
+ """
+ Returns the default SearchKey instance for the
+ requested column. There is one instance per
+ search_key (incl. virtual keys surch as
+ source_title) and we try to compute it once
+ only and cache it.
+ If search_key is provided, it is used as the
+ name of the search key class to return.
+ """
+
+ def getComparisonOperator(operator):
+ """
+ Return a comparison operator matching given string.
+ String must be a valid SQL comparison operator (=, LIKE, IN, ...).
+ String case does not matter.
+ There is one comparison operator instance per possible string value.
+ """
+
+ # TODO: add support for other operators (logical, ensemblist (?))
+
+ def searchResults(REQUEST=None, **kw):
+ """
+ Invokes queryResults with the appropriate
+ ZSQL Method to return a list of results
+ """
+
+ def countResults(REQUEST=None, **kw):
+ """
+ Invokes queryResults with the appropriate
+ ZSQL Method to return a statistics for
+ a list of results
+ """
+
+ def queryResults(sql_method, REQUEST=None, src__=0, build_sql_query_method=None, **kw):
+ """
+ Return the result of the given 'sql_method' ZSQL Method after
+ processing all parameters to build a Query object passed to
+ that method.
+
+ The implementation should do the following.
+
+ 1- Use **kw parameters to build a Query object
+ by invoking buildQuery
+
+ 2- Build a ColumnMap instance by invoking
+ the buildColumnMap on the Query. Some
+ optmisation may happen here to try
+ to build the best possible ColumnMap and
+ use the best possible indices for joining.
+ During the ColumnMap build process, the
+ Search Key associated to each Query node
+ in the Query tree registers the columns
+ which are used (ex. to search) or provided
+ (ex. MATCH value for full text search,
+ interleave expression or parameter in a
+ UNION Query)
+
+ 3- Render the query object as an SQLExpression
+ instance. This instance contains all necessary
+ parts to generate:
+ - where_expression
+ - sort_expression
+ - group_by_expression
+ - select_expression
+
+ 4- Invoke sql_method
+ """
diff --git a/product/ZSQLCatalog/Interface/ISQLExpression.py b/product/ZSQLCatalog/Interface/ISQLExpression.py
new file mode 100644
index 0000000000000000000000000000000000000000..b8b99877fd0baabc2eb0dd5cb69103f0a8a3c862
--- /dev/null
+++ b/product/ZSQLCatalog/Interface/ISQLExpression.py
@@ -0,0 +1,119 @@
+##############################################################################
+#
+# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+from Interface import Interface
+
+class ISQLExpression(Interface):
+ """
+ This is a container for chunks of SQL generated by a Query.
+ Each Query instance generates its own SQLExpression instance.
+ SQLExpressions can be nested (an SQLExpression can contain 0, 1 or more
+ SQLExpressions).
+
+ Chunks are:
+ - table_alias_dict
+ - order_by_list
+ - group_by_list
+ - select_dict
+ - limit
+ Mutualy exclusive chunks are:
+ - where expression
+ - sql_expression_list
+ Deprecated chunks are:
+ - from_expression
+
+ Providing sql_expression_list with more than one entry makes
+ where_expression_operator mandatory.
+ """
+
+ def __init__(query,
+ table_alias_dict=None,
+ order_by_list=None,
+ order_by_dict=None,
+ group_by_list=None,
+ where_expression=None,
+ where_expression_operator=None,
+ sql_expression_list=None,
+ select_dict=None,
+ limit=None,
+ from_expression=None):
+ """
+ Instantiate an SQLExpression object.
+
+ This method does consistency checks on received parameters, so that
+ failures are detected as early as possible.
+
+ Also, it casts most optional parameters into empty lists and empty dicts
+ to make code using those values simpler.
+
+ query (Query)
+ The Query instance which called this constructor.
+ table_alias_dict (dict, key: string, value: string)
+ Table alias dict as returned by ColumnMap.getTableAliasDict() .
+ order_by_list (list of strings)
+ List of result ordering, pre-rendered.
+ order_by_dict (dict, key: string, value: string)
+ Column rendering replacement specific to order_by.
+ group_by_list (list of strings)
+ List of column names on which result line list will be grouped.
+ where_expression (string)
+ Text representing a "where" expression of an SQL query.
+ where_expression_operator ("and", "or", "not", None)
+ Operator to apply on immediately contained SQLExpressions.
+ It must be "and" or "or" when there are multiple contained
+ SQLExpressions, it can be "not" if there is exactly one contained
+ SQLExpression, and must not be provided if there is no contained
+ SQLExpression.
+ sql_expression_list (list of SQLExpression)
+ List of immediately contained SQLExpressions.
+ select_dict (dict, key:string, value:string or Null)
+ Lists all columns to be part of select expression.
+ Key is column alias.
+ Value is column name, or Null. If it is Null, the alias will also be
+ used as column name.
+ limit (1-tuple, 2-tuple, other)
+ First item is the number of lines expected, second one if given is the
+ offset of limited result list within the unlimited result list.
+ If it is not a tuple it is treated as the value of a 1-tuple
+ parameter.
+ from_expression (string)
+ This parameter is deprecated.
+ """
+
+ def asSQLExpressionDict():
+ """
+ Returns a dictionnary usable as a **kw for a catalog sql method.
+
+ It renders aliases (see getTableAliasDict) as a list of strings and
+ "from" expression (see getFromExpression) as a list of strings.
+
+ See getWhereExpression, getOrderByExpression, getLimitExpression,
+ getSelectExpression, getGroupByExpression.
+ """
+
diff --git a/product/ZSQLCatalog/Interface/ISearchKey.py b/product/ZSQLCatalog/Interface/ISearchKey.py
new file mode 100644
index 0000000000000000000000000000000000000000..9dc86d3cdb52ca30c782b36e8f35156360d13125
--- /dev/null
+++ b/product/ZSQLCatalog/Interface/ISearchKey.py
@@ -0,0 +1,190 @@
+##############################################################################
+#
+# Copyright (c) 2002-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+from Interface import Interface
+
+class ISearchKey(Interface):
+ """
+ A SearchKey generates Query instances.
+ It is responsible for parsing (or not) input value and generating the
+ appropriate Query structure to match the intended search.
+
+ This class is designed as a "singleton per column".
+ """
+
+ def __init__(column):
+ """
+ column (string)
+ The column this SearchKey is instanciated for.
+ """
+
+ def getColumn():
+ """
+ Returns: string
+ The column this SearchKey is instanciated for.
+ """
+
+ def buildSQLExpression(operator, value, column_map, only_group_columns, group):
+ """
+ This method passes on SQLExpression creation to operator, giving it the
+ resolved column of this RelatedKey.
+
+ operator (Operator)
+ A comparison operator. It is used to render value, column and itself
+ as valid SQL.
+ value (anything)
+ column_map (ColumnMap)
+ The (built) column map to render our column.
+ group (string)
+ The group our column was part of.
+
+ Returns: SQLExpression
+ SQLExpression build by operator for current Query.
+ """
+
+ def buildSearchTextExpression(operator, value, column=None):
+ """
+ Render comparison using operator between value and our column as
+ SearchText.
+
+ operator (Operator)
+ A comparison operator. It is used to render value as value SearchText.
+ value (anything)
+ column (string)
+ If given, it overrides our column in SearchText rendering. This is
+ useful when SearchText rendering is done for a virtual column,
+ because the name of the virtual column must appear when rendered as
+ SearchText.
+ """
+
+ def registerColumnMap(column_map, group, simple_query):
+ """
+ Register the column of this SearchKey to given column map.
+
+ column_map (ColumnMap)
+ The ColumnMap instance to register to.
+ group (string)
+ The group registration will be part of.
+ simple_query (SimpleQuery)
+ The SimpleQuery being registered.
+
+ Returns: string
+ The group assigned to caller. See ColumnMap for methods redefining
+ caller's group.
+ """
+
+ def buildQuery(search_value, group=None, logical_operator=None, comparison_operator=None):
+ """
+ Generate queries from given search_value.
+ logical_operator ('and', 'or', 'not', None)
+ If no logical operator can be found in search_value, it will use given
+ default operator.
+ comparison_operator (string, None)
+ If given, expresses the comparison between column and value.
+ """
+
+ def parseSearchText(value):
+ """
+ Parse given value to generate an Abstract Syntax Tree representing its
+ logical structure, or None if there is no obvious structure in given
+ value.
+ See SearchText for parsing code.
+
+ value (string)
+ The string to parse.
+
+ Returns: (None, AbstratSyntaxNode)
+ AbstratSyntaxNode complies with the IAbstractSyntaxNode interface.
+ """
+
+class IRelatedKey(ISearchKey):
+ """
+ A RelatedKey is a special variation of a SearchKey.
+ Only a small set of methods differ. They are defined in this class.
+ """
+
+ def registerColumnMap(column_map, table_alias_list=None):
+ """
+ This is slightly different for regular registerColumnMap in that it must
+ register multiple tables (and not columns, since RelatedKeys do not
+ provide this information).
+ Also, it must register namely "catalog" table and resolve its alias,
+ angain since it's hardcoded in RelatedKey ZSQLMethods and not provided
+ by their definitions.
+
+ column_map (ColumnMap)
+
+ Deprecated:
+ table_alias_list (None, list of 2-tuples of strings)
+ This list must have the exact same length as the list of tables
+ """
+
+ def buildSQLExpression(sql_catalog, column_map, only_group_columns, group):
+ """
+ operator and value parameters are useless, since this type of SearhKey
+ does not compare a value to any column, but uses a ZSQLMethod.
+ To reach that ZSQLMethod, it also required a new sql_catalog parameter.
+
+ sql_catalog (SQLCatalog)
+ Used to retrieve related key's ZSQLMethod.
+ """
+
+ def buildQuery(sql_catalog, related_key_definition, search_value=None, search_key_name=None, logical_operator=None, comparison_operator=None):
+ """
+ group is useless here, since group is determined by ColumnMap at
+ registration time. search_value becomes optional.
+
+ sql_catalog (SQLCatalog)
+ Used to retrieve real column's SearchKey. For example, a RelatedKey
+ used to compare with a "title" column will retrieve title's default
+ SearchKey (which should be a KeywordKey).
+ related_key_definition (string)
+ Describes parameters of a RelatedKey. It is composed of 3 mains parts,
+ separated by '/':
+ - a list of table names
+ Table names are separated by ','
+ - a column name
+ - the name of the related key ZSQLMethod
+ search_value (anything)
+ If given, a condition on real column will be generated.
+ Otherwise, only the SQL required to reach that column will be
+ generated. This is usefull when sorting on a virtual column, for
+ example.
+ search_key_name (string, None)
+ If given, it overrides real column's default SearchKey.
+ logical_operator (string, None)
+ If given, expresses the default logical link between operands.
+ It must be one of None, 'or' and 'and'.
+ It is overriden by operator present in search_value if it is a dict
+ and contains an 'operator' key.
+ 'or' is assumed if not given or given with a None value.
+ comparison_operator (string, None)
+ If given, expresses the comparison between column and value.
+ """
+
diff --git a/product/ZSQLCatalog/Interface/Verify.py b/product/ZSQLCatalog/Interface/Verify.py
new file mode 100644
index 0000000000000000000000000000000000000000..4c13e2d9b38a9a36ee63fa88c9708a6cfc51b348
--- /dev/null
+++ b/product/ZSQLCatalog/Interface/Verify.py
@@ -0,0 +1,3 @@
+# This imports from Zope's products, which would otherwise be unreachable from parent folder.
+from Interface.Verify import verifyClass
+
diff --git a/product/ZSQLCatalog/Interface/__init__.py b/product/ZSQLCatalog/Interface/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc
--- /dev/null
+++ b/product/ZSQLCatalog/Interface/__init__.py
@@ -0,0 +1 @@
+
diff --git a/product/ZSQLCatalog/Operator/ComparisonOperator.py b/product/ZSQLCatalog/Operator/ComparisonOperator.py
new file mode 100644
index 0000000000000000000000000000000000000000..d990fadb92cd4534499e79594005fc2ddf9002dc
--- /dev/null
+++ b/product/ZSQLCatalog/Operator/ComparisonOperator.py
@@ -0,0 +1,147 @@
+##############################################################################
+#
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+from OperatorBase import OperatorBase
+from Products.ZSQLCatalog.SQLExpression import SQLExpression
+from Products.ZSQLCatalog.Interface.IOperator import IOperator
+from Interface.Verify import verifyClass
+from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
+
+class ComparisonOperatorBase(OperatorBase):
+ @profiler_decorator
+ def asSQLExpression(self, column, value_list, only_group_columns):
+ """
+ In a Comparison Operator, rendering order is:
+ <column> <operator> <value_list>
+ """
+ column, value_list = self.render(column, value_list)
+ return SQLExpression(self, where_expression='%s %s %s' % (column, self.getOperator().upper(), value_list))
+
+ def render(self, column, value_list):
+ raise NotImplementedError, 'This method must be overloaded by a subclass.'
+
+ def renderValue(self, value_list):
+ raise NotImplementedError, 'This method must be overloaded by a subclass.'
+
+verifyClass(IOperator, ComparisonOperatorBase)
+
+class MonovaluedComparisonOperator(ComparisonOperatorBase):
+ @profiler_decorator
+ def renderValue(self, value_list):
+ """
+ value_list must either be a non-list or a single-value list.
+ """
+ if isinstance(value_list, (tuple, list)):
+ if len(value_list) > 1:
+ raise ValueError, '%r: value_list must not contain more than one item. Got %r' % (self, value_list)
+ value_list = value_list[0]
+ return self._renderValue(value_list)
+
+ @profiler_decorator
+ def render(self, column, value_list):
+ """
+ value_list must either be a non-list or a single-value list.
+ """
+ if isinstance(value_list, (tuple, list)):
+ if len(value_list) > 1:
+ raise ValueError, '%r: value_list must not contain more than one item. Got %r' % (self, value_list)
+ value_list = value_list[0]
+ return self._render(column, value_list)
+
+verifyClass(IOperator, MonovaluedComparisonOperator)
+
+class MultivaluedComparisonOperator(ComparisonOperatorBase):
+ @profiler_decorator
+ def renderValue(self, value_list):
+ """
+ value_list must be a multi-value list (more than one item).
+ """
+ if not isinstance(value_list, (tuple, list)) or len(value_list) < 2:
+ raise ValueError, '%r: value_list must be a list of more than one item. Got %r' % (self, value_list)
+ return '(%s)' % (', '.join([self._renderValue(x) for x in value_list]), )
+
+ @profiler_decorator
+ def render(self, column, value_list):
+ """
+ value_list must be a multi-value list (more than one item).
+ """
+ if not isinstance(value_list, (tuple, list)) or len(value_list) < 2:
+ raise ValueError, '%r: value_list must be a list of more than one item. Got %r' % (self, value_list)
+ return column, '(%s)' % (', '.join([self._renderValue(x) for x in value_list]), )
+
+verifyClass(IOperator, MultivaluedComparisonOperator)
+
+class MatchComparisonOperator(MonovaluedComparisonOperator):
+ def __init__(self, operator, mode=''):
+ MonovaluedComparisonOperator.__init__(self, operator)
+ self.where_expression_format_string = 'MATCH (%%(column)s) AGAINST (%%(value_list)s%s)' % (mode, )
+
+ @profiler_decorator
+ def asSQLExpression(self, column, value_list, only_group_columns):
+ """
+ This operator can emit a select expression, so it overrides
+ asSQLExpression inseatd of just defining a render method.
+ """
+ match_string = self.where_expression_format_string % {
+ 'column': column,
+ 'value_list': self.renderValue(value_list),
+ }
+ select_dict = {}
+ if not only_group_columns:
+ select_dict[column.replace('`', '').split('.')[-1]] = match_string
+ # Sort on this column uses relevance.
+ # TODO: Add a way to allow sorting by raw column value.
+ order_by_dict = {
+ column: self.where_expression_format_string,
+ }
+ return SQLExpression(
+ self,
+ select_dict=select_dict,
+ where_expression=match_string,
+ order_by_dict=order_by_dict,
+ )
+
+verifyClass(IOperator, MatchComparisonOperator)
+
+operator_dict = {
+ '=': MonovaluedComparisonOperator('='),
+ '!=': MonovaluedComparisonOperator('!='),
+ '>': MonovaluedComparisonOperator('>'),
+ '<': MonovaluedComparisonOperator('<'),
+ '<=': MonovaluedComparisonOperator('<='),
+ '>=': MonovaluedComparisonOperator('>='),
+ 'like': MonovaluedComparisonOperator('like'),
+ 'match': MatchComparisonOperator('match'),
+ 'match_boolean': MatchComparisonOperator('match_boolean', mode=' IN BOOLEAN MODE'),
+ 'match_expansion': MatchComparisonOperator('match_expansion', mode=' WITH QUERY EXPANSION'),
+ 'in': MultivaluedComparisonOperator('in'),
+ 'is': MonovaluedComparisonOperator('is'),
+}
+
diff --git a/product/ZSQLCatalog/Operator/OperatorBase.py b/product/ZSQLCatalog/Operator/OperatorBase.py
new file mode 100644
index 0000000000000000000000000000000000000000..7e5e8c561ff98f109aebe0e21ab1db624e9ed76c
--- /dev/null
+++ b/product/ZSQLCatalog/Operator/OperatorBase.py
@@ -0,0 +1,155 @@
+##############################################################################
+#
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+from zLOG import LOG
+from Products.ZSQLCatalog.Interface.IOperator import IOperator
+from Interface.Verify import verifyClass
+from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
+
+@profiler_decorator
+def escapeString(value):
+ # Inspired from ERP5Type/Utils:sqlquote, but this product must not depend on it.
+ return "'" + value.replace('\\', '\\\\').replace("'", "''") + "'"
+
+@profiler_decorator
+def valueFloatRenderer(value):
+ if isinstance(value, basestring):
+ value = float(value.replace(' ', ''))
+ return repr(value)
+
+@profiler_decorator
+def valueDateTimeRenderer(value):
+ return '"%s"' % (value.toZone('UTC').ISO(), )
+
+@profiler_decorator
+def valueDefaultRenderer(value):
+ LOG('OperatorBase', 0, 'Unhandled value class: %s (%r). Converted to string and escaped.' % (value.__class__.__name__, value))
+ return escapeString(str(value))
+
+@profiler_decorator
+def valueNoneRenderer(value):
+ return 'NULL'
+
+value_renderer = {
+ 'int': str,
+ 'long': str,
+ 'float': valueFloatRenderer,
+ 'DateTime': valueDateTimeRenderer,
+ 'NoneType': valueNoneRenderer,
+}
+
+value_search_text_renderer = {
+ 'DateTime': str,
+}
+
+@profiler_decorator
+def valueDefaultSearchTextRenderer(value):
+ """
+ This is just repr, but always surrounding text strings with doublequotes.
+ """
+ if isinstance(value, basestring):
+ result = '"%s"' % (value.replace('\\', '\\\\').replace('"', '\\"'), )
+ else:
+ result = repr(value)
+ return result
+
+@profiler_decorator
+def columnFloatRenderer(column, format=None):
+ if format is not None:
+ if '.' in format:
+ format = format.replace(' ', '')
+ column = "TRUNCATE(%s, %s)" % (column, len(format.split('.')[-1]))
+ return column
+
+@profiler_decorator
+def columnDefaultRenderer(column, format=None):
+ return column
+
+column_renderer = {
+ 'float': columnFloatRenderer
+}
+
+class OperatorBase(object):
+
+ __implements__ = IOperator
+
+ def __init__(self, operator):
+ self.operator = operator
+
+ def getOperator(self):
+ return self.operator
+
+ @profiler_decorator
+ def _render(self, column, value):
+ """
+ Render given column and value for use in SQL.
+ Value is rendered to convert it to SQL-friendly value.
+ Column is rendered to include possible cast code.
+
+ column (string)
+ Column on which the value will be matched
+ value (see _renderValue)
+ Value to render.
+ """
+ if isinstance(value, dict):
+ type = value['type']
+ column = column_renderer.get(type, columnDefaultRenderer)(column, format=value['format'])
+ value = value_renderer.get(type, valueDefaultRenderer)(value['query'])
+ else:
+ value = self._renderValue(value)
+ return column, value
+
+ @profiler_decorator
+ def _renderValue(self, value):
+ """
+ Render given value as string.
+
+ value (int, float, long, DateTime, string, None)
+ Value to render as a string for use in SQL (quoted, escaped).
+ """
+ if isinstance(value, basestring):
+ value = escapeString(value)
+ else:
+ value = value_renderer.get(value.__class__.__name__, valueDefaultRenderer)(value)
+ return value
+
+ @profiler_decorator
+ def asSearchText(self, value):
+ return value_search_text_renderer.get(value.__class__.__name__, valueDefaultSearchTextRenderer)(value)
+
+ def asSQLExpression(self, column, value_list, only_group_columns):
+ raise NotImplementedError, 'This method must be overloaded by a subclass ' \
+ 'to be able to get an SQL representation of this operator.'
+
+ def __repr__(self):
+ return '<%s(%r) at %s>' % (self.__class__.__name__, self.getOperator(), id(self))
+
+verifyClass(IOperator, OperatorBase)
+
diff --git a/product/ZSQLCatalog/Operator/__init__.py b/product/ZSQLCatalog/Operator/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..4710ca60bafa16f16dbbe710e18cb8213558f741
--- /dev/null
+++ b/product/ZSQLCatalog/Operator/__init__.py
@@ -0,0 +1,2 @@
+from ComparisonOperator import operator_dict
+
diff --git a/product/ZSQLCatalog/Query/AutoQuery.py b/product/ZSQLCatalog/Query/AutoQuery.py
new file mode 100644
index 0000000000000000000000000000000000000000..6d8331ff4a7c678a4edc45869e99d5de5b0d1b64
--- /dev/null
+++ b/product/ZSQLCatalog/Query/AutoQuery.py
@@ -0,0 +1,139 @@
+##############################################################################
+#
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+from Query import Query
+from zLOG import LOG
+from Products.ZSQLCatalog.Interface.IQuery import IQuery
+from Interface.Verify import verifyClass
+from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
+
+class AutoQuery(Query):
+ """
+ An AutoQuery is a compatibility layer for former Query class.
+ It passes parameters given at instantiation time to SQLCatalog's
+ buildQuery or buildSingleQuery, and wraps resulting Query instance (proxy
+ behaviour).
+
+ This is only here for backward compatibility, and use is strongly
+ discouraged. Use SQLCatalog API instead.
+ """
+ wrapped_query = None
+
+ @profiler_decorator
+ def __init__(self, *args, **kw):
+ """
+ Note: "operator" might contain a logical or a comparison operator.
+ """
+ if len(args):
+ LOG('AutoQuery', 100, 'Got extra positional parameters (will be ignored): %r' % (args, ))
+ self.table_alias_list = kw.pop('table_alias_list', None)
+ self.kw = kw
+ operator = kw.pop('operator', None)
+ if isinstance(operator, basestring):
+ operator = operator.lower()
+ self.operator = operator
+ self.ignore_empty_string = kw.pop('ignore_empty_string', True)
+ if 'key' in kw and len(kw) > 2:
+ raise ValueError, '"key" parameter cannot be used when more than one column is given. key=%r' % (self.search_key, )
+ self.search_key = kw.pop('key', None)
+
+ @profiler_decorator
+ def _createWrappedQuery(self, sql_catalog):
+ """
+ Create wrapped query. This requires being able to reach catalog, since
+ we use it as a query producer.
+ """
+ kw = self.kw
+ operator = self.operator
+ if 'range' in kw:
+ # If we received a range parameter we are building a single query.
+ # Recreate value as a dict and pass it to buildSingleQuery.
+ range = kw.pop('range')
+ assert len(kw) == 1, repr(kw)
+ key, value = kw.items()[0]
+ query = sql_catalog.buildSingleQuery(key, {'query': value,
+ 'range': range})
+ elif operator == 'in':
+ # 'in' is a *comparison* operator, not a logical operator.
+ # Transform kw into the proper form.
+ assert len(kw) == 1, repr(kw)
+ key, value = kw.items()[0]
+ query = sql_catalog.buildSingleQuery(key, {'query': value,
+ 'operator': operator})
+ elif len(kw) == 1 and isinstance(kw.values()[0], (tuple, list)) and \
+ operator in ('and', 'or'):
+ # If there is only one parameter, and operator was given and is a
+ # known logical operator, then operator will apply to it.
+ # For example (from testDomainTool):
+ # kw = {'portal_type': ['!=a', '!=b'], 'operator': 'AND'}
+ # In such case, expected result is
+ # "portal_type!='a' AND portal_type!='b'"
+ key, value = kw.items()[0]
+ query = sql_catalog.buildSingleQuery(key, value, logical_operator=operator)
+ else:
+ # Otherwise, the operator will apply to the relationship between
+ # parameters.
+ if operator is None:
+ operator = 'and'
+ if self.search_key is not None:
+ key, value = kw.items()[0]
+ kw = {key: {'query': value, 'key': self.search_key}}
+ query = sql_catalog.buildQuery(kw, operator=operator, ignore_empty_string=self.ignore_empty_string)
+ if self.table_alias_list is not None:
+ query.setTableAliasList(self.table_alias_list)
+ self.wrapped_query = query
+
+ @profiler_decorator
+ def asSearchTextExpression(self, sql_catalog, column=None):
+ if self.wrapped_query is None:
+ self._createWrappedQuery(sql_catalog)
+ return self.wrapped_query.asSearchTextExpression(sql_catalog, column=column)
+
+ @profiler_decorator
+ def asSQLExpression(self, sql_catalog, column_map, only_group_columns):
+ if self.wrapped_query is None:
+ self._createWrappedQuery(sql_catalog)
+ return self.wrapped_query.asSQLExpression(sql_catalog, column_map, only_group_columns=only_group_columns)
+
+ @profiler_decorator
+ def registerColumnMap(self, sql_catalog, column_map):
+ if self.wrapped_query is None:
+ self._createWrappedQuery(sql_catalog)
+ return self.wrapped_query.registerColumnMap(sql_catalog, column_map)
+
+ def __repr__(self):
+ if self.wrapped_query is None:
+ result = '<%s(**%r) at %s>' % (self.__class__.__name__, self.kw, id(self))
+ else:
+ result = '<%s %r>' % (self.__class__.__name__, self.wrapped_query)
+ return result
+
+verifyClass(IQuery, AutoQuery)
+
diff --git a/product/ZSQLCatalog/Query/ComplexQuery.py b/product/ZSQLCatalog/Query/ComplexQuery.py
index 16919945a734b9ff410463d81e7419ca389de68d..d817be737d3c31e927ec7b101bdda4b6165aeda0 100644
--- a/product/ZSQLCatalog/Query/ComplexQuery.py
+++ b/product/ZSQLCatalog/Query/ComplexQuery.py
@@ -1,7 +1,9 @@
##############################################################################
#
-# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
-# Ivan Tyagov <ivan@nexedi.com>
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
@@ -26,68 +28,127 @@
#
##############################################################################
-from Products.PythonScripts.Utility import allow_class
-from Query import QueryMixin
+from Query import Query
+from Products.ZSQLCatalog.SQLExpression import SQLExpression
+from SQLQuery import SQLQuery
+from Products.ZSQLCatalog.Interface.IQuery import IQuery
+from Interface.Verify import verifyClass
+from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
-class ComplexQuery(QueryMixin):
+class ComplexQuery(Query):
"""
- Used in order to concatenate many queries
+ A ComplexQuery represents logical operations between Query instances.
"""
-
+ @profiler_decorator
def __init__(self, *args, **kw):
- self.query_list = args
- self.operator = kw.pop('operator', 'AND')
- # XXX: What is that used for ?! It's utterly dangerous.
- #self.__dict__.update(kw)
-
- def getQueryList(self):
- return self.query_list
-
- def getRelatedTableMapDict(self):
- result = {}
- for query in self.getQueryList():
- if not(isinstance(query, basestring)):
- result.update(query.getRelatedTableMapDict())
- return result
-
- def asSQLExpression(self, key_alias_dict=None,
- ignore_empty_string=1,
- keyword_search_keys=None,
- datetime_search_keys=None,
- full_text_search_keys=None,
- stat__=0):
"""
- Build the sql string
+ *args (tuple of Query or of list of Query)
+ list-type entry will extend subquery list, other entries will be
+ appended.
+ logical_operator ('and', 'or', 'not')
+ Logical operator.
+ Default: 'and'
+
+ Deprecated
+ operator ('and', 'or', 'not')
+ See logical_operator.
+ logical_operator takes precedence if given.
+ unknown_column_dict (dict)
+ Only one key of this dictionnary is used here:
+ key: 'from_expression'
+ value: string
+ This value will be passed through to SQLExpression. If it is
+ provided, this ComplexQuery must have no subquery (regular
+ SQLExpression limitation)
+ implicit_table_list (list of strings)
+ Each entry in this list will be registered to column map. This is
+ used to make column mapper choose tables differently.
"""
- sql_expression_list = []
- select_expression_list = []
- for query in self.getQueryList():
- if isinstance(query, basestring):
- sql_expression_list.append(query)
+ self.logical_operator = kw.pop('logical_operator', kw.pop('operator', 'and')).lower()
+ assert self.logical_operator in ('and', 'or', 'not'), self.logical_operator
+ unknown_column_dict = kw.pop('unknown_column_dict', {})
+ self.from_expression = unknown_column_dict.pop('from_expression', None)
+ self.implicit_table_list = kw.pop('implicit_table_list', [])
+ query_list = []
+ append = query_list.append
+ extend = query_list.extend
+ # Flaten the first level of list-type arguments
+ for arg in args:
+ if isinstance(arg, (list, tuple)):
+ extend(arg)
else:
- query_result = query.asSQLExpression(key_alias_dict=key_alias_dict,
- ignore_empty_string=ignore_empty_string,
- keyword_search_keys=keyword_search_keys,
- datetime_search_keys=datetime_search_keys,
- full_text_search_keys=full_text_search_keys,
- stat__=stat__)
- sql_expression_list.append(query_result['where_expression'])
- select_expression_list.extend(query_result['select_expression_list'])
- operator = self.getOperator()
- result = {'where_expression':('(%s)' % \
- (' %s ' % operator).join(['(%s)' % x for x in sql_expression_list])),
- 'select_expression_list':select_expression_list}
+ append(arg)
+ new_query_list = []
+ append = new_query_list.append
+ # Iterate over the flaten argument list to cast each into a query type.
+ for query in query_list:
+ if not isinstance(query, Query):
+ query = SQLQuery(query)
+ append(query)
+ self.query_list = new_query_list
+
+ @profiler_decorator
+ def asSearchTextExpression(self, sql_catalog, column=None):
+ if column in (None, ''):
+ query_column = column
+ else:
+ query_column = ''
+ search_text_list = [y for y in [x.asSearchTextExpression(sql_catalog, column=query_column) for x in self.query_list] if y is not None]
+ if len(search_text_list) == 0:
+ result = ''
+ else:
+ if self.logical_operator in ('and', 'or'):
+ if len(search_text_list) == 1:
+ result = search_text_list[0]
+ else:
+ logical_operator = ' %s ' % (self.logical_operator.upper(), )
+ result = '(%s)' % (logical_operator.join(search_text_list), )
+ elif self.logical_operator == 'not':
+ assert len(search_text_list) == 1
+ result = '(NOT %s)' % (search_text_list[0], )
+ else:
+ raise ValueError, 'Unknown operator %r' % (self.logical_operator, )
+ if column not in (None, ''):
+ result = '%s:%s' % (column, result)
return result
- def getSQLKeyList(self):
+ @profiler_decorator
+ def asSQLExpression(self, sql_catalog, column_map, only_group_columns):
+ sql_expression_list = [x.asSQLExpression(sql_catalog, column_map, only_group_columns)
+ for x in self.query_list]
+ if len(sql_expression_list) == 0:
+ sql_expression_list = [SQLExpression(self, where_expression='1')]
+ return SQLExpression(self,
+ sql_expression_list=sql_expression_list,
+ where_expression_operator=self.logical_operator,
+ from_expression=self.from_expression)
+
+ @profiler_decorator
+ def registerColumnMap(self, sql_catalog, column_map):
+ for implicit_table_column in self.implicit_table_list:
+ column_map.registerColumn(implicit_table_column)
+ for query in self.query_list:
+ query.registerColumnMap(sql_catalog, column_map)
+
+ def __repr__(self):
+ return '<%s of %r.join(%r)>' % (self.__class__.__name__, self.logical_operator, self.query_list)
+
+ @profiler_decorator
+ def setTableAliasList(self, table_alias_list):
"""
- Returns the list of keys used by this
- instance
+ This function is here for backward compatibility.
+ This can only be used when there is one and only one subquery which
+ defines a setTableAliasList method.
+
+ See RelatedQuery.
"""
- key_list=[]
- for query in self.getQueryList():
- if not(isinstance(query, basestring)):
- key_list.extend(query.getSQLKeyList())
- return key_list
+ assert len(self.query_list) == 1
+ self.query_list[0].setTableAliasList(table_alias_list)
+
+ @profiler_decorator
+ def setGroup(self, group):
+ for query in self.query_list:
+ query.setGroup(group)
+
+verifyClass(IQuery, ComplexQuery)
-allow_class(ComplexQuery)
diff --git a/product/ZSQLCatalog/Query/EntireQuery.py b/product/ZSQLCatalog/Query/EntireQuery.py
new file mode 100644
index 0000000000000000000000000000000000000000..37eee743dd70fdafbf4d8b05e42deb07cd2f42da
--- /dev/null
+++ b/product/ZSQLCatalog/Query/EntireQuery.py
@@ -0,0 +1,182 @@
+##############################################################################
+#
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+from Products.ZSQLCatalog.SQLExpression import SQLExpression
+from Products.ZSQLCatalog.ColumnMap import ColumnMap
+from zLOG import LOG
+from Products.ZSQLCatalog.Interface.IEntireQuery import IEntireQuery
+from Interface.Verify import verifyClass
+from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
+
+class EntireQuery(object):
+ """
+ This is not a Query subclass, since it does not define a
+ registerColumnMap method, and instead does the ColumnMap handling
+ internaly.
+ """
+
+ __implements__ = IEntireQuery
+
+ column_map = None
+
+ @profiler_decorator
+ def __init__(self, query, order_by_list=None, group_by_list=None,
+ select_dict=None, limit=None, catalog_table_name=None,
+ extra_column_list=None, from_expression=None,
+ order_by_override_list=None):
+ def default(value):
+ if value is None:
+ return []
+ assert isinstance(value, (tuple, list))
+ return value
+ def defaultDict(value):
+ if value is None:
+ return {}
+ assert isinstance(value, dict)
+ return value
+ self.query = query
+ self.order_by_list = default(order_by_list)
+ self.order_by_override_set = frozenset(default(order_by_override_list))
+ self.group_by_list = default(group_by_list)
+ self.select_dict = defaultDict(select_dict)
+ self.limit = limit
+ self.catalog_table_name = catalog_table_name
+ self.extra_column_list = default(extra_column_list)
+ self.from_expression = from_expression
+
+ def asSearchTextExpression(self, sql_catalog):
+ return query.asSearchTextExpression(sql_catalog)
+
+ @profiler_decorator
+ def asSQLExpression(self, sql_catalog, only_group_columns):
+ column_map = self.column_map
+ if column_map is None:
+ # XXX: should we provide a way to register column map as a separate mathod or do it here ?
+ # Column Map was not built yet, do it.
+ self.column_map = column_map = ColumnMap(catalog_table_name=self.catalog_table_name)
+ for extra_column in self.extra_column_list:
+ table, column = extra_column.replace('`', '').split('.')
+ if table != self.catalog_table_name:
+ raise ValueError, 'Extra columns must be catalog columns. %r does not follow this rule (catalog=%r, extra_column_list=%r)' % (extra_column, self.catalog_table_name, self.extra_column_list)
+ column_map.registerColumn(extra_column)
+ for column in self.group_by_list:
+ column_map.registerColumn(column)
+ for alias, column in self.select_dict.iteritems():
+ if column is None:
+ column = alias
+ else:
+ column_map.ignoreColumn(alias)
+ column_map.registerColumn(column)
+ for override in self.order_by_override_set:
+ column_map.ignoreColumn(override)
+ for order_by in self.order_by_list:
+ assert isinstance(order_by, (tuple, list))
+ assert len(order_by)
+ column_map.registerColumn(order_by[0])
+ self.query.registerColumnMap(sql_catalog, column_map)
+ column_map.build(sql_catalog)
+ # Replace given group_by_list entries by their mapped representations.
+ new_column_list = []
+ append = new_column_list.append
+ for column in self.group_by_list:
+ try:
+ append(column_map.asSQLColumn(column))
+ except KeyError:
+ LOG('EntireQuery', 100, 'Group-by column %r could not be mapped, but is passed through. This use is strongly discouraged.' % (column, ))
+ append(column)
+ self.group_by_list = new_column_list
+ # Build a dictionnary from select_dict aliasing their mapped representations
+ self.final_select_dict = select_dict = {}
+ for alias, raw_column in self.select_dict.iteritems():
+ if raw_column is None:
+ column = alias
+ else:
+ column = raw_column
+ try:
+ rendered = column_map.asSQLColumn(column)
+ except KeyError:
+ LOG('EntireQuery', 100, 'Select column %r could not be mapped, but is passed through. This use is strongly discouraged.' % (column, ))
+ rendered = column
+ select_dict[alias] = rendered
+ # Replace given order_by_list entries by their mapped representations.
+ new_order_by_list = []
+ append = new_order_by_list.append
+ for order_by in self.order_by_list:
+ column = order_by[0]
+ if column in self.order_by_override_set:
+ LOG('EntireQuery', 100, 'Order-by column %r is forcibly accepted. This use is strongly discouraged.' % (column, ))
+ rendered = column
+ else:
+ try:
+ rendered = column_map.asSQLColumn(column)
+ except KeyError:
+ LOG('SQLCatalog', 100, 'Order by %r ignored: it could not be mapped to a known column.' % (order_by, ))
+ rendered = None
+ if rendered is not None:
+ if len(order_by) > 1:
+ if len(order_by) > 2 and order_by[2] not in (None, ''):
+ rendered = 'CAST(%s AS %s)' % (rendered, order_by[2])
+ rendered = '%s %s' % (rendered, order_by[1])
+ append(rendered)
+ self.order_by_list = new_order_by_list
+ # generate SQLExpression from query
+ sql_expression_list = [self.query.asSQLExpression(sql_catalog, column_map, only_group_columns)]
+ # generate join expression based on column_map.getJoinTableAliasList
+ append = sql_expression_list.append
+ for join_query in column_map.iterJoinQueryList():
+ append(join_query.asSQLExpression(sql_catalog, column_map, only_group_columns))
+ join_table_list = column_map.getJoinTableAliasList()
+ if len(join_table_list):
+ # XXX: Is there any special rule to observe when joining tables ?
+ # Maybe we could check which column is a primary key instead of
+ # hardcoding "uid".
+ where_pattern = '`%s`.`uid` = `%%s`.`uid`' % \
+ (column_map.getCatalogTableAlias(), )
+ # XXX: It would cleaner from completeness point of view to use column
+ # mapper to render column, but makes code much more complex to just do
+ # a simple text rendering. If there is any reason why we should have
+ # those column in the mapper, then we should use the clean way.
+ append(SQLExpression(self, where_expression=' AND '.join(
+ where_pattern % (x, ) for x in join_table_list
+ )))
+ self.sql_expression_list = sql_expression_list
+ return SQLExpression(
+ self,
+ table_alias_dict=column_map.getTableAliasDict(),
+ from_expression=self.from_expression,
+ order_by_list=self.order_by_list,
+ group_by_list=self.group_by_list,
+ select_dict=self.final_select_dict,
+ limit=self.limit,
+ where_expression_operator='and',
+ sql_expression_list=self.sql_expression_list)
+
+verifyClass(IEntireQuery, EntireQuery)
+
diff --git a/product/ZSQLCatalog/Query/Query.py b/product/ZSQLCatalog/Query/Query.py
index 2d159bca490e64bacabc47d65847dcdd90717403..5e3689062b72aa1d32cac4e525fdb0f81cbfdc8c 100644
--- a/product/ZSQLCatalog/Query/Query.py
+++ b/product/ZSQLCatalog/Query/Query.py
@@ -1,7 +1,9 @@
##############################################################################
#
-# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
-# Ivan Tyagov <ivan@nexedi.com>
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
@@ -26,117 +28,35 @@
#
##############################################################################
-from DocumentTemplate.DT_Var import sql_quote
+from Products.ZSQLCatalog.Interface.IQuery import IQuery
+from Interface.Verify import verifyClass
-class QueryMixin:
+class Query(object):
"""
- Mixing class which implements methods which are
- common to all kinds of Queries
+ This is the base class of all kind of queries. Its only purpose is to be
+ able to distinguish any kind of value from a query.
"""
- operator = None
- format = None
- type = None
- def __call__(self, **kw):
- return self.asSQLExpression(**kw)
+ __implements__ = IQuery
+ __allow_access_to_unprotected_subobjects__ = 1
- def getOperator(self):
- return self.operator
-
- def getFormat(self):
- return self.format
-
- def getType(self):
- return self.type
-
- def getRange(self):
- return self.range
-
- def getTableAliasList(self):
- return self.table_alias_list
-
- def getSearchMode(self):
- """Search mode used for Full Text search
+ def asSQLExpression(self, sql_catalog, column_map, only_group_columns):
"""
- return self.search_mode
-
- def getSearchKey(self):
- """Search mode used for Full Text search
+ To enable SQL rendering, overload this method in a subclass.
"""
- return self.search_key
-
- def getKey(self):
- return self.key
+ raise TypeError, 'A %s cannot be rendered as an SQL expression.' % (self.__class__.__name__, )
- def getValue(self):
- return self.value
-
- def getOperator(self):
- return self.operator.upper().strip()
-
- def asSearchTextExpression(self):
- raise NotImplementedError
-
- def asSQLExpression(self, key_alias_dict=None,
- keyword_search_keys=None,
- datetime_search_keys=None,
- full_text_search_keys=None,
- ignore_empty_string=1, stat__=0):
+ def asSearchTextExpression(self, sql_catalog, column=None):
"""
- Return a dictionnary containing the keys and value types:
- 'where_expression': string
- 'select_expression_list': string
+ To enable Search Text rendering, overload this method in a subclass.
"""
- raise NotImplementedError
+ raise TypeError, 'A %s cannot be rendered as a SearchText expression.' % (self.__class__.__name__, )
- def getSQLKeyList(self):
- """
- Return a list of keys used by this query and its subqueries.
- """
- raise NotImplementedError
-
- def getRelatedTableMapDict(self):
+ def registerColumnMap(self, sql_catalog, column_map):
"""
- Return for each key used by this query (plus ones used by its
- subqueries) the table alias mapping.
+ This method must always be overloaded by subclasses.
"""
- raise NotImplementedError
+ raise NotImplementedError, '%s is incompeltely implemented.' % (self.__class__.__name__, )
- def _quoteSQLString(self, value):
- """Return a quoted string of the value.
- XXX: Left for backwards compatability!
- """
- format = self.getFormat()
- type = self.getType()
- if format is not None and type is not None:
- if type == 'date':
- if hasattr(value, 'strftime'):
- value = value.strftime(format)
- if isinstance(value, basestring):
- value = "STR_TO_DATE('%s','%s')" % (value, format)
- if type == 'float':
- # Make sure there is no space in float values
- value = value.replace(' ','')
- value = "'%s'" % value
- else:
- if getattr(value, 'ISO', None) is not None:
- value = "'%s'" % value.toZone('UTC').ISO()
- else:
- value = "'%s'" % sql_quote(str(value))
- return value
+verifyClass(IQuery, Query)
- def _quoteSQLKey(self, key):
- """Return a quoted string of the value.
- XXX: Left for backwards compatability!
- """
- format = self.getFormat()
- type = self.getType()
- if format is not None and type is not None:
- if type == 'date':
- key = "STR_TO_DATE(DATE_FORMAT(%s,'%s'),'%s')" % (key, format, format)
- if type == 'float':
- float_format = format.replace(' ','')
- if float_format.find('.') >= 0:
- precision = len(float_format.split('.')[1])
- key = "TRUNCATE(%s,%s)" % (key, precision)
- return key
diff --git a/product/ZSQLCatalog/Query/RelatedQuery.py b/product/ZSQLCatalog/Query/RelatedQuery.py
new file mode 100644
index 0000000000000000000000000000000000000000..ccfa4cf05d17c1a23eb4850ad822e3cd7f07a45e
--- /dev/null
+++ b/product/ZSQLCatalog/Query/RelatedQuery.py
@@ -0,0 +1,99 @@
+##############################################################################
+#
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+from Query import Query
+from Products.ZSQLCatalog.SQLExpression import SQLExpression
+from Products.ZSQLCatalog.Interface.IQuery import IQuery
+from Interface.Verify import verifyClass
+from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
+
+class RelatedQuery(Query):
+ """
+ A RelatedQuery represents the is a container for a join condition.
+ """
+ @profiler_decorator
+ def __init__(self, search_key, join_condition=None, table_alias_list=None):
+ """
+ search_key (SearchKey)
+ join_condition (Query)
+ If given, it will be registered and rendered by this query.
+
+ Deprecated
+ table_alias_list (list of 2-tuple of strings)
+ See setTableAliasList.
+ """
+ self.search_key = search_key
+ self.join_condition = join_condition
+ self.table_alias_list = table_alias_list
+
+ @profiler_decorator
+ def setTableAliasList(self, table_alias_list):
+ """
+ This function is here for backward compatibility.
+
+ table_alias_list (list of 2-tuples of strings)
+ Each 2-tuple contains the name of a related key parameter and the
+ table alias it must be mapped on (respectively).
+ """
+ self.table_alias_list = table_alias_list
+
+ @profiler_decorator
+ def asSearchTextExpression(self, sql_catalog, column=None):
+ assert column is None
+ join_condition = self.join_condition
+ if join_condition is None:
+ result = None
+ else:
+ result = join_condition.asSearchTextExpression(sql_catalog, column=self.search_key.getColumn())
+ return result
+
+ @profiler_decorator
+ def asSQLExpression(self, sql_catalog, column_map, only_group_columns):
+ sql_expression_list = [self.search_key.buildSQLExpression(sql_catalog, column_map, only_group_columns, self.group)]
+ join_condition = self.join_condition
+ if join_condition is not None:
+ sql_expression_list.append(join_condition.asSQLExpression(sql_catalog, column_map, only_group_columns))
+ return SQLExpression(self, sql_expression_list=sql_expression_list, where_expression_operator='and')
+
+ @profiler_decorator
+ def registerColumnMap(self, sql_catalog, column_map):
+ self.group = self.search_key.registerColumnMap(column_map, table_alias_list=self.table_alias_list)
+ join_condition = self.join_condition
+ if join_condition is not None:
+ # Update its group
+ join_condition.setGroup(self.group)
+ # Propagate registration to embeded query
+ join_condition.registerColumnMap(sql_catalog, column_map)
+
+ def __repr__(self):
+ return '<%s on %r with %r>' % (self.__class__.__name__, self.search_key.getColumn(), self.join_condition)
+
+verifyClass(IQuery, RelatedQuery)
+
diff --git a/product/ZSQLCatalog/Query/SQLQuery.py b/product/ZSQLCatalog/Query/SQLQuery.py
new file mode 100644
index 0000000000000000000000000000000000000000..40546105037091c289b81ecaa75088817bd003fc
--- /dev/null
+++ b/product/ZSQLCatalog/Query/SQLQuery.py
@@ -0,0 +1,68 @@
+##############################################################################
+#
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+from Query import Query
+from Products.ZSQLCatalog.SQLExpression import SQLExpression
+from Products.ZSQLCatalog.Interface.IQuery import IQuery
+from Interface.Verify import verifyClass
+
+class SQLQuery(Query):
+ """
+ This Query subclass is used to wrap raw SQL text.
+ Use of this class is strongly discouraged, and it is only here for
+ backward compatibility.
+ """
+ def __init__(self, payload):
+ """
+ payload (string)
+ Raw SQL text.
+ """
+ if not isinstance(payload, basestring):
+ raise TypeError, 'Payload must be a string, got a %r: %r' % (type(payload), payload)
+ assert len(payload)
+ self.payload = '(' + payload + ')'
+
+ def asSearchText(self, sql_catalog):
+ return None
+
+ def asSQLExpression(self, sql_catalog, column_map, only_group_columns):
+ return SQLExpression(self, where_expression=self.payload)
+
+ def registerColumnMap(self, sql_catalog, column_map):
+ """
+ There is nothing to register for this type of Query subclass.
+ """
+ pass
+
+ def __repr__(self):
+ return '<%s (%r)>' % (self.__class__.__name__, self.payload)
+
+verifyClass(IQuery, SQLQuery)
+
diff --git a/product/ZSQLCatalog/Query/SimpleQuery.py b/product/ZSQLCatalog/Query/SimpleQuery.py
index 39966a317424a73934244bed00b00bc5fcbeacaf..21849e246005360da39bad17a7cf6026811d1a1e 100644
--- a/product/ZSQLCatalog/Query/SimpleQuery.py
+++ b/product/ZSQLCatalog/Query/SimpleQuery.py
@@ -1,7 +1,9 @@
##############################################################################
#
-# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
-# Ivan Tyagov <ivan@nexedi.com>
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
@@ -26,282 +28,104 @@
#
##############################################################################
-from Products.PythonScripts.Utility import allow_class
-from DateTime import DateTime
-from Query import QueryMixin
-from pprint import pprint
+from Query import Query
+from Products.ZSQLCatalog.SQLExpression import SQLExpression
+from Products.ZSQLCatalog.Interface.IQuery import IQuery
+from Interface.Verify import verifyClass
+from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
-# valid search modes for queries
-FULL_TEXT_SEARCH_MODE = 'FullText'
-EXACT_MATCH_SEARCH_MODE = 'ExactMatch'
-KEYWORD_SEARCH_MODE = 'Keyword'
-DATETIME_SEARCH_MODE = 'DateTime'
-
-def isSimpleType(value):
- return isinstance(value, basestring) or \
- isinstance(value, int) or \
- isinstance(value, long) or \
- isinstance(value, float)
-
-# XXX Bad name JPS - NotQuery or NegativeQuery is better NegationQuery
-class NegatedQuery(QueryMixin):
+class SimpleQuery(Query):
"""
- Do a boolean negation of given query.
+ A SimpleQuery represents a single comparison between a single column and
+ one or more values.
"""
+ @profiler_decorator
+ def __init__(self, search_key=None, operator='=', group=None, **kw):
+ """
+ search_key (None, SearchKey instance)
+ If given, the instance of SearchKey which is responsible for column
+ map registration and rendering (SQL and SearchText).
+ operator (string)
+ The comparison operator which will be applied between column and
+ values.
+ See Operator/ComparisonOperator.py for possible values.
+ group
+ See ColumnMap.
+ **kw
+ Must contain exactly one item.
+ item key (string)
+ column name
+ item value
+ one or more values
+ """
+ self.search_key = search_key
+ if len(kw) != 1:
+ raise ValueError, 'SimpleQuery can support one and one only column. Got %r.' % (kw, )
+ self.column, value = kw.popitem()
+ # Backward compatibility code (those changes should not be needed when
+ # this Query is instanciated by a SearchKey, as operator should be correct
+ # already).
+ operator = operator.lower()
+ if operator == 'in':
+ if isinstance(value, (list, tuple)):
+ if len(value) == 0:
+ raise ValueError, 'Empty lists are not allowed.'
+ elif len(value) == 1:
+ value = value[0]
+ operator = '='
+ else:
+ operator = '='
+ elif operator == '=':
+ if isinstance(value, (list, tuple)):
+ if len(value) == 0:
+ raise ValueError, 'Empty lists are not allowed.'
+ elif len(value) == 1:
+ value = value[0]
+ else:
+ operator = 'in'
+ self.value = value
+ self.operator = operator
+ self.group = group
- def __init__(self, query):
- self._query = query
-
- def asSQLExpression(self, *args, **kw):
- sql_expression_dict = self._query.asSQLExpression(*args, **kw)
- sql_expression_dict['where_expression'] = '(NOT (%s))' % \
- (sql_expression_dict['where_expression'], )
- return sql_expression_dict
-
- def getSQLKeyList(self, *args, **kw):
- return self._query.getSQLKeyList(*args, **kw)
-
- def getRelatedTableMapDict(self, *args, **kw):
- return self._query.getRelatedTableMapDict(*args, **kw)
-
-allow_class(NegatedQuery)
+ @profiler_decorator
+ def asSearchTextExpression(self, sql_catalog, column=None):
+ return self.getSearchKey(sql_catalog).buildSearchTextExpression(self.getOperator(sql_catalog), self.getValue(), column=column)
-class SimpleQuery(QueryMixin):
- """
- This allow to define constraints on a sql column
+ @profiler_decorator
+ def asSQLExpression(self, sql_catalog, column_map, only_group_columns):
+ return self.getSearchKey(sql_catalog).buildSQLExpression(
+ self.getOperator(sql_catalog), self.getValue(),
+ column_map, only_group_columns, group=self.group)
- format - type date : %d/%m/%Y
- type float : 1 234.12
- """
-
- def __init__(self, format=None, operator=None, range=None, key=None,
- search_mode=None, table_alias_list=None, type=None, **kw):
- self.format = format
- if operator is None:
- operator = 'OR'
- self.operator = operator
- self.range = range
- self.search_mode = search_mode
- self.table_alias_list = table_alias_list
- key_list = kw.keys()
- if len(key_list) != 1:
- raise KeyError, 'Query must have only one key'
- self.key = key_list[0]
- self.value = kw[self.key]
- self.type = type
- self.search_key = key
+ @profiler_decorator
+ def registerColumnMap(self, sql_catalog, column_map):
+ self.group = self.getSearchKey(sql_catalog).registerColumnMap(column_map, group=self.group, simple_query=self)
- def getRelatedTableMapDict(self):
- result = {}
- table_alias_list = self.getTableAliasList()
- if table_alias_list is not None:
- result[self.getKey()] = table_alias_list
- return result
-
- def getSQLKeyList(self):
+ def getOperator(self, sql_catalog):
"""
- Returns the list of keys used by this
- instance
+ Return an instance of OperatorBase class.
"""
- return [self.getKey()]
-
- def asSearchTextExpression(self):
- # This will be the standard way to represent
- # complex values in listbox. Some fixed
- # point must be garanteed
- value = self.getValue()
- if isSimpleType(value) or isinstance(value, DateTime):
- return str(value)
- elif isinstance(value, (list, tuple)):
- value = map(lambda x:str(x), value)
- return (' %s ' % self.operator).join(value)
-
- def _getSearchKeyClassByType(self, type, search_key_class = None):
- """ Return search key class based on type of value. """
- name_search_key_map = {'keyword': KeyWordKey,
- 'default': DefaultKey,
- 'fulltext': FullTextKey,
- 'date': DateTimeKey,
- 'float': FloatKey,
- 'int': DefaultKey,}
- return name_search_key_map.get(type, search_key_class)
-
- def _getSearchKeyClassByValue(self, value, search_key_class = None):
- """ Return search key class based on type of value. """
- if isinstance(value, basestring):
- if value.find('%')!=-1:
- # it's likely a KeyWordKey
- search_key_class = KeyWordKey
- else:
- search_key_class = DefaultKey
- elif isinstance(value, DateTime):
- search_key_class = DateTimeKey
- elif isinstance(value, (int, long,)):
- search_key_class = DefaultKey
- elif isinstance(value, float):
- search_key_class = FloatKey
- elif value is None:
- return RawKey
- return search_key_class
+ return sql_catalog.getComparisonOperator(self.operator)
- def _asSQLExpression(self, search_key_class, key, value, format=None, mode=None, range_value=None, stat__=None):
- """ Generate SQL expressions based on respective search_key passed. """
- lexer = getSearchKeyInstance(search_key_class)
- where_expression, select_expression_list = \
- lexer.buildSQLExpression(key, value, format, mode, range_value, stat__)
- sql_expressions = {'where_expression': where_expression,
- 'select_expression_list': select_expression_list,}
- return sql_expressions
-
- def asSQLExpression(self, key_alias_dict=None, keyword_search_keys=None,
- datetime_search_keys=None, full_text_search_keys=None,
- ignore_empty_string=1, stat__=0):
+ def getSearchKey(self, sql_catalog):
"""
- Build the sql expressions string
+ Return an instance of SearchKey class.
"""
- search_key_class = None
- value = self.getValue()
- key = self.getKey()
- operator = self.getOperator()
- type = self.getType()
- format = self.getFormat()
- search_mode = self.getSearchMode()
- range_value = self.getRange()
- search_key = self.getSearchKey()
+ if self.search_key is None:
+ self.search_key = sql_catalog.getSearchKey(self.getColumn())
+ return self.search_key
- if keyword_search_keys is None:
- keyword_search_keys = []
- if datetime_search_keys is None:
- datetime_search_keys = []
- if full_text_search_keys is None:
- full_text_search_keys = []
+ def getColumn(self):
+ return self.column
- # key can have an alias definition which we should acquire
- if key_alias_dict is not None:
- key = key_alias_dict.get(key, None)
+ def getValue(self):
+ return self.value
- search_key_class = None
- where_expression_list = []
- select_expression_list = []
- sql_expressions = {'where_expression': '1',
- 'select_expression_list': []}
-
- # try to get search key type by the key definitions passed
- if search_key_class is None:
- if search_key == EXACT_MATCH_SEARCH_MODE:
- search_key_class = RawKey
- elif search_key == KEYWORD_SEARCH_MODE or \
- (key in keyword_search_keys):
- search_key_class = KeyWordKey
- elif search_key == DATETIME_SEARCH_MODE or \
- (key in datetime_search_keys):
- search_key_class = DateTimeKey
- elif search_key == FULL_TEXT_SEARCH_MODE or \
- (key in full_text_search_keys):
- search_key_class = FullTextKey
-
- # get search class based on explicitly passed key type
- if search_key_class is None:
- search_key_class = self._getSearchKeyClassByType(type)
-
- # some use cases where we can just return SQL without grammar staff
- if key is None or (ignore_empty_string and \
- isinstance(value, basestring) and \
- value.strip() == ''):
- # do not further generate sql expressions because
- # we ignore empty strings by default
- return sql_expressions
- elif (not ignore_empty_string) \
- and isinstance(value, basestring) \
- and value.strip() == '':
- # explicitly requested not to ignore empty strings
- sql_expressions = {'where_expression': "%s = ''" % key,
- 'select_expression_list': []}
- return sql_expressions
- else:
- # search for 'NULL' values
- if value is None:
- sql_expressions = {'where_expression': "%s is NULL" % (key),
- 'select_expression_list': [],}
- return sql_expressions
-
- # we have a list of values and respective operator defined
- if isinstance(value, (tuple, list)):
- if range_value is None:
- # use operators to build sql expressions
- if operator in ('IN',):
- # values in list are not treated as searchable strings but
- # they should be SQL quoted at least
- if len(value) > 1:
- if search_key_class is None:
- # no explicitly defined, try to find by value
- search_key_class = self._getSearchKeyClassByValue(value[0])
- search_key_instance = getSearchKeyInstance(search_key_class)
- escaped_value_list = [search_key_instance.quoteSQLString(x, format) for x in value]
- escaped_value_string = ', '.join(escaped_value_list)
- where_expression_list.append("%s IN (%s)" % (key, escaped_value_string))
- elif len(value) == 1:
- if search_key_class is None:
- # no explicitly defined, try to find by value
- search_key_class = self._getSearchKeyClassByValue(value[0])
- search_key_instance = getSearchKeyInstance(search_key_class)
- where_expression_list.append("%s = %s"
- %(key, search_key_instance.quoteSQLString(value[0], format)))
- else:
- # empty list
- where_expression_list.append("0")
- elif operator in ('OR', 'AND',):
- # each of the list elements can be treated as a Key, so
- # leave SQL generation to Key itself
- if len(value) > 1:
- sql_logical_sub_expressions = []
- if search_key_class is None:
- # no explicitly defined, try to find by value
- search_key_class = self._getSearchKeyClassByValue(value[0])
- for item in value:
- list_item_sql_expressions = self._asSQLExpression(search_key_class, key, \
- item, format, search_mode, range_value, stat__)
- sql_logical_sub_expressions.append('%s' %list_item_sql_expressions['where_expression'])
- # join list items (now sql logical expressions) using respective operator
- where_expression = (' %s ' %operator).join(sql_logical_sub_expressions)
- where_expression_list.append("(%s)" % (where_expression))
- elif len(value) == 1:
- if search_key_class is None:
- # no explicitly defined, try to find by value
- search_key_class = self._getSearchKeyClassByValue(value[0])
- item_sql_expressions = self._asSQLExpression(search_key_class, key, \
- value[0], format, search_mode, range_value, stat__)
- where_expression_list.append(item_sql_expressions['where_expression'])
- # join where expressions list
- where_expression = ' '.join(where_expression_list)
- sql_expressions = {'where_expression': where_expression,
- 'select_expression_list': [],}
- return sql_expressions
- else:
- # we can have range specified
- if search_key_class is None:
- # try to guess by type of first_element in list
- search_key_class = self._getSearchKeyClassByValue(value[0])
-
- # get search class based on value of value
- if search_key_class is None:
- search_key_class = self._getSearchKeyClassByValue(value)
-
- # last fallback case
- if search_key_class is None:
- search_key_class = DefaultKey
-
- # use respective search key.
- sql_expressions = self._asSQLExpression(search_key_class, key,
- value, format, search_mode, range_value, stat__)
- return sql_expressions
+ def __repr__(self):
+ return '<%s %r %s %r>' % (self.__class__.__name__, self.getColumn(), self.operator, self.getValue())
+
+ def setGroup(self, group):
+ self.group = group
-allow_class(SimpleQuery)
+verifyClass(IQuery, SimpleQuery)
-from Products.ZSQLCatalog.SearchKey.DefaultKey import DefaultKey
-from Products.ZSQLCatalog.SearchKey.RawKey import RawKey
-from Products.ZSQLCatalog.SearchKey.KeyWordKey import KeyWordKey
-from Products.ZSQLCatalog.SearchKey.DateTimeKey import DateTimeKey
-from Products.ZSQLCatalog.SearchKey.FullTextKey import FullTextKey
-from Products.ZSQLCatalog.SearchKey.FloatKey import FloatKey
-from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance
diff --git a/product/ZSQLCatalog/SQLCatalog.py b/product/ZSQLCatalog/SQLCatalog.py
index 009b0391874033f79cd2d3a4e629d383057e0828..d7006f05e1a27de807debfb26261a01672a4958b 100644
--- a/product/ZSQLCatalog/SQLCatalog.py
+++ b/product/ZSQLCatalog/SQLCatalog.py
@@ -1,6 +1,6 @@
##############################################################################
#
-# Copyright (c) 2002 Nexedi SARL. All Rights Reserved.
+# Copyright (c) 2002-2009 Nexedi SARL. All Rights Reserved.
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
@@ -45,7 +45,16 @@ from xml.dom.minidom import parse
from xml.sax.saxutils import escape, quoteattr
import os
import md5
-from sets import ImmutableSet
+
+from Interface.IQueryCatalog import ISearchKeyCatalog
+from Interface.Verify import verifyClass
+
+PROFILING_ENABLED = False
+if PROFILING_ENABLED:
+ from tiny_profiler import profiler_decorator, profiler_report, profiler_reset
+else:
+ def profiler_decorator(func):
+ return func
try:
from Products.CMFCore.Expression import Expression
@@ -64,6 +73,7 @@ try:
from Products.ERP5Type.Cache import enableReadOnlyTransactionCache
from Products.ERP5Type.Cache import disableReadOnlyTransactionCache, CachingMethod
except ImportError:
+ LOG('SQLCatalog', 100, 'Count not import CachingMethod, expect slowness.')
def doNothing(context):
pass
class CachingMethod:
@@ -76,15 +86,53 @@ except ImportError:
return self.function(*opts, **kw)
enableReadOnlyTransactionCache = doNothing
disableReadOnlyTransactionCache = doNothing
-
+
+class caching_class_method_decorator:
+ def __init__(self, *args, **kw):
+ self.args = args
+ self.kw = kw
+
+ def __call__(self, method):
+ caching_method = CachingMethod(method, *self.args, **self.kw)
+ return lambda *args, **kw: caching_method(*args, **kw)
+ #def wrapper(wrapped_self):
+ # LOG('caching_class_method_decorator', 0, 'lookup')
+ # return caching_method(wrapped_self)
+ #return wrapper
+
+try:
+ from Products.ERP5Type.TransactionalVariable import getTransactionalVariable
+except ImportError:
+ LOG('SQLCatalog', 100, 'Count not import getTransactionalVariable, expect slowness.')
+ def getTransactionalVariable(context):
+ return {}
+
+class transactional_cache_decorator:
+ """
+ Implements singleton-style caching.
+ Wrapped method must have no parameters (besides "self").
+ """
+ def __init__(self, cache_id):
+ self.cache_id = cache_id
+
+ def __call__(self, method):
+ def wrapper(wrapped_self):
+ transactional_cache = getTransactionalVariable(None)
+ try:
+ return transactional_cache[self.cache_id]
+ except KeyError:
+ result = transactional_cache[self.cache_id] = method(wrapped_self)
+ return result
+ return wrapper
+
try:
from ZPublisher.HTTPRequest import record
except ImportError:
dict_type_list = (dict, )
else:
dict_type_list = (dict, record)
-
-
+
+
UID_BUFFER_SIZE = 300
OBJECT_LIST_SIZE = 300
MAX_PATH_LEN = 255
@@ -184,6 +232,10 @@ class UidBuffer(TM):
tid = get_ident()
self.temporary_buffer.setdefault(tid, []).extend(iterable)
+DEBUG = False
+
+related_key_definition_cache = {}
+
class Catalog(Folder,
Persistent,
Acquisition.Implicit,
@@ -219,6 +271,9 @@ class Catalog(Folder,
- optmization: indexing objects should be deferred
until timeout value or end of transaction
"""
+
+ __implements__ = ISearchKeyCatalog
+
meta_type = "SQLCatalog"
icon = 'misc_/ZCatalog/ZCatalog.gif' # FIXME: use a different icon
security = ClassSecurityInfo()
@@ -463,6 +518,13 @@ class Catalog(Folder,
'a monovalued local role',
'type': 'lines',
'mode': 'w' },
+ { 'id': 'sql_catalog_table_vote_scripts',
+ 'title': 'Table vote scripts',
+ 'description': 'Scripts helping column mapping resolution',
+ 'type': 'multiple selection',
+ 'select_variable' : 'getPythonMethodIds',
+ 'mode': 'w' },
+
)
sql_catalog_produce_reserved = ''
@@ -500,6 +562,7 @@ class Catalog(Folder,
sql_catalog_scriptable_keys = ()
sql_catalog_role_keys = ()
sql_catalog_local_role_keys = ()
+ sql_catalog_table_vote_scripts = ()
# These are ZODB variables, so shared by multiple Zope instances.
# This is set to the last logical time when clearReserved is called.
@@ -887,26 +950,24 @@ class Catalog(Folder,
keys = keys.keys()
keys.sort()
return keys
- return CachingMethod(_getColumnIds, id='SQLCatalog.getColumnIds', cache_factory='erp5_content_long')()
+ return CachingMethod(_getColumnIds, id='SQLCatalog.getColumnIds', cache_factory='erp5_content_long')()[:]
+ @profiler_decorator
+ @transactional_cache_decorator('SQLCatalog.getColumnMap')
+ @profiler_decorator
+ @caching_class_method_decorator(id='SQLCatalog.getColumnMap', cache_factory='erp5_content_long')
+ @profiler_decorator
def getColumnMap(self):
"""
Calls the show column method and returns dictionnary of
Field Ids
"""
- def _getColumnMap():
- keys = {}
- for table in self.getCatalogSearchTableIds():
- field_list = self._getCatalogSchema(table=table)
- for field in field_list:
- key = field
- if not keys.has_key(key): keys[key] = []
- keys[key].append(table)
- key = '%s.%s' % (table, key)
- if not keys.has_key(key): keys[key] = []
- keys[key].append(table) # Is this inconsistent ?
- return keys
- return CachingMethod(_getColumnMap, id='SQLCatalog.getColumnMap', cache_factory='erp5_content_long')()
+ result = {}
+ for table in self.getCatalogSearchTableIds():
+ for field in self._getCatalogSchema(table=table):
+ result.setdefault(field, []).append(table)
+ result.setdefault('%s.%s' % (table, field), []).append(table) # Is this inconsistent ?
+ return result
def getResultColumnIds(self):
"""
@@ -1332,7 +1393,7 @@ class Catalog(Folder,
if self.isMethodFiltered(method_name):
catalogged_object_list = []
filter = self.filter_dict[method_name]
- type_set = ImmutableSet(filter['type']) or None
+ type_set = frozenset(filter['type']) or None
expression = filter['expression_instance']
expression_cache_key_list = filter.get('expression_cache_key', '').split()
for object in object_list:
@@ -1633,7 +1694,8 @@ class Catalog(Folder,
""" Accesses a single record for a given path """
return self.getMetadataForPath(path)
- def getCatalogMethodIds(self):
+ def getCatalogMethodIds(self,
+ valid_method_meta_type_list=valid_method_meta_type_list):
"""Find Z SQL methods in the current folder and above
This function return a list of ids.
"""
@@ -1658,6 +1720,26 @@ class Catalog(Folder,
ids.sort()
return ids
+ def getPythonMethodIds(self):
+ """
+ Returns a list of all python scripts available in
+ current sql catalog.
+ """
+ return self.getCatalogMethodIds(valid_method_meta_type_list=('Script (Python)', ))
+
+ @profiler_decorator
+ @transactional_cache_decorator('SQLCatalog._getSQLCatalogRelatedKeyList')
+ @profiler_decorator
+ def _getSQLCatalogRelatedKeySet(self):
+ column_map = self.getColumnMap()
+ column_set = set(column_map)
+ for related_key in self.sql_catalog_related_keys:
+ related_key_id = related_key.split(' | ')[0].strip()
+ if related_key_id in column_set:
+ LOG('SQLCatalog', 100, 'Related key %r has the same name as an existing column on tables %r' % (related_key_id, column_map[related_key_id]))
+ column_set.add(related_key_id)
+ return column_set
+
def getSQLCatalogRelatedKeyList(self, key_list=None):
"""
Return the list of related keys.
@@ -1666,10 +1748,9 @@ class Catalog(Folder,
"""
if key_list is None:
key_list = []
+ column_map = self._getSQLCatalogRelatedKeySet()
# Do not generate dynamic related key for acceptable_keys
- dynamic_key_list = [k for k in key_list \
- if k not in self.getColumnMap().keys()]
-
+ dynamic_key_list = [k for k in key_list if k not in column_map]
dynamic_list = self.getDynamicRelatedKeyList(dynamic_key_list)
full_list = list(dynamic_list) + list(self.sql_catalog_related_keys)
return full_list
@@ -1702,486 +1783,412 @@ class Catalog(Folder,
%(table_index, table))
return table_index
return CachingMethod(_getTableIndex, id='SQLCatalog.getTableIndex', \
- cache_factory='erp5_content_long')(table=table)
-
-
- def getIndex(self, table, column_list, all_column_list):
- """
- Return possible index for a column list in a given table
- """
- def _getIndex(table, column_list, all_column_list):
- index_dict = self.getTableIndex(table)
- if isinstance(column_list, str):
- column_list = [column_list,]
- # Get possible that can be used
- possible_index = []
- for index in index_dict.keys():
- index_columns = index_dict[index]
- for column in index_columns:
- if column in column_list:
- if index not in possible_index:
- possible_index.append(index)
- if len(possible_index) == 0:
- return []
- # Get the most suitable index
- for index in possible_index:
- # Make sure all column in index are used by the query
- index_column = index_dict[index]
- for column in index_column:
- if column in column_list or column in all_column_list:
- continue
- else:
- possible_index.remove(index)
- LOG("SQLCatalog.getIndex", INFO, "index = %s for table %s and columns %s" \
- %(possible_index, table, column_list))
- return possible_index
- return CachingMethod(_getIndex, id='SQLCatalog.getIndex', cache_factory='erp5_content_long')\
- (table=table, column_list=column_list, all_column_list=all_column_list)
+ cache_factory='erp5_content_long')(table=table).copy()
+ @profiler_decorator
+ def getRelatedKeyDefinition(self, key):
+ """
+ Returns the definition of given related key name if found, None
+ otherwise.
+ """
+ try:
+ result = related_key_definition_cache[key]
+ except KeyError:
+ result = None
+ for entire_definition in self.getSQLCatalogRelatedKeyList([key]):
+ name, definition = entire_definition.split(' | ')
+ if name == key:
+ result = definition
+ break
+ if result is not None:
+ related_key_definition_cache[key] = result
+ return result
- def buildSQLQuery(self, query_table='catalog', REQUEST=None,
- ignore_empty_string=1, query=None, stat__=0, **kw):
- """ Builds a complex SQL query to simulate ZCatalog behaviour """
- # Get search arguments:
- if REQUEST is None and (kw is None or kw == {}):
- # We try to get the REQUEST parameter
- # since we have nothing handy
- try: REQUEST=self.REQUEST
- except AttributeError: pass
-
- #LOG('SQLCatalog.buildSQLQuery, kw',0,kw)
- # If kw and query are not set, then use REQUEST instead
- if query is None and (kw is None or kw == {}):
- kw = REQUEST
-
- acceptable_key_map = self.getColumnMap()
- full_text_search_keys = list(self.sql_catalog_full_text_search_keys)
- keyword_search_keys = list(self.sql_catalog_keyword_search_keys)
- datetime_search_keys = list(self.sql_catalog_datetime_search_keys)
- topic_search_keys = self.sql_catalog_topic_search_keys
- multivalue_keys = self.sql_catalog_multivalue_keys
-
-
- # Compute "sort_index", which is a sort index, or none:
- if kw.has_key('sort-on'):
- sort_index=kw['sort-on']
- elif hasattr(self, 'sort-on'):
- sort_index=getattr(self, 'sort-on')
- elif kw.has_key('sort_on'):
- sort_index=kw['sort_on']
- else: sort_index=None
-
- # Compute the sort order
- if kw.has_key('sort-order'):
- so=kw['sort-order']
- elif hasattr(self, 'sort-order'):
- so=getattr(self, 'sort-order')
- elif kw.has_key('sort_order'):
- so=kw['sort_order']
- else: so=None
-
- # We must now turn sort_index into
- # a dict with keys as sort keys and values as sort order
- if isinstance(sort_index, basestring):
- sort_index = [(sort_index, so)]
- elif not isinstance(sort_index, (list, tuple)):
- sort_index = None
-
- # Rebuild keywords to behave as new style query (_usage='toto:titi' becomes {'toto':'titi'})
- new_kw = {}
- usage_len = len('_usage')
- for k, v in kw.items():
- if k.endswith('_usage'):
- new_k = k[0:-usage_len]
- if not new_kw.has_key(new_k):
- new_kw[new_k] = {}
- if not isinstance(new_kw[new_k], dict_type_list):
- new_kw[new_k] = {'query': new_kw[new_k]}
- split_v = v.split(':')
- new_kw[new_k] = {split_v[0]: split_v[1]}
+ @profiler_decorator
+ def getColumnSearchKey(self, key, search_key_name=None):
+ """
+ Return a SearchKey instance for given key, using search_key_name
+ as a SearchKey name if given, otherwise guessing from catalog
+ configuration. If there is no search_key_name given and no
+ SearchKey can be found, return None.
+
+ Also return a related key definition string with following rules:
+ - If returned SearchKey is a RelatedKey, value is its definition
+ - Otherwise, value is None
+ """
+ # Is key a "real" column or some related key ?
+ related_key_definition = None
+ if key in self.getColumnMap():
+ search_key = self.getSearchKey(key, search_key_name)
+ else:
+ # Maybe a related key...
+ related_key_definition = self.getRelatedKeyDefinition(key)
+ if related_key_definition is None:
+ # Unknown
+ search_key = None
else:
- if not new_kw.has_key(k):
- new_kw[k] = v
- else:
- new_kw[k]['query'] = v
- kw = new_kw
-
- # Initialise Scriptable Dict
- scriptable_key_dict = {}
- for t in self.sql_catalog_scriptable_keys:
- t = t.split('|')
- key = t[0].strip()
- if len(t)>1:
- # method defined that will generate a ComplexQuery
- method_id = t[1].strip()
+ # It's a related key
+ search_key = self.getSearchKey(key, 'RelatedKey')
+ return search_key, related_key_definition
+
+ @profiler_decorator
+ def getColumnDefaultSearchKey(self, key):
+ """
+ Return a SearchKey instance which would ultimately receive the value
+ associated with given key.
+ """
+ search_key, related_key_definition = self.getColumnSearchKey(key)
+ if search_key is None:
+ result = None
+ else:
+ if related_key_definition is not None:
+ search_key = search_key.getSearchKey(self, related_key_definition)
+ return search_key
+
+ @profiler_decorator
+ def buildSingleQuery(self, key, value, search_key_name=None, logical_operator=None, comparison_operator=None):
+ """
+ From key and value, determine the SearchKey to use and generate a Query
+ from it.
+ """
+ search_key, related_key_definition = self.getColumnSearchKey(key, search_key_name)
+ if search_key is None:
+ result = None
+ else:
+ if related_key_definition is None:
+ result = search_key.buildQuery(value, logical_operator=logical_operator, comparison_operator=comparison_operator)
else:
- # no method define, let ScriptableKey generate a ComplexQuery
- method_id = None
- scriptable_key_dict[key] = method_id
-
- # Build the list of Queries and ComplexQueries
- query_dict = {}
- key_list = [] # the list of column keys
- key_alias_dict = {}
- query_group_by_list = None # Useful to keep a default group_by passed by scriptable keys
- query_related_table_map_dict = {}
- if query is not None:
- kw ['query'] = query
- for key in kw.keys():
- if key not in RESERVED_KEY_LIST:
- value = kw[key]
- current_query = None
- new_query_dict = {}
- if isinstance(value, (Query, ComplexQuery)):
- current_query = value
- elif scriptable_key_dict.has_key(key):
- if scriptable_key_dict[key] is not None:
- # Turn this key into a query by invoking a script
- method = getattr(self, scriptable_key_dict[key])
- current_query = method(value) # May return None
- else:
- # let default implementation of ScriptableKey generate ComplexQuery
- search_key_instance = getSearchKeyInstance(ScriptableKey)
- current_query = search_key_instance.buildQuery('', value)
- if hasattr(current_query, 'order_by'): query_group_by_list = current_query.order_by
+ result = search_key.buildQuery(search_value=value, sql_catalog=self, search_key_name=search_key_name, related_key_definition=related_key_definition, logical_operator=logical_operator, comparison_operator=comparison_operator)
+ return result
+
+ @profiler_decorator
+ def buildQueryFromAbstractSyntaxTreeNode(self, node, key):
+ """
+ Build a query from given Abstract Syntax Tree (AST) node by recursing in
+ its childs.
+ This method calls itself recursively when walking the tree.
+
+ node
+ AST node being treated.
+ key
+ Default column (used when there is no explicit column in an AST leaf).
+
+ Expected node API is described in Interface/IAbstractSyntaxNode.py .
+ """
+ if node.isLeaf():
+ result = self.buildSingleQuery(key, node.getValue(), comparison_operator=node.getComparisonOperator())
+ if result is None:
+ # Unknown, skip loudly
+ LOG('SQLCatalog', 100, 'Unknown column %r, skipped.' % (key, ))
+ elif node.isColumn():
+ result = self.buildQueryFromAbstractSyntaxTreeNode(node.getSubNode(), node.getColumnName())
+ else:
+ query_list = []
+ value_dict = {}
+ append = query_list.append
+ for subnode in node.getNodeList():
+ if subnode.isLeaf():
+ value_dict.setdefault(subnode.getComparisonOperator(), []).append(subnode.getValue())
else:
- if isinstance(value, dict_type_list):
- new_query_dict = value.copy()
- if 'query' in new_query_dict:
- new_query_dict[key] = new_query_dict.pop('query')
- else:
- new_query_dict[key] = value
- current_query = Query(**new_query_dict)
- if current_query is not None:
- query_dict[key] = current_query
- key_list.extend(current_query.getSQLKeyList())
- query_related_table_map_dict.update(current_query.getRelatedTableMapDict())
-
- # if we have a sort index, we must take it into account to get related
- # keys.
- sort_key_dict = dict()
- if sort_index:
- for sort_info in sort_index:
- sort_key = sort_info[0]
- if sort_key not in key_list:
- key_list.append(sort_key)
- sort_key_dict[sort_key] = 1
-
- related_tuples = self.getSQLCatalogRelatedKeyList(key_list=key_list)
-
- # Define related maps
- # each tuple from `related_tuples` has the form (key,
- # 'table1,table2,table3/column/where_expression')
- related_keys = {}
- related_method = {}
- related_table_map = {}
- related_column = {}
- related_table_list = {}
- table_rename_index = 0
- related_methods = {} # related methods which need to be used
- for t in related_tuples:
- t_tuple = t.split('|')
- key = t_tuple[0].strip()
- if key in key_list:
- if ignore_empty_string \
- and kw.get(key, None) in ('', [], ())\
- and key not in sort_key_dict:
- # We don't ignore 0 and None, but if the key is used for sorting,
- # we should not discard this key
- continue
- join_tuple = t_tuple[1].strip().split('/')
- related_keys[key] = None
- method_id = join_tuple[2]
- table_list = tuple(join_tuple[0].split(','))
- related_method[key] = method_id
- related_table_list[key] = table_list
- related_column[key] = join_tuple[1]
- # Check if some aliases where specified in queries
- map_list = query_related_table_map_dict.get(key,None)
- # Rename tables to prevent conflicts
- if not related_table_map.has_key((table_list,method_id)):
- if map_list is None:
- map_list = []
- for table_id in table_list:
- map_list.append((table_id,
- "related_%s_%s" % (table_id, table_rename_index))) # We add an index in order to alias tables in the join
- table_rename_index += 1 # and prevent name conflicts
- related_table_map[(table_list,method_id)] = map_list
-
- # We take additional parameters from the REQUEST
- # and give priority to the REQUEST
- if REQUEST is not None:
- for key in acceptable_key_map.iterkeys():
- if REQUEST.has_key(key):
- # Only copy a few keys from the REQUEST
- if key in self.sql_catalog_request_keys:
- kw[key] = REQUEST[key]
-
- def getNewKeyAndUpdateVariables(key):
- new_key = None
- if query_table:
- key_is_acceptable = key in acceptable_key_map # Only calculate once
- key_is_related = key in related_keys
- if key_is_acceptable or key_is_related:
- if key_is_related: # relation system has priority (ex. security_uid)
- # We must rename the key
- method_id = related_method[key]
- table_list = related_table_list[key]
- if not related_methods.has_key((table_list,method_id)):
- related_methods[(table_list,method_id)] = 1
- # Prepend renamed table name
- new_key = "%s.%s" % (related_table_map[(table_list,method_id)][-1][-1],
- related_column[key])
- elif key_is_acceptable:
- if key.find('.') < 0:
- # if the key is only used by one table, just append its name
- if len(acceptable_key_map[key]) == 1 :
- new_key = '%s.%s' % (acceptable_key_map[key][0], key)
- # query_table specifies what table name should be used by default
- elif '%s.%s' % (query_table, key) in acceptable_key_map:
- new_key = '%s.%s' % (query_table, key)
- elif key == 'uid':
- # uid is always ambiguous so we can only change it here
- new_key = 'catalog.uid'
- else:
- LOG('SQLCatalog', WARNING, 'buildSQLQuery this key is too ambiguous : %s' % key)
- else:
- new_key = key
- if new_key is not None:
- # Add table to table dict, we use catalog by default
- from_table_dict[acceptable_key_map[new_key][0]] = acceptable_key_map[new_key][0]
+ subquery = self.buildQueryFromAbstractSyntaxTreeNode(subnode, key)
+ if subquery is not None:
+ append(subquery)
+ for comparison_operator, value_list in value_dict.iteritems():
+ subquery = self.buildSingleQuery(key, value_list, comparison_operator=comparison_operator)
+ if subquery is None:
+ LOG('SQLCatalog', 100, 'Unknown column %r, skipped.' % (key, ))
+ else:
+ append(subquery)
+ operator = node.getLogicalOperator()
+ if operator == 'not' or len(query_list) > 1:
+ result = ComplexQuery(query_list, operator=operator)
+ elif len(query_list) == 1:
+ result = query_list[0]
else:
- new_key = key
- key_alias_dict[key] = new_key
- return new_key
-
- where_expression_list = []
- select_expression_list = []
- group_by_expression_list = []
- where_expression = ''
- select_expression = ''
- group_by_expression = ''
- select_expression_key = ''
-
- from_table_dict = {'catalog' : 'catalog'} # Always include catalog table
- if len(kw):
- if kw.has_key('select_expression_key'):
- select_expression_key = kw['select_expression_key']
- if type(select_expression_key) is type('a'):
- select_expression_key = [select_expression_key]
- if kw.has_key('select_expression'):
- select_expression_list.append(kw['select_expression'])
- if kw.has_key('group_by_expression'):
- group_by_expression_list.append(kw['group_by_expression'])
- # Grouping
- group_by_list = kw.get('group_by', query_group_by_list)
- if type(group_by_list) is type('a'): group_by_list = [group_by_list]
- if group_by_list is not None:
- try:
- for key in group_by_list:
- new_key = getNewKeyAndUpdateVariables(key)
- group_by_expression_list.append(new_key)
- except ConflictError:
- raise
- except:
- LOG('SQLCatalog', WARNING, 'buildSQLQuery could not build the new group by expression', error=sys.exc_info())
- group_by_expression = ''
- if len(group_by_expression_list)>0:
- group_by_expression = ','.join(group_by_expression_list)
- group_by_expression = str(group_by_expression)
-
- sort_on = None
- sort_key_list = []
- if sort_index is not None:
- new_sort_index = []
- for sort in sort_index:
- if len(sort) == 2:
- # Try to analyse expressions of the form "title AS unsigned"
- sort_key_list = sort[0].split()
- if len(sort_key_list) == 3:
- sort_key = sort_key_list[0]
- sort_type = sort_key_list[2]
- elif len(sort_key_list):
- sort_key = sort_key_list[0]
- sort_type = None
- else:
- sort_key = sort[0]
- sort_type = None
- new_sort_index.append((sort_key, sort[1], sort_type))
- elif len(sort) == 3:
- new_sort_index.append(sort)
- sort_index = new_sort_index
- try:
- new_sort_index = []
- for (original_key, so, as_type) in sort_index:
- key = getNewKeyAndUpdateVariables(original_key)
- if key is None:
- if original_key in select_expression_key:
- key = original_key
- if key is not None:
- sort_key_list.append(key)
- if as_type == 'int':
- key = 'CAST(%s AS SIGNED)' % key
- elif as_type:
- key = 'CAST(%s AS %s)' % (key, as_type) # Different casts are possible
- if so in ('descending', 'reverse', 'DESC'):
- new_sort_index.append('%s DESC' % key)
+ result = None
+ return result
+
+ @profiler_decorator
+ def buildQuery(self, kw, ignore_empty_string=True, operator='and'):
+ query_list = []
+ append = query_list.append
+ # unknown_column_dict: contains all (key, value) pairs which could not be
+ # changed into queries. This is here for backward compatibility, because
+ # scripts can invoke this method and expect extra parameters (such as
+ # from_expression) to be handled. As they are normaly handled at
+ # buildSQLQuery level, we must store them into final ComplexQuery, which
+ # will handle them.
+ unknown_column_dict = {}
+ # implicit_table_list: contains all tables explicitely given as par of
+ # column names with empty values. This is for backward compatibility. See
+ # comment about empty values.
+ implicit_table_list = []
+ for key, value in kw.iteritems():
+ result = None
+ if isinstance(value, dict_type_list):
+ # Cast dict-ish types into plain dicts.
+ value = dict(value)
+ if ignore_empty_string and (
+ value == ''
+ or (isinstance(value, (list, tuple)) and len(value) == 0)
+ or (isinstance(value, dict) and (
+ 'query' not in value
+ or value['query'] == ''
+ or (isinstance(value['query'], (list, tuple))
+ and len(value['query']) == 0)))):
+ # We have an empty value:
+ # - do not create a query from it
+ # - if key has a dot, add its left part to the list of "hint" tables
+ # This is for backward compatibility, when giving a mapped column
+ # with an empty value caused a join with catalog to appear in
+ # resulting where-expression)
+ if '.' in key:
+ implicit_table_list.append(key)
+ LOG('buildQuery', WARNING, 'Discarding empty value for key %r: %r' % (key, value))
+ else:
+ if isinstance(value, _Query):
+ # Query instance: use as such, ignore key.
+ result = value
+ elif isinstance(value, basestring):
+ # String: parse using key's default search key.
+ search_key = self.getColumnDefaultSearchKey(key)
+ if search_key is not None:
+ abstract_syntax_tree = search_key.parseSearchText(value)
+ if abstract_syntax_tree is None:
+ # Parsing failed, create a query from the bare string.
+ result = self.buildSingleQuery(key, value)
else:
- new_sort_index.append('%s' % key)
- else:
- LOG('SQLCatalog', WARNING, 'buildSQLQuery could not build sort '
- 'index (%s -> %s)' % (original_key, key))
- sort_index = join(new_sort_index,',')
- sort_on = str(sort_index)
- except ConflictError:
- raise
- except:
- LOG('SQLCatalog', WARNING, 'buildSQLQuery could not build the new sort index', error=sys.exc_info())
- sort_on = ''
- sort_key_list = []
-
- for key in key_list:
- if not key_alias_dict.has_key(key):
- getNewKeyAndUpdateVariables(key)
- if len(query_dict):
- for key, query in query_dict.items():
- query_result = query.asSQLExpression(key_alias_dict=key_alias_dict,
- full_text_search_keys=full_text_search_keys,
- keyword_search_keys=keyword_search_keys,
- datetime_search_keys=datetime_search_keys,
- ignore_empty_string=ignore_empty_string,
- stat__=stat__)
- if query_result['where_expression'] not in ('',None):
- where_expression_list.append(query_result['where_expression'])
- select_expression_list.extend(query_result['select_expression_list'])
-
- # Calculate extra where_expression based on required joins
- if query_table:
- for k, tid in from_table_dict.items():
- if k != query_table:
- where_expression_list.append('%s.uid = %s.uid' % (query_table, tid))
- # Calculate extra where_expressions based on related definition
- for (table_list, method_id) in related_methods.keys():
- related_method = getattr(self, method_id, None)
- if related_method is not None:
- table_id = {'src__' : 1} # Return query source, do not evaluate
- table_id['query_table'] = query_table
- table_index = 0
- for t_tuple in related_table_map[(table_list,method_id)]:
- table_id['table_%s' % table_index] = t_tuple[1] # table_X is set to mapped id
- from_table_dict[t_tuple[1]] = t_tuple[0]
- table_index += 1
- where_expression_list.append(related_method(**table_id))
- # Concatenate expressions
- if kw.get('where_expression',None) not in (None,''):
- where_expression_list.append(kw['where_expression'])
- if len(where_expression_list)>1:
- where_expression_list = ['(%s)' % x for x in where_expression_list]
- where_expression = join(where_expression_list, ' AND ')
- select_expression= join(select_expression_list,',')
-
- limit_expression = kw.get('limit', None)
- if isinstance(limit_expression, (list, tuple)):
- limit_expression = '%s,%s' % (limit_expression[0], limit_expression[1])
- elif limit_expression is not None:
- limit_expression = str(limit_expression)
-
- # force index if exists when doing sort as mysql doesn't manage them efficiently
- if len(sort_key_list) > 0:
- index_from_table = {}
- # first group columns from a same table
- for key in sort_key_list:
- try:
- related_table, column = key.split('.')
- except ValueError:
- # key is not of the form table.column
- # so get table from dict
- if len(from_table_dict) != 1:
- continue
- column = key
- related_table = from_table_dict.keys()[0]
-
- table = from_table_dict[related_table]
- # Check if it's a column for which we want to specify index
- index_columns = getattr(self, 'sql_catalog_index_on_order_keys', [])
- sort_column = '%s.%s' %(table, column)
- if not sort_column in index_columns:
- continue
- # Group columns
- if not index_from_table.has_key(table):
- index_from_table[table] = [column,]
+ if DEBUG:
+ LOG('SQLCatalog', 0, 'Building queries from abstract syntax tree: %r' % (abstract_syntax_tree, ))
+ result = self.buildQueryFromAbstractSyntaxTreeNode(abstract_syntax_tree, key)
+ elif isinstance(value, dict):
+ # Dictionnary: might contain the search key to use.
+ search_key_name = value.get('key')
+ # Backward compatibility: former "Keyword" key is now named
+ # "KeywordKey".
+ if search_key_name == 'Keyword':
+ search_key_name = value['key'] = 'KeywordKey'
+ result = self.buildSingleQuery(key, value, search_key_name)
+ else:
+ # Any other type, just create a query. (can be a DateTime, ...)
+ result = self.buildSingleQuery(key, value)
+ if result is None:
+ # No query could be created, emit a log, add to unknown column dict.
+ unknown_column_dict[key] = value
+ else:
+ append(result)
+ if len(unknown_column_dict):
+ LOG('SQLCatalog', 100, 'Unknown columns %r, skipped.' % (unknown_column_dict.keys(), ))
+ return ComplexQuery(query_list, operator=operator, unknown_column_dict=unknown_column_dict, implicit_table_list=implicit_table_list)
+
+ @profiler_decorator
+ def buildOrderByList(self, sort_on=None, sort_order=None, order_by_expression=None):
+ """
+ Internal method. Should not be used by code outside buildSQLQuery.
+
+ It is in a separate method because this code is here to keep backward
+ compatibility with an ambiguous API, and as such is ugly. So it's better
+ to conceal it to its own method.
+
+ It does not preserve backward compatibility for:
+ 'sort-on' parameter
+ 'sort-on' property
+ 'sort-order' parameter
+ 'sort-order' property
+ """
+ order_by_list = []
+ append = order_by_list.append
+ if sort_on is not None:
+ if order_by_expression is not None:
+ LOG('SQLCatalog', 0, 'order_by_expression (%r) and sort_on (%r) were given. Ignoring order_by_expression.' % (order_by_expression, sort_on))
+ if not isinstance(sort_on, (tuple, list)):
+ sort_on = [[sort_on]]
+ for item in sort_on:
+ if isinstance(item, (tuple, list)):
+ item = list(item)
else:
- index_from_table[table].append(column)
- # second ask index
- for table in index_from_table.keys():
- available_index_list = self.getIndex(table, index_from_table[table], key_list)
- if len(available_index_list) > 0:
- # Always give MySQL a chance to use PRIMARY key. It is much faster if
- # current table is used in a join on primary key than forcing it to
- # use another index.
- # Note: due to a bug (?) in MySQL (at least 5.0.45 community), it is
- # a syntax error to put "PRIMARY" keyword anywere besides at first
- # position. Hence the "insert(0".
- if 'PRIMARY' not in available_index_list:
- available_index_list.insert(0, 'PRIMARY')
- # tell mysql to use these index
- table = from_table_dict.pop(related_table)
- index_list_string = ', '.join(available_index_list)
- table_with_index = "%s use index(%s)" %(related_table, index_list_string)
- from_table_dict[table_with_index] = table
-
- from_expression = kw.get('from_expression', None)
- if from_expression is not None:
- final_from_expression = ', '.join(
- [from_expression.get(table, '`%s` AS `%s`' % (table, alias))
- for alias, table in from_table_dict.iteritems()])
+ item = [item]
+ if sort_order is not None and len(item) == 1:
+ item.append(sort_order)
+ if len(item) > 1:
+ if item[1] in ('descending', 'reverse', 'DESC'):
+ item[1] = 'DESC'
+ else:
+ item[1] = 'ASC'
+ if len(item) > 2:
+ if item[2] == 'int':
+ item[2] = 'SIGNED'
+ append(item)
+ elif order_by_expression is not None:
+ if not isinstance(order_by_expression, basestring):
+ raise TypeError, 'order_by_expression must be a basestring instance. Got %r.' % (order_by_expression, )
+ order_by_list = [[x.strip()] for x in order_by_expression.split(',')]
+ return order_by_list
+
+ @profiler_decorator
+ def buildSQLQuery(self, query_table='catalog', REQUEST=None,
+ ignore_empty_string=1, only_group_columns=False,
+ limit=None, extra_column_list=None,
+ **kw):
+# from traceback import format_list, extract_stack
+# LOG('buildSQLQuery', 0, ''.join(format_list(extract_stack())))
+ if DEBUG:
+ LOG('buildSQLQuery', 0, repr(kw))
+ group_by_list = kw.pop('group_by_list', kw.pop('group_by', kw.pop('group_by_expression', None)))
+ if isinstance(group_by_list, basestring):
+ group_by_list = [x.strip() for x in group_by_list.split(',')]
+ select_dict = kw.pop('select_dict', kw.pop('select_list', kw.pop('select_expression', None)))
+ if isinstance(select_dict, basestring):
+ if len(select_dict):
+ real_select_dict = {}
+ for column in select_dict.split(','):
+ index = column.lower().find(' as ')
+ if index != -1:
+ real_select_dict[column[index + 4:].strip()] = column[:index].strip()
+ else:
+ real_select_dict[column.strip()] = None
+ select_dict = real_select_dict
+ else:
+ select_dict = None
+ elif isinstance(select_dict, (list, tuple)):
+ select_dict = dict([(x, None) for x in select_dict])
+ # Handle order_by_list
+ order_by_list = kw.pop('order_by_list', None)
+ sort_on = kw.pop('sort_on', None)
+ sort_order = kw.pop('sort_order', None)
+ order_by_expression = kw.pop('order_by_expression', None)
+ if order_by_list is None:
+ order_by_list = self.buildOrderByList(
+ sort_on=sort_on,
+ sort_order=sort_order,
+ order_by_expression=order_by_expression
+ )
else:
- final_from_expression = None
- # Use a dictionary at the moment.
- return { 'from_table_list' : from_table_dict.items(),
- 'from_expression' : final_from_expression,
- 'order_by_expression' : sort_on,
- 'where_expression' : where_expression,
- 'limit_expression' : limit_expression,
- 'select_expression': select_expression,
- 'group_by_expression' : group_by_expression}
+ if sort_on is not None:
+ LOG('SQLCatalog', 0, 'order_by_list and sort_on were given, ignoring sort_on.')
+ if sort_order is not None:
+ LOG('SQLCatalog', 0, 'order_by_list and sort_order were given, ignoring sort_order.')
+ if order_by_expression is not None:
+ LOG('SQLCatalog', 0, 'order_by_list and order_by_expression were given, ignoring order_by_expression.')
+ # Handle from_expression
+ from_expression = kw.pop('from_expression', None)
+ # Handle where_expression
+ where_expression = kw.get('where_expression', None)
+ if isinstance(where_expression, basestring) and len(where_expression):
+ LOG('SQLCatalog', 100, 'Giving where_expression a string value is deprecated.')
+ # Transform given where_expression into a query, and update kw.
+ kw['where_expression'] = SQLQuery(where_expression)
+ # Handle select_expression_key
+ # It is required to support select_expression_key parameter for backward
+ # compatiblity, but I'm not sure if there can be a serious use for it in
+ # new API.
+ order_by_override_list = kw.pop('select_expression_key', None)
+ query = EntireQuery(
+ query=self.buildQuery(kw, ignore_empty_string=ignore_empty_string),
+ order_by_list=order_by_list,
+ order_by_override_list=order_by_override_list,
+ group_by_list=group_by_list,
+ select_dict=select_dict,
+ limit=limit,
+ catalog_table_name=query_table,
+ extra_column_list=extra_column_list,
+ from_expression=from_expression)
+ result = query.asSQLExpression(self, only_group_columns).asSQLExpressionDict()
+ if DEBUG:
+ LOG('buildSQLQuery', 0, repr(result))
+ return result
# Compatibililty SQL Sql
buildSqlQuery = buildSQLQuery
- def queryResults(self, sql_method, REQUEST=None, used=None, src__=0, build_sql_query_method=None, **kw):
+ @profiler_decorator
+ @transactional_cache_decorator('SQLCatalog._getSearchKeyDict')
+ @profiler_decorator
+ @caching_class_method_decorator(id='SQLCatalog._getSearchKeyDict', cache_factory='erp5_content_long')
+ @profiler_decorator
+ def _getSearchKeyDict(self):
+ result = {}
+ search_key_column_dict = {
+ 'KeywordKey': self.sql_catalog_keyword_search_keys,
+ 'FullTextKey': self.sql_catalog_full_text_search_keys,
+ 'DateTimeKey': self.sql_catalog_datetime_search_keys,
+ }
+ for key, column_list in search_key_column_dict.iteritems():
+ for column in column_list:
+ if column in result:
+ LOG('SQLCatalog', 100, 'Ambiguous configuration: column %r is set to use %r key, but also to use %r key. Former takes precedence.' % (column, result[column], key))
+ else:
+ result[column] = key
+ return result
+
+ @profiler_decorator
+ def getSearchKey(self, column, search_key=None):
+ """
+ Return an instance of a SearchKey class.
+
+ column (string)
+ The column for which the search key will be returned.
+ search_key (string)
+ If given, must be the name of a SearchKey class to be returned.
+ Returned value will be an instance of that class, even if column has
+ been configured to use a different one.
+ """
+ if search_key is None:
+ search_key = self._getSearchKeyDict().get(column, 'DefaultKey')
+ return getSearchKeyInstance(search_key, column)
+
+ def getComparisonOperator(self, operator):
+ """
+ Return an instance of an Operator class.
+
+ operator (string)
+ String defining the expected operator class.
+ See Operator module to have a list of available operators.
+ """
+ return getComparisonOperatorInstance(operator)
+
+ @profiler_decorator
+ def _queryResults(self, REQUEST=None, build_sql_query_method=None, **kw):
""" Returns a list of brains from a set of constraints on variables """
if build_sql_query_method is None:
build_sql_query_method = self.buildSQLQuery
query = build_sql_query_method(REQUEST=REQUEST, **kw)
+ # XXX: decide if this should be made normal
+ ENFORCE_SEPARATION = True
+ if ENFORCE_SEPARATION:
+ new_kw = {}
+ # Some parameters must be propagated:
+ for parameter_id in ('selection_domain', 'selection_report'):
+ if parameter_id in kw:
+ new_kw[parameter_id] = kw[parameter_id]
+ kw = new_kw
kw['where_expression'] = query['where_expression']
kw['sort_on'] = query['order_by_expression']
kw['from_table_list'] = query['from_table_list']
- kw['from_expression'] = query.get('from_expression')
+ kw['from_expression'] = query['from_expression']
kw['limit_expression'] = query['limit_expression']
kw['select_expression'] = query['select_expression']
kw['group_by_expression'] = query['group_by_expression']
- # Return the result
+ return kw
- #LOG('acceptable_keys',0,'acceptable_keys: %s' % str(acceptable_keys))
- #LOG('acceptable_key_map',0,'acceptable_key_map: %s' % str(acceptable_key_map))
- #LOG('queryResults',0,'kw: %s' % str(kw))
- #LOG('queryResults',0,'from_table_list: %s' % str(query['from_table_list']))
- return sql_method(src__=src__, **kw)
+ def queryResults(self, sql_method, REQUEST=None, src__=0, build_sql_query_method=None, **kw):
+ sql_kw = self._queryResults(REQUEST=REQUEST, build_sql_query_method=build_sql_query_method, **kw)
+ if DEBUG and not src__:
+ LOG('queryResults', 0, sql_method(src__=1, **sql_kw))
+ return sql_method(src__=src__, **sql_kw)
- def searchResults(self, REQUEST=None, used=None, **kw):
+ def searchResults(self, REQUEST=None, **kw):
""" Returns a list of brains from a set of constraints on variables """
- # The used argument is deprecated and is ignored
method = getattr(self, self.sql_search_results)
- return self.queryResults(method, REQUEST=REQUEST, used=used, **kw)
+ return self.queryResults(method, REQUEST=REQUEST, extra_column_list=self.getCatalogSearchResultKeys(), **kw)
__call__ = searchResults
- def countResults(self, REQUEST=None, used=None, stat__=1, **kw):
+ def countResults(self, REQUEST=None, **kw):
""" Returns the number of items which satisfy the where_expression """
# Get the search method
method = getattr(self, self.sql_count_results)
- return self.queryResults(method, REQUEST=REQUEST, used=used, stat__=stat__, **kw)
+ return self.queryResults(method, REQUEST=REQUEST, extra_column_list=self.getCatalogSearchResultKeys(), only_group_columns=True, **kw)
def recordObjectList(self, path_list, catalog=1):
"""
@@ -2394,40 +2401,56 @@ Globals.default__class_init__(Catalog)
class CatalogError(Exception): pass
-# pool of global preinitialized search keys instances
-SEARCH_KEY_INSTANCE_POOL = threading.local()
-# hook search keys and Query implementation
-def getSearchKeyInstance(search_key_class):
- """ Return instance of respective search_key class.
- We should have them initialized only once."""
- global SEARCH_KEY_INSTANCE_POOL
- if not hasattr(SEARCH_KEY_INSTANCE_POOL, 'pool'):
- pool = dict()
- for klass in (DefaultKey, RawKey, KeyWordKey, DateTimeKey,
- FullTextKey, FloatKey, ScriptableKey, KeyMappingKey):
- search_key_instance = klass()
- search_key_instance.build()
- pool[klass] = search_key_instance
- SEARCH_KEY_INSTANCE_POOL.pool = pool
-
- return SEARCH_KEY_INSTANCE_POOL.pool[search_key_class]
-
-from Query.Query import QueryMixin
-from Query.SimpleQuery import NegatedQuery, SimpleQuery
+from Query.Query import Query as _Query
+from Query.SimpleQuery import SimpleQuery
from Query.ComplexQuery import ComplexQuery
-
-# for of backwards compatability
-QueryMixin = QueryMixin
-Query = SimpleQuery
-NegatedQuery = NegatedQuery
-ComplexQuery = ComplexQuery
-
-from Products.ZSQLCatalog.SearchKey.DefaultKey import DefaultKey
-from Products.ZSQLCatalog.SearchKey.RawKey import RawKey
-from Products.ZSQLCatalog.SearchKey.KeyWordKey import KeyWordKey
-from Products.ZSQLCatalog.SearchKey.DateTimeKey import DateTimeKey
-from Products.ZSQLCatalog.SearchKey.FullTextKey import FullTextKey
-from Products.ZSQLCatalog.SearchKey.FloatKey import FloatKey
-from Products.ZSQLCatalog.SearchKey.ScriptableKey import ScriptableKey, KeyMappingKey
-
+from Query.AutoQuery import AutoQuery as Query
+
+def NegatedQuery(query):
+ return ComplexQuery(query, operator='not')
+
+allow_class(SimpleQuery)
+allow_class(ComplexQuery)
+
+import SearchKey
+SEARCH_KEY_INSTANCE_POOL = {}
+SEARCH_KEY_CLASS_CACHE = {}
+
+@profiler_decorator
+def getSearchKeyInstance(search_key_class_name, column):
+ assert isinstance(search_key_class_name, basestring)
+ try:
+ search_key_class = SEARCH_KEY_CLASS_CACHE[search_key_class_name]
+ except KeyError:
+ search_key_class = getattr(getattr(SearchKey, search_key_class_name),
+ search_key_class_name)
+ SEARCH_KEY_CLASS_CACHE[search_key_class_name] = search_key_class
+ try:
+ instance_dict = SEARCH_KEY_INSTANCE_POOL[search_key_class]
+ except KeyError:
+ instance_dict = SEARCH_KEY_INSTANCE_POOL[search_key_class] = {}
+ try:
+ result = instance_dict[column]
+ except KeyError:
+ result = instance_dict[column] = search_key_class(column)
+ return result
+
+from Operator import operator_dict
+def getComparisonOperatorInstance(operator):
+ return operator_dict[operator]
+
+from Query.EntireQuery import EntireQuery
+from Query.SQLQuery import SQLQuery
+
+verifyClass(ISearchKeyCatalog, Catalog)
+
+if PROFILING_ENABLED:
+ def Catalog_dumpProfilerData(self):
+ return profiler_report()
+
+ def Catalog_resetProfilerData(self):
+ profiler_reset()
+
+ Catalog.dumpProfilerData = Catalog_dumpProfilerData
+ Catalog.resetProfilerData = Catalog_resetProfilerData
diff --git a/product/ZSQLCatalog/SQLExpression.py b/product/ZSQLCatalog/SQLExpression.py
new file mode 100644
index 0000000000000000000000000000000000000000..0a8bdb47710d9fc188f38242347e5067ded3e556
--- /dev/null
+++ b/product/ZSQLCatalog/SQLExpression.py
@@ -0,0 +1,349 @@
+##############################################################################
+#
+# Copyright (c) 2008-2009 Nexedi SARL and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+from zLOG import LOG
+from Interface.ISQLExpression import ISQLExpression
+from Interface.Verify import verifyClass
+from types import NoneType
+from SQLCatalog import profiler_decorator
+
+SQL_LIST_SEPARATOR = ', '
+SQL_TABLE_FORMAT = '%s' # XXX: should be changed to '`%s`', but this breaks some ZSQLMethods.
+SQL_SELECT_ALIAS_FORMAT = '%s AS `%s`'
+
+"""
+ TODO:
+ - change table_alias_dict in internals to represent computed tables:
+ ie: '(SELECT * FROM `bar` WHERE `baz` = "hoge") AS `foo`'
+ '`foo` LEFT JOIN `bar` WHERE (`baz` = "hoge")'
+"""
+
+# Set to true to keep a reference to the query which created us.
+# Set to false to avoid keeping a reference to an object.
+DEBUG = True
+
+def defaultDict(value):
+ if value is None:
+ return {}
+ assert isinstance(value, dict)
+ return value.copy()
+
+class SQLExpression(object):
+
+ __implements__ = ISQLExpression
+
+ @profiler_decorator
+ def __init__(self,
+ query,
+ table_alias_dict=None,
+ order_by_list=(),
+ order_by_dict=None,
+ group_by_list=(),
+ where_expression=None,
+ where_expression_operator=None,
+ sql_expression_list=(),
+ select_dict=None,
+ limit=None,
+ from_expression=None):
+ if DEBUG:
+ self.query = query
+ self.table_alias_dict = defaultDict(table_alias_dict)
+ self.order_by_list = list(order_by_list)
+ self.group_by_list = list(group_by_list)
+ self.order_by_dict = defaultDict(order_by_dict)
+ # Only one of (where_expression, where_expression_operator) must be given (never both)
+ assert None in (where_expression, where_expression_operator)
+ # Exactly one of (where_expression, where_expression_operator) must be given, except if sql_expression_list is given and contains exactly one entry
+ assert where_expression is not None or where_expression_operator is not None or (sql_expression_list is not None and len(sql_expression_list) == 1)
+ # where_expression must be a basestring instance if given
+ assert isinstance(where_expression, (NoneType, basestring))
+ # where_expression_operator must be 'and', 'or' or 'not' (if given)
+ assert where_expression_operator in (None, 'and', 'or', 'not'), where_expression_operator
+ self.where_expression = where_expression
+ self.where_expression_operator = where_expression_operator
+ # Exactly one of (where_expression, sql_expression_list) must be given (XXX: duplicate of previous conditions ?)
+ assert where_expression is not None or sql_expression_list is not None
+ if isinstance(sql_expression_list, (list, tuple)):
+ sql_expression_list = [x for x in sql_expression_list if x is not None]
+ self.sql_expression_list = list(sql_expression_list)
+ self.select_dict = defaultDict(select_dict)
+ if limit is None:
+ self.limit = ()
+ elif isinstance(limit, (list, tuple)):
+ if len(limit) < 3:
+ self.limit = limit
+ else:
+ raise ValueError, 'Unrecognized "limit" value: %r' % (limit, )
+ else:
+ self.limit = (limit, )
+ if from_expression is not None:
+ LOG('SQLExpression', 0, 'Providing a from_expression is deprecated.')
+ self.from_expression = from_expression
+
+ @profiler_decorator
+ def getTableAliasDict(self):
+ """
+ Returns a dictionary:
+ key: table alias (string)
+ value: table name (string)
+
+ If there are nested SQLExpressions, it aggregates their mappings and
+ checks that they don't alias different table with the same name. If they
+ do, it raises a ValueError.
+ """
+ result = self.table_alias_dict.copy()
+ for sql_expression in self.sql_expression_list:
+ for alias, table_name in sql_expression.getTableAliasDict().iteritems():
+ existing_value = result.get(alias)
+ if existing_value not in (None, table_name):
+ message = '%r is a known alias for table %r, can\'t alias it now to table %r' % (alias, existing_value, table_name)
+ if DEBUG:
+ message = message + '. I was created by %r, and I am working on %r (%r) out of [%s]' % (
+ self.query,
+ sql_expression,
+ sql_expression.getQuery(),
+ ', '.join('%r (%r)' % (x, x.getQuery()) for x in self.sql_expression_list))
+ raise ValueError, message
+ result[alias] = table_name
+ return result
+
+ @profiler_decorator
+ def getFromExpression(self):
+ """
+ Returns a string.
+
+ If there are nested SQLExpression, it checks that they either don't
+ define any from_expression or the exact same from_expression. Otherwise,
+ it raises a ValueError.
+ """
+ result = self.from_expression
+ for sql_expression in self.sql_expression_list:
+ from_expression = sql_expression.getFromExpression()
+ if None not in (result, from_expression):
+ message = 'I don\'t know how to merge from_expressions'
+ if DEBUG:
+ message = message + '. I was created by %r, and I am working on %r (%r) out of [%s]' % (
+ self.query,
+ sql_expression,
+ sql_expression.getQuery(),
+ ', '.join('%r (%r)' % (x, x.getQuery()) for x in self.sql_expression_list))
+ raise ValueError, message
+ return result
+
+ @profiler_decorator
+ def getOrderByList(self):
+ """
+ Returns a list of strings.
+
+ If there are nested SQLExpression, it checks that they don't define
+ sorts for columns which are already sorted. If they do, it raises a
+ ValueError.
+ """
+ result = self.order_by_list[:]
+ known_column_set = set([x[0] for x in result])
+ for sql_expression in self.sql_expression_list:
+ for order_by in sql_expression.getOrderByList():
+ if order_by[0] in known_column_set:
+ raise ValueError, 'I don\'t know how to merge order_by yet'
+ else:
+ result.append(order_by)
+ known_column_set.add(order_by[0])
+ return result
+
+ @profiler_decorator
+ def getOrderByDict(self):
+ result_dict = self.order_by_dict.copy()
+ for sql_expression in self.sql_expression_list:
+ order_by_dict = sql_expression.getOrderByDict()
+ for key, value in order_by_dict.iteritems():
+ if key in result_dict and value != result_dict[key]:
+ message = 'I don\'t know how to merge order_by_dict with ' \
+ 'conflicting entries for key %r: %r vs. %r' % (key, result_dict[key], value)
+ if DEBUG:
+ message = message + '. I was created by %r, and I am working on %r (%r) out of [%s]' % (
+ self.query,
+ sql_expression,
+ sql_expression.getQuery(),
+ ', '.join('%r (%r)' % (x, x.getQuery()) for x in self.sql_expression_list))
+ raise ValueError, message
+ result_dict.update(order_by_dict)
+ return result_dict
+
+ @profiler_decorator
+ def getOrderByExpression(self):
+ """
+ Returns a string.
+
+ Returns a rendered "order by" expression. See getOrderByList.
+ """
+ order_by_dict = self.getOrderByDict()
+ get = order_by_dict.get
+ return SQL_LIST_SEPARATOR.join(get(x, str(x)) \
+ for x in self.getOrderByList())
+
+ @profiler_decorator
+ def getWhereExpression(self):
+ """
+ Returns a string.
+
+ Returns a rendered "where" expression.
+ """
+ if self.where_expression is not None:
+ result = self.where_expression
+ else:
+ if self.where_expression_operator == 'not':
+ assert len(self.sql_expression_list) == 1
+ result = '(NOT %s)' % (self.sql_expression_list[0].getWhereExpression())
+ elif len(self.sql_expression_list) == 1:
+ result = self.sql_expression_list[0].getWhereExpression()
+ elif len(self.sql_expression_list) == 0:
+ result = '(1)'
+ else:
+ operator = '\n ' + self.where_expression_operator.upper() + ' '
+ result = '(%s)' % (operator.join(x.getWhereExpression() for x in self.sql_expression_list), )
+ return result
+
+ @profiler_decorator
+ def getLimit(self):
+ """
+ Returns a list of 1 or 2 items (int or string).
+
+ If there are nested SQLExpression, it checks that they either don't
+ define any limit or the exact same limit. Otherwise it raises a
+ ValueError.
+ """
+ result = list(self.limit)
+ for sql_expression in self.sql_expression_list:
+ other_limit = sql_expression.getLimit()
+ if other_limit not in ([], result):
+ message = 'I don\'t know how to merge limits yet'
+ if DEBUG:
+ message = message + '. I was created by %r, and I am working on %r (%r) out of [%s]' % (
+ self.query,
+ sql_expression,
+ sql_expression.getQuery(),
+ ', '.join('%r (%r)' % (x, x.getQuery()) for x in self.sql_expression_list))
+ raise ValueError, message
+ return result
+
+ @profiler_decorator
+ def getLimitExpression(self):
+ """
+ Returns a string.
+
+ Returns a rendered "limit" expression. See getLimit.
+ """
+ return SQL_LIST_SEPARATOR.join(str(x) for x in self.getLimit())
+
+ @profiler_decorator
+ def getGroupByset(self):
+ """
+ Returns a set of strings.
+
+ If there are nested SQLExpression, it merges (union of sets) them with
+ local value.
+ """
+ result = set(self.group_by_list)
+ for sql_expression in self.sql_expression_list:
+ result.update(sql_expression.getGroupByset())
+ return result
+
+ @profiler_decorator
+ def getGroupByExpression(self):
+ """
+ Returns a string.
+
+ Returns a rendered "group by" expression. See getGroupBySet.
+ """
+ return SQL_LIST_SEPARATOR.join(self.getGroupByset())
+
+ @profiler_decorator
+ def getSelectDict(self):
+ """
+ Returns a dict:
+ key: alias (string)
+ value: column (string) or None
+
+ If there are nested SQLExpression, it aggregates their mappings and
+ checks that they don't alias different columns with the same name. If
+ they do, it raises a ValueError.
+ """
+ result = self.select_dict.copy()
+ for sql_expression in self.sql_expression_list:
+ for alias, column in sql_expression.getSelectDict().iteritems():
+ existing_value = result.get(alias)
+ if existing_value not in (None, column):
+ message = '%r is a known alias for column %r, can\'t alias it now to column %r' % (alias, existing_value, column)
+ if DEBUG:
+ message = message + '. I was created by %r, and I am working on %r (%r) out of [%s]' % (
+ self.query,
+ sql_expression,
+ sql_expression.getQuery(),
+ ', '.join('%r (%r)' % (x, x.getQuery()) for x in self.sql_expression_list))
+ raise ValueError, message
+ result[alias] = column
+ return result
+
+ @profiler_decorator
+ def getSelectExpression(self):
+ """
+ Returns a string.
+
+ Returns a rendered "select" expression. See getSelectDict.
+ """
+ return SQL_LIST_SEPARATOR.join(
+ SQL_SELECT_ALIAS_FORMAT % (column, alias)
+ for alias, column in self.getSelectDict().iteritems())
+
+ @profiler_decorator
+ def asSQLExpressionDict(self):
+ table_alias_dict = self.getTableAliasDict()
+ from_table_list = []
+ append = from_table_list.append
+ for alias, table in table_alias_dict.iteritems():
+ append((SQL_TABLE_FORMAT % (alias, ), SQL_TABLE_FORMAT % (table, )))
+ from_expression_dict = self.getFromExpression()
+ if from_expression_dict is not None:
+ from_expression = SQL_LIST_SEPARATOR.join(
+ from_expression_dict.get(table, '`%s` AS `%s`' % (table, alias))
+ for alias, table in table_alias_dict.iteritems())
+ else:
+ from_expression = None
+ return {
+ 'where_expression': self.getWhereExpression(),
+ 'order_by_expression': self.getOrderByExpression(),
+ 'from_table_list': from_table_list,
+ 'from_expression': from_expression,
+ 'limit_expression': self.getLimitExpression(),
+ 'select_expression': self.getSelectExpression(),
+ 'group_by_expression': self.getGroupByExpression()
+ }
+
+verifyClass(ISQLExpression, SQLExpression)
+
diff --git a/product/ZSQLCatalog/SearchKey/DateTimeKey.py b/product/ZSQLCatalog/SearchKey/DateTimeKey.py
index 893bc5ffc494b01d7685d5fc99a42a197943175a..9b69572aa7ff4cec1be7d9afecbcdb8283c58170 100644
--- a/product/ZSQLCatalog/SearchKey/DateTimeKey.py
+++ b/product/ZSQLCatalog/SearchKey/DateTimeKey.py
@@ -1,7 +1,9 @@
##############################################################################
#
-# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
-# Ivan Tyagov <ivan@nexedi.com>
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
@@ -26,171 +28,241 @@
#
##############################################################################
-from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery as Query
-from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
-from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance
-from DateTime import DateTime
+import sys
from SearchKey import SearchKey
-from pprint import pprint
+from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery
+from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
+#from Products.ZSQLCatalog.SQLExpression import SQLExpression
+from zLOG import LOG
+from DateTime.DateTime import DateTime, DateTimeError, _cache
+from Products.ZSQLCatalog.Interface.ISearchKey import ISearchKey
+from Interface.Verify import verifyClass
+from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
+MARKER = []
-class DateTimeKey(SearchKey):
- """ DateTimeKey key is an ERP5 portal_catalog search key which is used to render
- SQL expression that will try to match values in DateTime MySQL columns.
- It supports following special operator ['=', '%', '>' , '>=', '<', '<='] in
- addition to main logical operators like ['OR', 'or', 'AND', 'and'].
+timezone_dict = _cache._zmap
+
+date_completion_format_dict = {
+ None: ['01/01/%s', '01/%s'],
+ 'international': ['%s/01/01', '%s/01']
+}
+
+@profiler_decorator
+def _DateTime(*args, **kw):
+ return DateTime(*args, **kw)
+
+@profiler_decorator
+def castDate(value):
+ date_kw = {'datefmt': 'international'}
+ if isinstance(value, dict):
+ # Convert value into a DateTime, and guess desired delta from what user
+ # input.
+ assert value['type'] == 'date'
+ format = value.get('format')
+ value = value['query']
+ if format == '%m/%d/%Y':
+ date_kw.pop('datefmt')
+ if isinstance(value, DateTime):
+ pass
+ elif isinstance(value, basestring):
+ try:
+ value = _DateTime(value, **date_kw)
+ except DateTimeError:
+ delimiter_count = countDelimiters(value)
+ if delimiter_count < 3:
+ split_value = value.split()
+ if split_value[-1].lower() in timezone_dict:
+ value = '%s %s' % (date_completion_format_dict[date_kw.get('datefmt')][delimiter_count] % (' '.join(split_value[:-1]), ), split_value[-1])
+ else:
+ value = date_completion_format_dict[date_kw.get('datefmt')][delimiter_count] % (value, )
+ value = _DateTime(value, **date_kw)
+ else:
+ raise
+ else:
+ raise TypeError, 'Unknown date type: %r' % (value)
+ return value.toZone('UTC')
+
+# (strongly) inspired from DateTime.DateTime.py
+delimiter_list = ' -/.:,+'
+
+def getMonthLen(datetime):
+ return datetime._month_len[datetime.isLeapYear()][datetime.month()]
+
+def getYearLen(datetime):
+ return 365 + datetime.isLeapYear()
+
+delta_list = [getYearLen, getMonthLen, 1, 1.0 / 24, 1.0 / (24 * 60), 1.0 / (24 * 60 * 60)]
+
+@profiler_decorator
+def countDelimiters(value):
+ assert isinstance(value, basestring)
+ # Detect if timezone was provided, to avoid counting it as in precision computation.
+ split_value = value.split()
+ if split_value[-1].lower() in timezone_dict:
+ value = ' '.join(split_value[:-1])
+ # Count delimiters
+ delimiter_count = 0
+ for char in value:
+ if char in delimiter_list:
+ delimiter_count += 1
+ return delimiter_count
+
+@profiler_decorator
+def getPeriodBoundaries(value):
+ first_date = castDate(value)
+ if isinstance(value, dict):
+ value = value['query']
+ # Try to guess how much was given in query.
+ if isinstance(value, basestring):
+ delimiter_count = countDelimiters(value)
+ elif isinstance(value, DateTime):
+ raise TypeError, 'Impossible to guess a precision from a DateTime type.'
+ else:
+ raise TypeError, 'Unknown date type: %r' % (value)
+ delta = delta_list[delimiter_count]
+ if callable(delta):
+ delta = delta(first_date)
+ return first_date, first_date + delta
+
+@profiler_decorator
+def wholePeriod(search_key, group, column, value_list, exclude=False):
+ if exclude:
+ first_operator = '<'
+ second_operator = '>='
+ logical_operator = 'or'
+ else:
+ first_operator = '>='
+ second_operator = '<'
+ logical_operator = 'and'
+ query_list = []
+ append = query_list.append
+ for value in value_list:
+ first_date, second_date = getPeriodBoundaries(value)
+ append(ComplexQuery([SimpleQuery(search_key=search_key, operator=first_operator, group=group, **{column: first_date}),
+ SimpleQuery(search_key=search_key, operator=second_operator, group=group, **{column: second_date})],
+ operator=logical_operator))
+ return query_list
+
+def matchWholePeriod(search_key, group, column, value_list, *ignored):
+ return wholePeriod(search_key, group, column, value_list)
- Note: because all ERP5 datetime values are indexed in MySQL in 'UTC'
- the respective passed date will be first converted to 'UTC' before inserted into
- respective SQL query!
+def matchNotWholePeriod(search_key, group, column, value_list, *ignored):
+ return wholePeriod(search_key, group, column, value_list, exclude=True)
- Examples (GMT+02, Bulgaria/Sofia for 'delivery.start_date'):
+@profiler_decorator
+def matchExact(search_key, group, column, value_list, comparison_operator, logical_operator):
+ if comparison_operator is None:
+ comparison_operator = '='
+ value_list = [castDate(x) for x in value_list]
+ if logical_operator == 'or' and comparison_operator == '=':
+ query_list = [SimpleQuery(search_key=search_key, operator='in', group=group, **{column: value_list})]
+ else:
+ query_list = [SimpleQuery(search_key=search_key, operator=comparison_operator, group=group, **{column: x}) for x in value_list]
+ return query_list
- * '15/01/2008' --> "delivery.start_date = '2008-01-14 22:00'"
+def getNextPeriod(value):
+ return getPeriodBoundaries(value)[1]
- * '>=15/01/2008' --> "delivery.start_date >= '2008-01-14 22:00'"
+@profiler_decorator
+def matchBeforeNextPeriod(search_key, group, column, value_list, comparison_operator, logical_operator):
+ return matchExact(search_key, group, column, [getNextPeriod(x) for x in value_list], '<', logical_operator)
- * '>=15/01/2008 or <=20/01/2008'
- --> "delivery.start_date >= '2008-01-14 22:00' or delivery.start_date<='2008-01-19 22:00'"
+@profiler_decorator
+def matchAfterPeriod(search_key, group, column, value_list, comparison_operator, logical_operator):
+ return matchExact(search_key, group, column, [getNextPeriod(x) for x in value_list], '>=', logical_operator)
- * '>=15/01/2008 10:00 GMT+02 OR <=20/01/2008 05:12 Universal'
- -->
- "delivery.start_date >= '2008-01-15 08:00 Universal'
- OR
- delivery.start_date <= '2008-01-20 05:12 Universal'
- "
+operator_matcher_dict = {
+ None: matchWholePeriod,
+ '=': matchWholePeriod,
+ '!=': matchNotWholePeriod,
+ '<': matchExact,
+ '>=': matchExact,
+ '<=': matchBeforeNextPeriod,
+ '>': matchAfterPeriod,
+}
+
+# Behaviour of date time operators
+# Objects:
+# 2005/03/14 23:59:59
+# 2005/03/15 00:00:00
+# 2005/03/15 00:00:01
+# 2005/03/15 23:59:59
+# 2005/03/16 00:00:00
+# 2005/03/16 00:00:01
+#
+# Searches:
+# "2005/03/15" (operator = None)
+# Implicitely matches the whole period.
+# 2005/03/15 00:00:00
+# 2005/03/15 00:00:01
+# 2005/03/15 23:59:59
+#
+# "=2005/03/15" (operator = '=')
+# Behaves the same way as None operator.
+# 2005/03/15 00:00:00
+# 2005/03/15 00:00:01
+# 2005/03/15 23:59:59
+#
+# "!=2005/03/15" (operator = '!=')
+# Complementary of '=' operator.
+# 2005/03/14 23:59:59
+# 2005/03/16 00:00:00
+# 2005/03/16 00:00:01
+#
+# "<2005/03/15" (operator = '<')
+# Non-ambiguous (no difference wether time is considered as a period or a single point in time).
+# 2005/03/14 23:59:59
+#
+# ">=2005/03/15" (operator = '>=')
+# Complementary of '<' operator, and also non-ambiguous.
+# 2005/03/15 00:00:00
+# 2005/03/15 00:00:01
+# 2005/03/15 23:59:59
+# 2005/03/16 00:00:00
+# 2005/03/16 00:00:01
+#
+# "<=2005/03/15" (operator = '<=')
+# Union of results from '=' and '<' operators.
+# 2005/03/14 23:59:59
+# 2005/03/15 00:00:00
+# 2005/03/15 00:00:01
+# 2005/03/15 23:59:59
+#
+# ">2005/03/15" (operator = '>')
+# Complementary of '<=' operator.
+# 2005/03/16 00:00:00
+# 2005/03/16 00:00:01
+
+class DateTimeKey(SearchKey):
"""
+ This SearchKey allows generating date ranges from single, user-input dates.
+ """
+
+ default_comparison_operator = None
+ get_operator_from_value = True
- tokens = ('DATE', 'OR', 'AND', 'NOT', 'EQUAL',
- 'GREATERTHAN', 'GREATERTHANEQUAL',
- 'LESSTHAN', 'LESSTHANEQUAL')
-
- sub_operators = ('GREATERTHAN', 'GREATERTHANEQUAL',
- 'LESSTHAN', 'LESSTHANEQUAL', 'NOT', 'EQUAL',)
-
- def t_OR(self, t):
- r'\s+(OR|or)\s+'
- # operator has leading and trailing ONLY one white space character
- t.value = 'OR'
- return t
-
- def t_AND(self, t):
- r'\s+(AND|and)\s+'
- # operator has leading and trailing ONLY one white space character
- t.value = 'AND'
- return t
-
- def t_NOT(self, t):
- r'(\s+(NOT|not)\s+|!=)'
- # operator has leading and trailing ONLY one white space character
- t.value = t.value.upper().strip()
- return t
-
- t_GREATERTHANEQUAL = r'>='
- t_LESSTHANEQUAL = r'<='
- t_GREATERTHAN = r'>'
- t_LESSTHAN = r'<'
- t_EQUAL = r'='
- t_DATE = r'\d{1,4}[(/|\.|\-) /.]\d{1,4}[(/|\.|\-) /.]\d{1,4}((\s.)*\d{0,2}:\d{0,2}(:\d{0,2})?)?(\sUniversal|\sGMT\+\d\d)?|\d\d\d\d%?'
-
- def quoteSQLString(self, value, format):
- """ Return a quoted string of the value.
- Make sure to convert it to UTC first."""
- if getattr(value, 'ISO', None) is not None:
- value = "'%s'" % value.toZone('UTC').ISO()
- else:
- value = "'%s'" %DateTime(value).toZone('UTC').ISO()
- return value
-
- def buildQueryForTokenList(self, tokens, key, value, format):
- """ Build a ComplexQuery for a token list """
+ def _renderValueAsSearchText(self, value, operator):
+ return '"%s"' % (DateTime(value).ISO(), )
+
+ @profiler_decorator
+ def _buildQuery(self, operator_value_dict, logical_operator, parsed, group):
+ column = self.getColumn()
query_list = []
- for group_tokens in self.groupByLogicalOperator(tokens, 'AND'):
- token_values = [x.value for x in group_tokens]
- sub_operator, sub_tokens = self.getOperatorForTokenList(group_tokens)
- date_value = sub_tokens[0].value
- days_offset = 0
- # some format require special handling
- if format != '%Y':
- # full format (Year/Month/Day)
- if sub_operator in ('=',):
- # 2007/01/01 00:00 <= date < 2007/01/02
- days_offset = 1
- elif format == '%Y':
- # incomplete format only Year because DateTime can not handle
- # extend format and value by assumption that start of year is ment
- # add days ofset accordingly
- format = '%%%s/%%m/%%d' %format
- date_value = '%s/01/01' %date_value
- days_offset_map = {'=' : 366, '>' : 366,
- '>=' : 366, '<': -366, '<=':-366}
- days_offset = days_offset_map[sub_operator]
-
- # convert to UTC in given format
- is_valid_date = 1
+ extend = query_list.extend
+ for comparison_operator, value_list in operator_value_dict.iteritems():
try:
- if format != '%m/%d/%Y':
- # treat ambigious dates as "days before month before year"
- date_value = DateTime(date_value, datefmt="international").toZone('UTC')
- else:
- # US style "month before day before year"
- date_value = DateTime(date_value).toZone('UTC')
- except:
- is_valid_date = 0
-
- query_kw = None
- if is_valid_date:
- if sub_operator == '=':
- # transform to range 'key >= date AND date < key'
- query_kw = {key: (date_value, date_value + days_offset,),
- 'range': 'minmax'}
+ if parsed:
+ subquery_list = operator_matcher_dict[comparison_operator](
+ self, group, column, value_list, comparison_operator,
+ logical_operator)
else:
- query_kw = {key: date_value + days_offset,
- 'range': sub_operator}
- query_kw['type'] = 'date'
+ subquery_list = matchExact(self, group, column, value_list, comparison_operator, logical_operator)
+ except DateTimeError:
+ LOG('DateTimeKey', 100, 'Got an exception while generating a query for %r %r.' % (comparison_operator, value_list), error=sys.exc_info())
else:
- # not a valid date, try to get an year range
- is_year = 1
- date_value = date_value.replace('%', '')
- try: date_value = int(date_value)
- except: is_year = 0
- if is_year:
- date_value = '%s/01/01' % date_value
- date_value = DateTime(date_value).toZone('UTC')
- query_kw = {key: (date_value, date_value + 366,),
- 'type': 'date',
- 'range': 'minmax'}
-
- # append only if it was possible to generate query
- if query_kw is not None:
- query_list.append(Query(**query_kw))
-
- # join query list in one really big ComplexQuery
- if len(query_list):
- complex_query = ComplexQuery(*query_list,
- **{'operator': 'AND'})
- return complex_query
-
-## def buildSQLExpressionFromSearchString(self, key, value, format, mode, range_value, stat__):
-## """ Tokenize/analyze passed string value and generate SQL query expressions. """
-## where_expression = ''
-## key = self.quoteSQLKey(key, format)
-## tokens = self.tokenize(value)
-## operators_mapping_list = self.groupByOperator(tokens)
-## # new one
-## for item in operators_mapping_list:
-## row_tokens_values = []
-## tokens = item['tokens']
-## operator = item['operator']
-## operator_value = None
-## if operator is not None:
-## # operator is standalone expression
-## operator_value = operator.value
-## where_expressions.append('%s' %operator_value)
-## if len(tokens):
-## # no it's not a stand alone expression,
-## # determine it from list of tokens
-## operator_value, sub_tokens = self.getOperatorForTokenList(tokens)
-## row_tokens_values = [self.quoteSQLString(x.value, format) for x in sub_tokens]
-## where_expression = "%s %s %s" %(key, operator_value, ' '.join(row_tokens_values))
-## return where_expression, []
+ extend(subquery_list)
+ return query_list
+
+verifyClass(ISearchKey, DateTimeKey)
+
diff --git a/product/ZSQLCatalog/SearchKey/DefaultKey.py b/product/ZSQLCatalog/SearchKey/DefaultKey.py
index ab097b1c6cc5eb2cb1d2e75d0f93aaee9940f810..f1bc2d19c4a29299c0efcf745c6d7c7c128e78a2 100644
--- a/product/ZSQLCatalog/SearchKey/DefaultKey.py
+++ b/product/ZSQLCatalog/SearchKey/DefaultKey.py
@@ -1,7 +1,9 @@
##############################################################################
#
-# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
-# Ivan Tyagov <ivan@nexedi.com>
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
@@ -26,124 +28,36 @@
#
##############################################################################
-from DocumentTemplate.DT_Var import sql_quote
from SearchKey import SearchKey
+from Products.ZSQLCatalog.Interface.ISearchKey import ISearchKey
+from Interface.Verify import verifyClass
class DefaultKey(SearchKey):
- """ DefaultKey key is an ERP5 portal_catalog search key which is used to render
- SQL expression that will try to exactly one value.
- It supports following special operator ['=', '%', '>' , '>=', '<', '<='] in
- addition to main logical operators like ['OR', 'or', 'AND', 'and'].
-
- Examples for title column:
- * 'foo or bar' --> "title = 'foo' OR title = 'bar'"
- * 'foo or =bar' --> "title = 'foo' OR title = 'bar'"
- * '%foo% or bar' --> "title = '%foo%' OR title = 'bar'"
- * 'Organisation Module' --> "title = 'Organisation Module'"
- * '"Organisation Module"' --> "title = 'Organisation Module'"
- * '="Organisation Module"' --> "title = 'Organisation Module'"
"""
-
- # default type of sub Queries to be generated out fo a search string
- default_key_type = 'default'
-
- tokens = ('OR', 'AND', 'NOT', 'WORDSET', 'WORD',
- 'GREATERTHAN', 'GREATERTHANEQUAL',
- 'LESSTHAN', 'LESSTHANEQUAL')
-
- sub_operators = ('GREATERTHAN', 'GREATERTHANEQUAL',
- 'LESSTHAN', 'LESSTHANEQUAL', 'NOT')
-
-
- # Note: Order of placing rules (t_WORD for example) is very important
- def t_OR(self, t):
- r'\s+(OR|or)\s+'
- # operator must have leading and trailing ONLY one white space character
- # otherwise it's treated as a WORD
- t.value = 'OR'
- return t
-
- def t_AND(self, t):
- r'\s+(AND|and)\s+'
- # operator must have leading and trailing ONLY one white space character
- # otherwise it's treated as a WORD
- t.value = 'AND'
- return t
-
- def t_NOT(self, t):
- r'(\s+(NOT|not)\s+|!=)'
- # operator must have leading and trailing ONLY one white space character
- # otherwise it's treated as a WORD
- t.value = '!='
- return t
-
- t_GREATERTHANEQUAL = r'>='
- t_LESSTHANEQUAL = r'<='
- t_GREATERTHAN = r'>'
- t_LESSTHAN = r'<'
-
- def t_WORD(self, t):
- r'([^"\s<>!][\S\n]*|!([^=\s][\S\n]*)?)'
- # newlines are allowed, because variations are delimited by newlines.
- # WORD may contain arbitrary letters and numbers without white space
- # WORD may contain '%' but not at the beginning or end (otherwise it's KEYWORD)
- value = t.value.strip()
- if value[0] == '=':
- value = value[1:]
- t.value = value
- return t
-
- def t_WORDSET(self, t):
- r'"[^"]*"'
- #r'"[\x7F-\xFF\w\d\s/%][\x7F-\xFF\w\d\s/%]*"'
- # WORDSET is a combination of WORDs separated by white space
- # and starting/ending with "
- value = t.value.replace('"', '').strip()
- t.value = value
- return t
-
- def quoteSQLString(self, value, format):
- """ Return a quoted string of the value. """
- if isinstance(value, (int, long,)):
- return str(value)
- return "'%s'" % sql_quote(value)
-
-
-## def buildSQLExpressionFromSearchString(self, key, value, format, mode, range_value, stat__):
-## """ Tokenize/analyze passed string value and generate SQL query expressions. """
-## where_expressions = []
-## select_expressions = []
-## tokens = self.tokenize(value)
-## operators_mapping_list = self.groupByOperator(tokens)
-##
-## # find if any logical operator exists
-## tokens_values = []
-## logical_operator_found = 0
-## for token in tokens:
-## if token.type not in ('WORDSET', 'WORD',):
-## logical_operator_found = 1
-## break
-## tokens_values.append(token.value.replace("'", ""))
-##
-## # build expressions
-## if not logical_operator_found:
-## # no logical operator found so we assume that we search for a combination of words
-## where_expressions.append("%s = '%s'" %(key, ' '.join(tokens_values)))
-## else:
-## # in the search string we have explicitly defined an operator
-## for item in operators_mapping_list:
-## row_tokens_values = []
-## tokens = item['tokens']
-## operator = item['operator']
-## operator_value = None
-## if operator is not None:
-## # operator is standalone expression
-## operator_value = operator.value
-## where_expressions.append('%s' %operator_value)
-## if len(tokens):
-## # no it's not a stand alone expression,
-## # determine it from list of tokens
-## operator_value, sub_tokens = self.getOperatorForTokenList(tokens)
-## row_tokens_values = [x.value for x in sub_tokens]
-## where_expressions.append("%s %s '%s'" %(key, operator_value, ' '.join(row_tokens_values)))
-## return where_expressions, select_expressions
+ This SearchKey behaves like an ExactMatch SearchKey, except if value is a
+ string and contains a '%' sign, in which case it behaves like a
+ KeywordKey.
+ """
+ default_comparison_operator = '='
+ get_operator_from_value = True
+
+ def _guessComparisonOperator(self, value):
+ if isinstance(value, basestring) and '%' in value:
+ operator = 'like'
+ else:
+ operator = SearchKey._guessComparisonOperator(self, value)
+ return operator
+
+ def buildSearchTextExpression(self, operator, value, column=None):
+ operator_text = operator.getOperator()
+ if column is None:
+ column = self.getColumn()
+ if operator_text == 'like':
+ assert isinstance(value, basestring)
+ assert '%' in value
+ result = '%s:%s' % (column, value)
+ else:
+ result = SearchKey.buildSearchTextExpression(self, operator, value, column=column)
+ return result
+
+verifyClass(ISearchKey, DefaultKey)
diff --git a/product/ZSQLCatalog/SearchKey/FloatKey.py b/product/ZSQLCatalog/SearchKey/FloatKey.py
deleted file mode 100644
index 5cb458322938b6a53025654875e4d928fc35c42a..0000000000000000000000000000000000000000
--- a/product/ZSQLCatalog/SearchKey/FloatKey.py
+++ /dev/null
@@ -1,92 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
-# Ivan Tyagov <ivan@nexedi.com>
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsability of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# garantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License,] or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-#
-##############################################################################
-
-from SearchKey import SearchKey
-
-class FloatKey(SearchKey):
- """ FloatKey key is an ERP5 portal_catalog search key which is used to render
- float like SQL expression.
- """
- # default type of sub Queries to be generated out fo a search string
- default_key_type = 'float'
-
- tokens = ('OR', 'AND', 'NOT', 'FLOAT',
- 'GREATERTHAN', 'GREATERTHANEQUAL',
- 'LESSTHAN', 'LESSTHANEQUAL')
-
- sub_operators = ('GREATERTHAN', 'GREATERTHANEQUAL',
- 'LESSTHAN', 'LESSTHANEQUAL', 'NOT')
-
-
- # Note: Order of placing rules (t_WORD for example) is very important
- def t_OR(self, t):
- r'\s+(OR|or)\s+'
- # operator must have leading and trailing ONLY one white space character
- # otherwise it's treated as a WORD
- t.value = 'OR'
- return t
-
- def t_AND(self, t):
- r'\s+(AND|and)\s+'
- # operator must have leading and trailing ONLY one white space character
- # otherwise it's treated as a WORD
- t.value = 'AND'
- return t
-
- def t_NOT(self, t):
- r'(\s+(NOT|not)\s+|!=)'
- # operator must have leading and trailing ONLY one white space character
- # otherwise it's treated as a WORD
- t.value = '!='
- return t
-
- t_GREATERTHANEQUAL = r'>='
- t_LESSTHANEQUAL = r'<='
- t_GREATERTHAN = r'>'
- t_LESSTHAN = r'<'
-
- def t_FLOAT(self, t):
- r'[\d.][\d.]*'
- # FLOAT is a float number
- value = t.value.replace('"', '').strip()
- t.value = value
- return t
-
- def quoteSQLString(self, value, format):
- """ Return a quoted string of the value. """
- # Make sure there is no space in float values
- return "'%s'" %str(value).replace(' ', '')
-
- def quoteSQLKey(self, key, format):
- """ Return a quoted string of the value. """
- if format is not None:
- float_format = format.replace(' ', '')
- if float_format.find('.') >= 0:
- precision = len(float_format.split('.')[1])
- key = "TRUNCATE(%s,%s)" % (key, precision)
- return key
diff --git a/product/ZSQLCatalog/SearchKey/FullTextKey.py b/product/ZSQLCatalog/SearchKey/FullTextKey.py
index 0a56558426efaae40b6cef10d7f45ddf26f2c4a7..2feeb2a382de99a330ff811b08b77188a37393ac 100644
--- a/product/ZSQLCatalog/SearchKey/FullTextKey.py
+++ b/product/ZSQLCatalog/SearchKey/FullTextKey.py
@@ -1,7 +1,9 @@
##############################################################################
#
-# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
-# Ivan Tyagov <ivan@nexedi.com>
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
@@ -27,77 +29,19 @@
##############################################################################
from SearchKey import SearchKey
-import re
-
-SEARCH_MODE_MAPPING = {'in_boolean_mode': 'IN BOOLEAN MODE',
- 'with_query_expansion': 'WITH QUERY EXPANSION'}
+from Products.ZSQLCatalog.SearchText import parse
+from Products.ZSQLCatalog.Interface.ISearchKey import ISearchKey
+from Interface.Verify import verifyClass
class FullTextKey(SearchKey):
- """ FullTextKey key is an ERP5 portal_catalog search key which is used to render
- SQL expression that will try match all possible values using
- MySQL's fulltext search support.
- See syntax see MySQL's FullText search reference:
- http://dev.mysql.com/doc/refman/5.0/en/fulltext-search.html
"""
+ This SearchKey generates SQL fulltext comparisons.
+ """
+ default_comparison_operator = 'match'
+ get_operator_from_value = False
- tokens = ('PLUS', 'MINUS', 'WORD', 'GREATERTHAN', 'LESSTHAN', 'LEFTPARENTHES',
- 'RIGHTPARENTHES', 'TILDE', 'ASTERISK', 'DOUBLEQUOTE',)
-
- # SQL expressions patterns
- relevance = '%s_relevance'
- where_match_against = "MATCH %s AGAINST (%s %s)"
- select_match_against_as = "MATCH %s AGAINST (%s %s) AS %s"
-
- t_PLUS = r'(\+)'
- t_MINUS = r'(\-)'
- t_GREATERTHAN = r'(\>)'
- t_LESSTHAN = r'(\<)'
- t_LEFTPARENTHES = r'(\()'
- t_RIGHTPARENTHES = r'(\))'
- t_TILDE = r'(\~)'
- t_ASTERISK = r'(\*)'
- t_DOUBLEQUOTE = r'(\")'
+ def parseSearchText(self, value):
+ return parse(value)
- def t_WORD(self, t):
- r'[^\+\-<>\(\)\~\*\"\s]$|[^\+\-<>\(\)\~\*\"\s]+[^\*\"\s\)]'
- #r'[^\+\-<>\(\)\~\*\"\s]\S*'
- #r'[\x7F-\xFF\w\d][\x7F-\xFF\w\d]*'
- # WORD may contain arbitrary letters and numbers without white space
- return t
+verifyClass(ISearchKey, FullTextKey)
- def buildSQLExpression(self, key, value,
- format=None, mode=None, range_value=None, stat__=None):
- """ Analize token list and generate SQL expressions."""
- tokens = self.tokenize(value)
- # based on type tokens we may switch to different search mode
- mode = SEARCH_MODE_MAPPING.get(mode, '')
- if mode == '':
- # determine it based on list of tokens i.e if we have only words
- # leave as its but if we have '-' or '+' use boolean mode
- for token in tokens:
- if token.type != 'WORD':
- mode = SEARCH_MODE_MAPPING['in_boolean_mode']
- break
- if mode == '' and len(tokens) > 1:
- value = ' '.join(['+%s' % x.value for x in tokens])
- mode = SEARCH_MODE_MAPPING['in_boolean_mode']
- # split (if possible) to column.key
- if key.find('.') != -1:
- table, column = key.split('.')
- relevance_key1 = self.relevance %key.replace('.', '_')
- relevance_key2 = self.relevance %column
- else:
- relevance_key1 = self.relevance %key
- relevance_key2 = None
- select_expression_list = []
- where_expression = self.where_match_against % (key,
- self.quoteSQLString(value, ''), mode)
- if not stat__:
- # stat__ is an internal implementation artifact to prevent adding
- # select_expression for countFolder
- select_expression_list = [self.select_match_against_as % (key,
- self.quoteSQLString(value, ''), mode, relevance_key1),]
- if relevance_key2 is not None:
- select_expression_list.append(self.select_match_against_as % (
- key, self.quoteSQLString(value, ''), mode, relevance_key2))
- return where_expression, select_expression_list
diff --git a/product/ZSQLCatalog/SearchKey/KeyWordKey.py b/product/ZSQLCatalog/SearchKey/KeyWordKey.py
deleted file mode 100644
index 9dbea72ed027bd5d970343f7ed489757d98eb02e..0000000000000000000000000000000000000000
--- a/product/ZSQLCatalog/SearchKey/KeyWordKey.py
+++ /dev/null
@@ -1,234 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
-# Ivan Tyagov <ivan@nexedi.com>
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsability of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# garantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-#
-##############################################################################
-
-from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery as Query
-from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
-from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance
-from SearchKey import SearchKey
-from pprint import pprint
-
-class KeyWordKey(SearchKey):
- """ KeyWordKey key is an ERP5 portal_catalog search key which is used to render
- SQL expression that will try to match all possible values in a greedy manner.
- It supports following special operator ['=', '%', '>' , '>=', '<', '<='] in
- addition to main logical operators like ['OR', 'or', 'AND', 'and'].
-
- Examples for title column:
- * 'foo or bar' --> "title LIKE '%foo%' OR title LIKE '%bar%'"
- * 'foo or =bar' --> "title LIKE '%foo%' OR title = 'bar'"
- * 'Organisation Module' --> "title LIKE '%Organisation Module%'"
- * '"Organisation Module"' --> "title LIKE '%Organisation Module%'"
- * '="Organisation Module"' --> "title = 'Organisation Module'"
-
- """
-
- tokens = ('OR', 'AND', 'NOT',
- 'KEYWORD', 'WORDSET', 'WORD', 'EXPLICITEQUALLITYWORD',
- 'GREATERTHAN', 'GREATERTHANEQUAL',
- 'LESSTHAN', 'LESSTHANEQUAL')
-
- sub_operators = ('GREATERTHAN', 'GREATERTHANEQUAL',
- 'LESSTHAN', 'LESSTHANEQUAL', 'NOT')
-
- # this is the default operator
- default_operator = 'like'
-
- # if token's list starts with left sided operator
- # use this map to transfer it to range operator
- token_operator_range_map = {'like': 'like',
- '!=': 'not_like',
- '=': '=',}
-
- # Note: Order of placing rules (t_WORD for example) is very important
- def t_OR(self, t):
- r'\s+(OR|or)\s+'
- # operator must have leading and trailing ONLY one white space character
- # otherwise it's treated as a WORD
- t.value = 'OR'
- return t
-
- def t_AND(self, t):
- r'\s+(AND|and)\s+'
- # operator must have leading and trailing ONLY one white space character
- # otherwise it's treated as a WORD
- t.value = 'AND'
- return t
-
- def t_NOT(self, t):
- r'(\s+(NOT|not)\s+|!=)'
- # operator must have leading and trailing ONLY one white space character
- # otherwise it's treated as a WORD
- t.value = t.value.upper().strip()
- return t
-
- t_GREATERTHANEQUAL = r'>='
- t_LESSTHANEQUAL = r'<='
- t_GREATERTHAN = r'>'
- t_LESSTHAN = r'<'
-
- def t_EXPLICITEQUALLITYWORD(self, t):
- r'=\S*'
- # EXPLICITEQUALLITYWORD may contain arbitrary letters and numbers without white space
- # EXPLICITEQUALLITYWORD must contain '=' at the beginning
- value = t.value.strip()
- # get rid of leading '='
- t.value = value[1:]
- return t
-
- def t_KEYWORD(self, t):
- r'(%\S*|[^!<>=\s%]*%)(?!\S)'
- # KEYWORD must start and/or end with '%'.
- # It may contain arbitrary letters and numbers without white space
- value = t.value.strip()
- t.value = value
- return t
-
- def t_WORD(self, t):
- r'([^"\s<>!=%]([^ \t\r\f\v]*[^ \t\r\f\v%])?|!([^= \t\r\f\v%]|[^= \t\r\f\v][\S\n]*[^ \t\r\f\v%])?)'
- # WORD may contain arbitrary letters and numbers without white space
- # WORD may contain '%' but not at the beginning or end (otherwise it's KEYWORD)
- value = t.value.strip()
- t.value = value
- return t
-
- def t_WORDSET(self, t):
- r'=?"[^"]*"'
- # WORDSET is a combination of WORDs separated by white space
- # and starting/ending with " (optionally with '=')
- value = t.value.replace('"', '')
- if value[0] == '=':
- value = value[1:]
- t.value = value
- return t
-
- def getOperatorForTokenList(self, tokens):
- """ Generic implementation that will return respective
- operator for a token list. The first found occurence wins."""
- token = tokens[0]
- token_type = token.type
- if token_type in self.sub_operators:
- return token.value, tokens[1:]
- elif token.type == 'EXPLICITEQUALLITYWORD':
- # even though it's keyword key we can still explicitly define
- # that we want equality
- return '=', tokens
- else:
- return self.default_operator, tokens
-
- def buildQueryForTokenList(self, tokens, key, value, format):
- """ Build a ComplexQuery for a token list """
- query_list = []
- for group_tokens in self.groupByLogicalOperator(tokens, 'AND'):
- token_values = [x.value for x in group_tokens]
- sub_operator, sub_tokens = self.getOperatorForTokenList(group_tokens)
- first_token = sub_tokens[0]
- range = self.token_operator_range_map.get(sub_operator)
-
- sub_tokens_values = [x.value for x in sub_tokens]
- right_side_expression = ' '.join(sub_tokens_values)
- if first_token.type == 'WORDSET' and first_token.value.startswith('='):
- range = '='
- right_side_expression = first_token.value[1:]
- elif first_token.type in ('WORDSET', 'WORD',) and range == 'like':
- if '%' not in right_side_expression:
- # If the search string doesn't already contain '%', add trailing and
- # leading '%' to get more results
- right_side_expression = '%%%s%%' % right_side_expression
- query_kw = {key: right_side_expression,
- 'range': range}
- query_list.append(Query(**query_kw))
-
- # join query list in one really big ComplexQuery
- complex_query = ComplexQuery(*query_list,
- **{'operator': 'AND'})
- return complex_query
-
-
-## def buildSQLExpressionFromSearchString(self, key, value, format, mode, range_value, stat__):
-## """ Tokenize/analyze passed string value and generate SQL query expressions. """
-## where_expressions = []
-## select_expressions = []
-## tokens = self.tokenize(value)
-## operators_mapping_list = self.groupByOperator(tokens)
-##
-## # find if any logical operator exists
-## tokens_values = []
-## logical_operator_found = 0
-## for token in tokens:
-## if token.type not in ('WORD',):
-## logical_operator_found = 1
-## break
-## tokens_values.append(token.value.replace("'", ""))
-##
-## # build expressions
-## if not logical_operator_found:
-## # no logical operator found so we assume that we search
-## # for a combination of words
-## where_expressions.append("%s LIKE '%%%s%%'" %(key, ' '.join(tokens_values)))
-## else:
-## # in the search string we have explicitly defined an operator
-## for item in operators_mapping_list:
-## row_tokens_values = []
-## tokens = item['tokens']
-## operator = item['operator']
-## operator_value = None
-## if operator is not None:
-## # operator is standalone expression
-## where_expressions.append('%s' %operator.value)
-## if len(tokens):
-## # no it's not a stand alone expression,
-## # determine it from list of tokens
-## sub_where_expression = ''
-## tokens_number = len(tokens)
-## if tokens_number == 1:
-## # no left sided operator (<, >, >=, <=) found
-## token = tokens[0]
-## if token.type == 'WORD':
-## sub_where_expression = "LIKE '%%%s%%'" %token.value
-## elif token.type == 'KEYWORD':
-## sub_where_expression = "LIKE '%s'" %token.value
-## elif token.type == 'EXPLICITEQUALLITYWORD':
-## sub_where_expression = "= '%s'" %token.value
-## elif token.type == 'WORDSET' and token.value.startswith('='):
-## # if WORDSET starts with '=' it's an equality
-## sub_where_expression = " = '%s'" %token.value[1:]
-## else:
-## sub_where_expression = "LIKE '%%%s%%'" %token.value
-## else:
-## # we have two or more tokens, by definition first one should be
-## # logical operator like (<, >, >=, <=)
-## operator = tokens[0]
-## operator_value = operator.value
-## if operator.type in ('KEYWORD', 'WORDSET', 'WORD'):
-## # no operator for this token list, assume it's 'LIKE'
-## sub_where_expression = "LIKE '%s'" %' '.join([x.value for x in tokens])
-## else:
-## # we have operator and by convention if operator is used it's applyied to one token only
-## sub_where_expression = "%s'%s'" %(operator_value, tokens[1].value)
-## where_expressions.append('%s %s' %(key, sub_where_expression))
-## return where_expressions, select_expressions
diff --git a/product/ZSQLCatalog/SearchKey/KeywordKey.py b/product/ZSQLCatalog/SearchKey/KeywordKey.py
new file mode 100644
index 0000000000000000000000000000000000000000..3d9aa8e14c2409fded027bed5a6da52041c400be
--- /dev/null
+++ b/product/ZSQLCatalog/SearchKey/KeywordKey.py
@@ -0,0 +1,48 @@
+##############################################################################
+#
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+from SearchKey import SearchKey
+from Products.ZSQLCatalog.SearchText import parse
+from Products.ZSQLCatalog.Interface.ISearchKey import ISearchKey
+from Interface.Verify import verifyClass
+
+class KeywordKey(SearchKey):
+ """
+ This SearchKey generates matching comparison Queries suited for strings
+ with wilcards.
+ """
+ default_comparison_operator = 'like'
+ get_operator_from_value = True
+
+ def parseSearchText(self, value):
+ return parse(value)
+
+verifyClass(ISearchKey, KeywordKey)
+
diff --git a/product/ZSQLCatalog/SearchKey/RawKey.py b/product/ZSQLCatalog/SearchKey/RawKey.py
index 718a352e0eab9b6a7a07cc77b6612f53d6ee39b7..7f34e032ac862e6a821cf6d79f60be27ddc0a487 100644
--- a/product/ZSQLCatalog/SearchKey/RawKey.py
+++ b/product/ZSQLCatalog/SearchKey/RawKey.py
@@ -1,7 +1,9 @@
##############################################################################
#
-# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
-# Ivan Tyagov <ivan@nexedi.com>
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
@@ -27,23 +29,15 @@
##############################################################################
from SearchKey import SearchKey
+from Products.ZSQLCatalog.Interface.ISearchKey import ISearchKey
+from Interface.Verify import verifyClass
class RawKey(SearchKey):
- """ RawKey key is an ERP5 portal_catalog search key which is used to render
- SQL expression that will match exactly what's passed to it using equality ."""
+ """
+ This SearchKey does not do any parsing of given value.
+ """
+ default_comparison_operator = '='
+ get_operator_from_value = False
- def build(self, **kwargs):
- # this key doesn't require parsing
- # It's required to implement it as it's used ONLY for ExactMath
- pass
+verifyClass(ISearchKey, RawKey)
- def buildSQLExpression(self, key, value,
- format=None, mode=None, range_value=None, stat__=None):
-
- if value is not None:
- value = self.quoteSQLString(value, format)
- key = self.quoteSQLKey(key, format)
- where_expression = "%s = %s" % (key, value)
- else:
- where_expression = "%s is NULL" % (key)
- return where_expression, []
diff --git a/product/ZSQLCatalog/SearchKey/RelatedKey.py b/product/ZSQLCatalog/SearchKey/RelatedKey.py
new file mode 100644
index 0000000000000000000000000000000000000000..5c092d2783343d93e6332e9182b383cf32036620
--- /dev/null
+++ b/product/ZSQLCatalog/SearchKey/RelatedKey.py
@@ -0,0 +1,206 @@
+##############################################################################
+#
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+from SearchKey import SearchKey
+from Products.ZSQLCatalog.Query.Query import Query
+from Products.ZSQLCatalog.Query.RelatedQuery import RelatedQuery
+from Products.ZSQLCatalog.Query.SQLQuery import SQLQuery
+from Products.ZSQLCatalog.SQLExpression import SQLExpression
+from zLOG import LOG
+from Products.ZSQLCatalog.Interface.ISearchKey import IRelatedKey
+from Interface.Verify import verifyClass
+from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
+
+MARKER = []
+BACKWARD_COMPATIBILITY = True
+
+class RelatedKey(SearchKey):
+ """
+ This SearchKey handles searched on virtual columns of RelatedKey type.
+ It generates joins required by the virtual column to reach the actual
+ column to compare, plus a regular query on that column if needed.
+ """
+
+ __implements__ = IRelatedKey
+
+ related_key_definition = None
+
+ @profiler_decorator
+ def _buildRelatedKey(self, related_key_definition):
+ """
+ Extract RelatedKey parameters from its definition, and cache this
+ result. If related_key_definition changes since last computation, cached
+ values will be refreshed.
+
+ related_key_definition (string)
+ Describes parameters of a RelatedKey. It is composed of 3 mains parts,
+ separated by '/':
+ - a list of table names
+ Table names are separated by ','
+ - a column name
+ - the name of the related key ZSQLMethod
+ """
+ assert related_key_definition is not None
+ if self.related_key_definition != related_key_definition:
+ self.related_key_definition = related_key_definition
+ # Extract related_key_id, column_id and table_list from related_key_definition
+ table_list, self.real_column, self.related_key_id = related_key_definition.split('/')
+ self.table_list = table_list.split(',')
+
+ @profiler_decorator
+ def _getSearchKey(self, sql_catalog, search_key_name):
+ """
+ Get search key relevant to the actual column.
+
+ sql_catalog (SQLCatalog)
+ Used to access SearchKey provider.
+ search_key_name (string, None)
+ See SQLCatalog.getSearchKey.
+ """
+ return sql_catalog.getSearchKey(self.real_column, search_key_name)
+
+ @profiler_decorator
+ def getSearchKey(self, sql_catalog, related_key_definition, search_key_name=None):
+ """
+ Get search key relevant to the actual column, extracting information
+ about that column first if needed.
+
+ sql_catalog (SQLCatalog)
+ Used to access SearchKey provider.
+ related_key_definition (string)
+ See _buildRelatedKey.
+ search_key_name (string, None)
+ See SQLCatalog.getSearchKey.
+ """
+ self._buildRelatedKey(related_key_definition)
+ return self._getSearchKey(sql_catalog, search_key_name)
+
+ @profiler_decorator
+ def buildQuery(self, sql_catalog, related_key_definition,
+ search_value=MARKER, search_key_name=None,
+ logical_operator=None, comparison_operator=None):
+ self._buildRelatedKey(related_key_definition)
+ if search_value is MARKER:
+ join_condition = None
+ else:
+ join_condition = self._getSearchKey(sql_catalog, search_key_name).buildQuery(
+ search_value, group=self.getColumn(),
+ logical_operator=logical_operator,
+ comparison_operator=comparison_operator)
+ return RelatedQuery(search_key=self,
+ join_condition=join_condition)
+
+ @profiler_decorator
+ def registerColumnMap(self, column_map, table_alias_list=None):
+ related_column = self.getColumn()
+ group = column_map.registerRelatedKey(related_column, self.real_column)
+ # Each table except last one must be registered to their own group, so that
+ # the same table can be used multiple time (and aliased multiple times)
+ # in the same related key. The last one must be register to related key
+ # "main" group (ie, the value of the "group" variable) to be the same as
+ # the ta ble used in join_condition.
+ if table_alias_list is not None:
+ assert len(self.table_list) == len(table_alias_list)
+ for table_position in xrange(len(self.table_list) - 1):
+ table_name = self.table_list[table_position]
+ local_group = column_map.registerRelatedKeyColumn(related_column, table_position, group)
+ column_map.registerTable(table_name, group=local_group)
+ if table_alias_list is not None:
+ # Pre-resolve all tables with given aliases
+ given_name, given_alias = table_alias_list[table_position]
+ assert table_name == given_name
+ column_map.resolveTable(table_name, given_alias, group=local_group)
+ table_name = self.table_list[-1]
+ column_map.registerTable(table_name, group=group)
+ if table_alias_list is not None:
+ given_name, given_alias = table_alias_list[-1]
+ assert table_name == given_name
+ column_map.resolveTable(table_name, given_alias, group=group)
+ # Resolve (and register) related key column in related key group with its last table.
+ column_map.registerColumn(self.real_column, group=group)
+ column_map.resolveColumn(self.real_column, table_name, group=group)
+ # Always register catalog, since it is always the "base" table of
+ # RelatedKeys.
+ column_map.registerCatalog()
+ return group
+
+ @profiler_decorator
+ def buildSQLExpression(self, sql_catalog, column_map, only_group_columns, group):
+ """
+ Render RelatedKey's ZSQLMethod by providing it table aliases from
+ ColumnMap.
+
+ sql_catalog (SQLCatalog)
+ column_map (ColumnMap)
+ group (string)
+ only_group_columns (bool)
+ Ignored.
+ """
+ related_key = getattr(sql_catalog, self.related_key_id)
+ table_alias_dict = dict(
+ [('table_%s' % (x, ), column_map.getTableAlias(self.table_list[x], group=column_map.getRelatedKeyGroup(x, group)))
+ for x in xrange(len(self.table_list) - 1)])
+ x = len(table_alias_dict)
+ assert x == len(self.table_list) - 1
+ table_alias_dict['table_%s' % (x, )] = column_map.getTableAlias(self.table_list[x], group=group)
+ rendered_related_key = related_key(
+ query_table=column_map.getCatalogTableAlias(),
+ src__=1,
+ **table_alias_dict)
+ # Important:
+ # Former catalog separated join condition from related query.
+ # Example:
+ # ComplexQuery(Query(title="foo"),
+ # Query(subordination_title="bar")
+ # , operator='OR')
+ # Former catalog rendering (truncated where-expression):
+ # AND ((catalog.title LIKE '%bil%') OR
+ # (related_catalog_1.title = 'My Organisation'))
+ # AND (related_catalog_1.uid = related_category_0.category_uid AND
+ # related_category_0.base_category_uid = 873 AND
+ # related_category_0.uid = catalog.uid)
+ # As you can see, the part of the query joining the tables is *out* of the
+ # OR expression, and therefor applies to the entire query.
+ # This was done on purpose, because doing otherwise gives very poor
+ # performances (on a simple data set, similar query can take *minutes* to
+ # execute - as of MySQL 5.x).
+ # Doing the same way as the former catalog is required for backward
+ # compatibility, until a decent alternative is found (like spliting the
+ # "OR" expression into ensemblist operations at query level).
+ # Note that doing this has a side effect on result list, as objects
+ # lacking a relation will never appear in the result.
+ if BACKWARD_COMPATIBILITY:
+ column_map.addJoinQuery(SQLQuery(rendered_related_key))
+ return None
+ else:
+ return SQLExpression(self, where_expression=rendered_related_key)
+
+verifyClass(IRelatedKey, RelatedKey)
+
diff --git a/product/ZSQLCatalog/SearchKey/ScriptableKey.py b/product/ZSQLCatalog/SearchKey/ScriptableKey.py
deleted file mode 100644
index 60868f8689617af1e6e5780078e94847c425ed46..0000000000000000000000000000000000000000
--- a/product/ZSQLCatalog/SearchKey/ScriptableKey.py
+++ /dev/null
@@ -1,193 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
-# Ivan Tyagov <ivan@nexedi.com>
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsability of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# garantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-#
-##############################################################################
-
-from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery as Query
-from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
-from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance
-from Products.PythonScripts.Utility import allow_class
-
-from SearchKey import SearchKey
-from pprint import pprint
-
-# these keys are used to build query in case for ScriptableKey
-# when no key was specified in fornt of value
-DEFAULT_SEARCH_KEYS = ('SearchableText', 'reference', 'title',)
-
-class KeyMappingKey(SearchKey):
- """ Usable lexer class used (internally) by ScriptableKey lexer than can parse following:
- VALUE OPERATOR VALUE
-
- Examples:
- * "portal_type : Person"
- * "creation_date > 2007-01-01"
- """
-
- tokens = ('OPERATOR', 'COLONOPERATOR', 'VALUE',)
-
- t_OPERATOR = r'>=|<=|>|<'
- t_VALUE = r'[\x7F-\xFF\w\d\/~!@#$^&*()_+-][\x7F-\xFF\w\d\/~!@#$^&*()_+-]*'
-
- def t_COLONOPERATOR(self, t):
- r':'
- # ':' is the same as '=' (equality)
- t.value = '='
- return t
-
-class ScriptableKey(SearchKey):
- """ KeyWordKey key is an ERP5 portal_catalog search key which is used to generate a
- ComplexQuery instance out of an arbitrary search string.
-
- Examples:
- * "John Doe AND portal_type:Person AND creation_date > 2007-01-01"
-
- would be turned into following ComplexQuery:
-
- * ComplexQuery(Query(portal_type='Person'),
- Query(creation_date='2007-01-01', operator='>'),
- ComplexQuery(Query(searchable_text='John Doe'),
- Query(title='John Doe'),
- Query(reference='John Doe'),
- operator='OR')
- operator='AND'))
- """
- sub_operators = ('GREATERTHAN', 'GREATERTHANEQUAL',
- 'LESSTHAN', 'LESSTHANEQUAL',)
-
- tokens = ('OR', 'AND',
- 'DATE', 'WORD', 'KEYMAPPING',
- 'GREATERTHAN', 'GREATERTHANEQUAL',
- 'LESSTHAN', 'LESSTHANEQUAL', 'EQUAL')
-
- t_GREATERTHANEQUAL = r'>='
- t_LESSTHANEQUAL = r'<='
- t_GREATERTHAN = r'>'
- t_LESSTHAN = r'<'
- t_EQUAL = r'='
-
- # Note: Order of placing rules (t_WORD for example) is very important
- def t_OR(self, t):
- r'\s+(OR|or)\s+'
- # operator must have leading and trailing ONLY one white space character
- # otherwise it's treated as a WORD
- t.value = 'OR'
- return t
-
- def t_AND(self, t):
- r'\s+(AND|and)\s+'
- # operator must have leading and trailing ONLY one white space character
- # otherwise it's treated as a WORD
- t.value = 'AND'
- return t
-
- def t_KEYMAPPING(self, t):
- r'[^<>=:\s]+\s*(>|<|<=|>=|:)\s*\S+'
- # KEYMAPPING has following format: KEY OPERATOR VALUE
- # where OPERATOR in ['<', '>', '<=', '>=', ':']
- # example: 'creation_date < 2007-12-12'
- value = t.value.strip()
- t.value = value
- return t
-
- def t_WORD(self, t):
- r'[^<>=\s:]+'
- # WORD may contain arbitrary letters and numbers without white space
- # WORD may contain '%' but not at the beginning or end (otherwise it's KEYWORD)
- value = t.value.strip()
- t.value = value
- return t
-
- def buildQueryForTokenList(self, tokens):
- """ Build a ComplexQuery for a token list """
- query_list = []
- for group in self.groupByLogicalOperator(tokens, 'AND'):
- group_tokens = group
- first_group_token = group_tokens[0]
- if first_group_token.type == 'KEYMAPPING':
- # user specified a full sub query definition following this format:
- # 'key operator value'
- sub_search_string = group_tokens[0].value
- keymapping_lexer = getSearchKeyInstance(KeyMappingKey)
- sub_tokens = keymapping_lexer.tokenize(sub_search_string)
- sub_tokens_values = [x.value for x in sub_tokens]
- search_key, search_operator, search_value = sub_tokens_values
- query_kw = {search_key: search_value,
- 'range' : search_operator,}
- query_list.append(Query( **query_kw))
- elif first_group_token.type in self.sub_operators:
- # user specified a incomplete sub query definition following this format:
- # 'operator value'. Assume that he ment to search for 'title' and
- # use supplied 'operator'
- search_operator = first_group_token.value
- simple_query_value = ' '.join([x.value for x in group_tokens[1:]])
- query_kw = {'title': simple_query_value,
- 'range' : search_operator,}
- query_list.append(Query( **query_kw))
- else:
- # user specified a VERY incomplete sub query definition following this format:
- # 'value'. Let's search against most common search_keys and assume operator
- # is '=' (by default) and try to get as much possible results
- simple_query_value = ' '.join([x.value for x in group_tokens])
- sub_query_list = []
- for default_key in DEFAULT_SEARCH_KEYS:
- query_kw = {default_key: simple_query_value}
- sub_query_list.append(Query(**query_kw))
- query_list.append(ComplexQuery(*sub_query_list,
- **{'operator':'OR'}))
- # join query list in one really big ComplexQuery
- complex_query = ComplexQuery(*query_list,
- **{'operator':'AND'})
- return complex_query
-
- def buildQuery(self, key, value,
- format=None, mode=None, range_value=None, stat__=None):
- """ Build ComplexQuery from passed search string value.
- When grouping expressions we use the following assumptions
- that 'OR' operator has higher priority in a sense:
-
- * "John Doe AND portal_type:Person OR creation_date>=2005/12/12"
-
- is considered as:
-
- * (John Doe AND portal_type:Person) OR (creation_date>=2005/12/12)"
- """
- query_list = []
- tokens = self.tokenize(value)
-
- # split tokens list into one or many OR concatanated expressions
- sub_tokens_or_groups = self.groupByLogicalOperator(tokens, 'OR')
-
- # get a ComplexQuery for a sub token list
- for tokens_or_group in sub_tokens_or_groups:
- query_list.append(self.buildQueryForTokenList(tokens_or_group))
-
- # join query list in one really big ComplexQuery
- complex_query = ComplexQuery(*query_list,
- **{'operator':'OR'})
- return complex_query
-
-allow_class(ScriptableKey)
diff --git a/product/ZSQLCatalog/SearchKey/SearchKey.py b/product/ZSQLCatalog/SearchKey/SearchKey.py
index 1babb855460e041ee14f13324548705627ec33f3..64bd8a78cfbcaea99bd27801f3cb338716ee1877 100644
--- a/product/ZSQLCatalog/SearchKey/SearchKey.py
+++ b/product/ZSQLCatalog/SearchKey/SearchKey.py
@@ -1,7 +1,9 @@
##############################################################################
#
-# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
-# Ivan Tyagov <ivan@nexedi.com>
+# Copyright (c) 2002-2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2007-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jean-Paul Smets-Solanes <jp@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
@@ -13,7 +15,7 @@
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
-# of the License,] or (at your option) any later version.
+# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -26,220 +28,322 @@
#
##############################################################################
-from DocumentTemplate.DT_Var import sql_quote
-from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery as Query
+from zLOG import LOG
+from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery
from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
-from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance
-
-import ply.yacc as yacc
-import ply.lex as lex
-
-class SearchKey:
- """ BaseKey is a base class that implements a parser of
- search grammar used in ERP5. It also implements all generic
- search key class methods."""
-
- # main logical operators
- operators = ('OR', 'AND',)
- default_operator = '='
-
- # in ERP5 search grammar white space is extremely important
- # so we can not ignore it.
- #t_ignore = ' \t'
-
- # no need to rack down line numbers
- #def t_newline(self, t):
- # r'\n+'
- # #t.lexer.lineno += len(t.value)
-
- def t_error(self, t):
- #print "Illegal character '%s'" % t.value[0]
- t.lexer.skip(1)
-
- def p_error(self, p):
- pass
-
- def build(self, **kwargs):
- """ This method will initialize respective search key class with
- tokens' definitions. """
- self.lexer = lex.lex(object = self, **kwargs)
-
- def tokenize(self, data):
- """ Return list of tokens according to respective
- search key tokens' definitions. """
- result = []
- self.lexer.input(data)
- while 1:
- tok = self.lexer.token()
- if not tok:
- break
- result.append(tok)
+from Products.ZSQLCatalog.SQLExpression import SQLExpression
+from Products.ZSQLCatalog.Interface.ISearchKey import ISearchKey
+from Interface.Verify import verifyClass
+from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
+
+single_operator_dict = {
+ 'min': '>=',
+ 'max': '<',
+ 'ngt': '<=',
+ 'nlt': '>'
+}
+
+dual_operator_dict = {
+ 'minmax': ('>=', '<'),
+ 'minngt': ('>=', '<=')
+}
+
+# List of operators searched for at value's begnining when it's a basestring.
+# Order is important: an operator whose left part would be matching another operator of lower index would never be used.
+operator_list = ('>=', '<=', '>', '<', '=', '!=')
+
+def preprocessLikeValue(value):
+ if '%' not in value:
+ value = '%%%s%%' % (value, )
+ return value
+
+operator_value_preprocessor_dict = {
+ 'like': preprocessLikeValue
+}
+
+def deprocessLikeValue(value):
+ assert isinstance(value, basestring)
+ if len(value) >= 2 and value[0] == '%' and value[-1] == '%':
+ value = value.strip('%')
+ return value
+
+operator_value_deprocessor_dict = {
+ 'like': deprocessLikeValue
+}
+
+class SearchKey(object):
+
+ __implements__ = ISearchKey
+
+ # Comparison operator to use when parsing a string value and no operator is
+ # found.
+ # Note: for non-string values, "=" is always used by default.
+ default_comparison_operator = '='
+ # Wether or not to allow a basestring value to be searched for a comparison
+ # operator.
+ get_operator_from_value = True
+
+ def __init__(self, column):
+ self.column = column
+
+ def getColumn(self):
+ return self.column
+
+ @profiler_decorator
+ def buildSQLExpression(self, operator, value, column_map, only_group_columns, group):
+ column_name = self.getColumn()
+ rendered_column = column_map.asSQLColumn(column_name, group=group)
+ return operator.asSQLExpression(rendered_column, value, only_group_columns)
+
+ @profiler_decorator
+ def _renderValueAsSearchText(self, value, operator):
+ """
+ Render a single value as valid SearchText using provided operator.
+ This is also responsible for undoing any formatting the value received
+ from the SearchKey.
+
+ value (anything)
+ operator (Operator)
+ The operator used to render value.
+ """
+ operator_value_deprocessor = operator_value_deprocessor_dict.get(operator.getOperator())
+ if operator_value_deprocessor is not None:
+ value = operator_value_deprocessor(value)
+ return operator.asSearchText(value)
+
+ @profiler_decorator
+ def buildSearchTextExpression(self, operator, value, column=None):
+ operator_text = operator.getOperator()
+ if column is None:
+ column = self.getColumn()
+ if isinstance(value, (list, tuple)):
+ assert operator_text == 'in'
+ assert len(value)
+ value = [self._renderValueAsSearchText(x, operator) for x in value]
+ if self.default_comparison_operator != '=':
+ value = ['=%s' % (x, ) for x in value]
+ # XXX: operator used to join value elements should be reused from parser data (?)
+ result = '(%s)' % (' OR '.join(value), )
+ else:
+ result = self._renderValueAsSearchText(value, operator)
+ if operator_text != self.default_comparison_operator:
+ result = '%s%s' % (operator_text, result)
+ if len(column):
+ result = '%s:%s' % (column, result)
return result
- # Grouping of tokens
- def getOperatorForTokenList(self, tokens):
- """ Generic implementation that will return respective
- operator for a token list. The first found occurence wins."""
- token = tokens[0]
- token_type = token.type
- if token_type in self.sub_operators:
- return token.value, tokens[1:]
+ @profiler_decorator
+ def registerColumnMap(self, column_map, group, simple_query):
+ column_map.registerColumn(self.getColumn(), group=group, simple_query=simple_query)
+ return group
+
+ @profiler_decorator
+ def _getComparisonOperator(self, value):
+ """
+ From a basestring instance, return a contained operator and value without
+ that operator.
+
+ value (string)
+
+ Returns: 2-tuple of strings
+ First element is the operator. None if there was no operator in value.
+ Second element is the value without the operator.
+ """
+ startswith = value.startswith
+ for operator in operator_list:
+ if startswith(operator):
+ value = value[len(operator):].lstrip()
+ break
else:
- return self.default_operator, tokens
+ operator = self._guessComparisonOperator(value)
+ return operator, value
- def groupByLogicalOperator(self, tokens, logical_operator ='OR'):
- """ Split tokens list into one or many OR concatanated tokens list
+ @profiler_decorator
+ def _guessComparisonOperator(self, value):
"""
- sub_tokens_or_groups = []
- tmp_token_list = []
- for token in tokens:
- if token.type != logical_operator:
- tmp_token_list.append(token)
+ From a basestring instance, return a contained operator.
+ Value cannot be altered in the process.
+
+ value (string)
+
+ Returns: 2-tuple of strings
+ First element is the operator. None if there was no operator in value.
+ Second element is the value without the operator.
+ """
+ return self.default_comparison_operator
+
+ @profiler_decorator
+ def _preprocessValue(self, value, operator):
+ operator_value_preprocessor = operator_value_preprocessor_dict.get(operator)
+ if operator_value_preprocessor is not None:
+ value = operator_value_preprocessor(value)
+ return value
+
+ @profiler_decorator
+ def _processSearchValue(self, search_value, default_logical_operator, comparison_operator):
+ """
+ Change search_value into a list of values, one or more logical operators,
+ and a comparison operator. If no default_logical_operator is given,
+ 'or' is used.
+
+ search_value
+ basestring
+ int
+ dict
+ list or tuple
+ Non-empty
+ Composed of homogeneous items
+ Returns: 3-tuple
+ dict:
+ key (string)
+ Comparison operator
+ value (list of anything)
+ List of values applying to this operator.
+ string:
+ Logical operator applied to all elements of returned dict.
+ bool:
+ True if logical operators were searched for in values, False
+ otherwise. Useful to give different meanings to in-value operators
+ and others.
+ """
+ if comparison_operator == '':
+ comparison_operator = None
+ get_operator_from_value = False
+ else:
+ get_operator_from_value = self.get_operator_from_value
+ logical_operator = None
+ if default_logical_operator is None:
+ default_logical_operator = 'or'
+ parsed = False
+ if isinstance(search_value, dict):
+ # comparison_operator parameter collides with dict's 'operator' key.
+ # Fail loudly.
+ assert comparison_operator is None
+ actual_value = search_value['query']
+ if search_value.get('key') not in (None, self.__class__.__name__):
+ LOG(self.__class__.__name__, 100, '"key" dict entry does not match current class: %r' % (search_value, ))
+ if 'type' in search_value:
+ assert 'operator' not in search_value, search_value
+ assert 'range' not in search_value, search_value
else:
- sub_tokens_or_groups.append(tmp_token_list)
- tmp_token_list = []
- # append remainig last tokens
- sub_tokens_or_groups.append(tmp_token_list)
- return sub_tokens_or_groups
-
- # SQL quoting (each search key should override them it if needed)
- def quoteSQLKey(self, key, format):
- """ Return a quoted string of the value. """
- return key
-
- def quoteSQLString(self, value, format):
- """ Return a quoted string of the value. """
- return "'%s'" % sql_quote(str(value))
-
- # SQL generation
- def buildSQLExpression(self, key, value,
- format=None, mode=None, range_value=None, stat__=0):
- """ Generic implementation. Leave details to respective key. """
- if range_value is not None:
- # if range_value we handle directly (i.e no parsing of search string)
- where_expressions, select_expressions = \
- self.buildSQLExpressionFromRange(key, value,
- format, mode, range_value, stat__)
+ value_operator = search_value.get('operator')
+ value_range = search_value.get('range')
+ if value_range is not None:
+ if value_operator is not None:
+ LOG('SearchKey', 100, '"range" and "operator" are mutualy exclusive, ignoring operator: %r' % (search_value, ))
+ if value_range in single_operator_dict:
+ comparison_operator = single_operator_dict[value_range]
+ elif value_range in dual_operator_dict:
+ if not isinstance(actual_value, (tuple, list)):
+ raise TypeError, 'Operator %r requires value to be a tuple/list. (%r)' % (value_range, search_value)
+ if len(actual_value) != 2:
+ raise TypeError, 'Operator %r requires value to have a length of 2. len(%r) = %i (%r)' % (value_range, actual_value, len(actual_value), search_value)
+ comparison_operator = dual_operator_dict[value_range]
+ logical_operator = 'and'
+ else:
+ raise ValueError, 'Unknown "range" value in %r' % (search_value, )
+ if value_operator is not None:
+ if not isinstance(value_operator, basestring):
+ raise TypeError, 'Operator must be of a string type. Got a %r' % (type(value_operator), )
+ value_operator = value_operator.lower()
+ if not isinstance(actual_value, (tuple, list)):
+ raise TypeError, 'When specifying an operator, query must be a list.'
+ if value_operator == 'in':
+ comparison_operator = '='
+ logical_operator = 'or'
+ else:
+ logical_operator = value_operator
+ search_value = actual_value
+ # Cast to list
+ if isinstance(search_value, (tuple, list)):
+ # Check list content (not empty, homogenous)
+ search_value_len = len(search_value)
+ if search_value_len == 0:
+ raise ValueError, 'Value cannot be an empty list/tuple: %r' % (search_value, )
+ reference_class = search_value[0].__class__
+ for x in search_value[1:]:
+ if x.__class__ != reference_class:
+ raise TypeError, 'List elements must be of the same class: %r' % (search_value, )
else:
- # search string parsing is needed
- where_expressions, select_expressions = \
- self.buildSQLExpressionFromSearchString(key, str(value),
- format, mode, range_value, stat__)
- return where_expressions, select_expressions
-
- def buildSQLExpressionFromSearchString(self, key, value, format, mode, range_value, stat__):
- complex_query = self.buildQuery(key, value, format, mode, range_value, stat__)
- if complex_query is None:
- # Query could not be generated from search string
- sql_expression = {'where_expression': '1',
- 'select_expression_list': []}
+ assert logical_operator is None
+ if isinstance(search_value, dict):
+ reference_class = search_value['query'].__class__
+ else:
+ reference_class = search_value.__class__
+ search_value = [search_value]
+ if logical_operator is None:
+ logical_operator = default_logical_operator
+ operator_value_dict = {}
+ if None in search_value:
+ if comparison_operator not in (None, 'is'):
+ LOG('KeywordKey', 100, 'None value requires an "is" comparison operator. Fixed.')
+ operator_value_dict['is'] = search_value
+ elif comparison_operator is None:
+ if issubclass(reference_class, basestring):
+ if get_operator_from_value:
+ parsed = True
+ for value in search_value:
+ if isinstance(value, dict):
+ operator, value['query'] = self._getComparisonOperator(value['query'])
+ else:
+ operator, value = self._getComparisonOperator(value)
+ operator_value_dict.setdefault(operator, []).append(self._preprocessValue(value, operator))
+ else:
+ for value in search_value:
+ if isinstance(value, dict):
+ operator = self._guessComparisonOperator(value['query'])
+ else:
+ operator = self._guessComparisonOperator(value)
+ operator_value_dict.setdefault(operator, []).append(self._preprocessValue(value, operator))
+ else:
+ # XXX: comparison operator is hardcoded for non-strings.
+ operator_value_dict['='] = search_value
+ elif isinstance(comparison_operator, (tuple, list)):
+ assert len(comparison_operator) == len(search_value)
+ for operator, value in zip(comparison_operator, search_value):
+ operator_value_dict.setdefault(operator, []).append(value)
else:
- sql_expression = complex_query(keyword_search_keys = [],
- datetime_search_keys = [],
- full_text_search_keys = [])
- return sql_expression['where_expression'], sql_expression['select_expression_list']
+ operator_value_dict[comparison_operator] = search_value
+ return operator_value_dict, logical_operator, parsed
- def buildQuery(self, key, value, format, mode, range_value, stat__):
- """ Build Query """
- query_list = []
- # tokenize searchs string into tokens for Search Key
- tokens = self.tokenize(value)
+ @profiler_decorator
+ def _buildQuery(self, operator_value_dict, logical_operator, parsed, group):
+ """
+ Create Queries from values, logical and comparison operators.
- # split tokens list into one or more 'OR' tokens lists
- tokens_or_groups = self.groupByLogicalOperator(tokens, 'OR')
+ operator_value_dict (dict)
+ See _processSearchValue.
+ logical_operator (string)
+ See _processSearchValue.
+ parsed (bool)
+ See _processSearchValue.
+ group (string)
+ The gorup all queries will belong to.
+ """
+ column = self.getColumn()
+ query_list = []
+ append = query_list.append
+ if logical_operator == 'or' and '=' in operator_value_dict:
+ # Special case for equality with an 'or' logical operator: use SQL 'in'.
+ append(SimpleQuery(search_key=self, operator='in', group=group, **{column: operator_value_dict.pop('=')}))
+ for comparison_operator, value_list in operator_value_dict.iteritems():
+ for value in value_list:
+ append(SimpleQuery(search_key=self, operator=comparison_operator, group=group, **{column: value}))
+ return query_list
- # remove empty tokens lists
- tokens_or_groups = filter(lambda x: len(x), tokens_or_groups)
+ @profiler_decorator
+ def buildQuery(self, search_value, group=None, logical_operator=None, comparison_operator=None):
+ assert logical_operator in (None, 'and', 'or'), repr(logical_operator)
+ operator_value_dict, logical_operator, parsed = self._processSearchValue(search_value, logical_operator, comparison_operator)
+ query_list = self._buildQuery(operator_value_dict, logical_operator, parsed, group)
+ if len(query_list) == 1:
+ query = query_list[0]
+ else:
+ query = ComplexQuery(query_list, operator=logical_operator)
+ return query
- # get a ComplexQuery for a sub token list
- for tokens_or_group in tokens_or_groups:
- query = self.buildQueryForTokenList(tokens_or_group, key, value, format)
- if query is not None:
- # query could be generated for token list
- query_list.append(query)
+ def parseSearchText(self, value):
+ return None
- if len(query_list):
- # join query list in one really big ComplexQuery
- return ComplexQuery(*query_list,
- **{'operator':'OR'})
+verifyClass(ISearchKey, SearchKey)
- def buildQueryForTokenList(self, tokens, key, value, format):
- """ Build a ComplexQuery for a token list """
- query_list = []
- logical_groups = self.groupByLogicalOperator(tokens, 'AND')
- for group_tokens in logical_groups:
- token_values = [x.value for x in group_tokens]
- sub_operator, sub_tokens = self.getOperatorForTokenList(group_tokens)
- sub_tokens_values = [x.value for x in sub_tokens]
- query_kw = {key: ' '.join(sub_tokens_values),
- 'type': self.default_key_type,
- 'format': format,
- 'range': sub_operator}
- query_list.append(Query(**query_kw))
-
- # join query list in one really big ComplexQuery
- complex_query = ComplexQuery(*query_list,
- **{'operator': 'AND'})
- return complex_query
-
- def buildSQLExpressionFromRange(self, key, value, format, mode, range_value, stat__):
- """ This method will generate SQL expressions
- from explicitly passed list of values and
- range_value in ('min', 'max', ..)"""
- key = self.quoteSQLKey(key, format)
- where_expression = ''
- select_expressions = []
- if isinstance(value, (list, tuple)):
- if len(value) > 1:
- # value should contain at least two items
- query_min = self.quoteSQLString(value[0], format)
- query_max = self.quoteSQLString(value[1], format)
- else:
- # value contains only one item
- query_min = query_max = self.quoteSQLString(value[0], format)
- else:
- query_min = query_max = self.quoteSQLString(value, format)
- if range_value == 'min':
- where_expression = "%s >= %s" % (key, query_min)
- elif range_value == 'max':
- where_expression = "%s < %s" % (key, query_max)
- elif range_value == 'minmax' :
- where_expression = "%s >= %s AND %s < %s" % (key, query_min, key, query_max)
- elif range_value == 'minngt' :
- where_expression = "%s >= %s AND %s <= %s" % (key, query_min, key, query_max)
- elif range_value == 'ngt':
- where_expression = "%s <= %s" % (key, query_max)
- elif range_value == 'nlt':
- where_expression = "%s > %s" % (key, query_max)
- elif range_value == 'like':
- where_expression = "%s LIKE %s" % (key, query_max)
- elif range_value == 'not_like':
- where_expression = "%s NOT LIKE %s" % (key, query_max)
- elif range_value in ('=', '>', '<', '>=', '<=','!=',):
- where_expression = "%s %s %s" % (key, range_value, query_max)
- return where_expression, select_expressions
-
-
-## def groupByOperator(self, tokens, group_by_operators_list = operators):
-## """ Generic implementation of splitting tokens into logical
-## groups defided by respective list of logical operator
-## defined for respective search key. """
-## items = []
-## last_operator = None
-## operators_mapping_list = []
-## last_operator = {'operator': None,
-## 'tokens': []}
-## for token in tokens:
-## token_type = token.type
-## token_value = token.value
-## if token_type in group_by_operators_list:
-## # (re) init it
-## last_operator = {'operator': token,
-## 'tokens': []}
-## operators_mapping_list.append(last_operator)
-## else:
-## # not an operator just a value token
-## last_operator['tokens'].append(token)
-## if last_operator not in operators_mapping_list:
-## operators_mapping_list.append(last_operator)
-## return operators_mapping_list
diff --git a/product/ZSQLCatalog/SearchKey/__init__.py b/product/ZSQLCatalog/SearchKey/__init__.py
index 8b137891791fe96927ad78e64b0aad7bded08bdc..40a84b841184bd87b4e76ab74796569bd3ee00c3 100644
--- a/product/ZSQLCatalog/SearchKey/__init__.py
+++ b/product/ZSQLCatalog/SearchKey/__init__.py
@@ -1 +1,13 @@
+# Automaticaly import all SearchKeys
+import os
+module_path = os.path.dirname(os.path.abspath(__file__))
+global_dict = globals()
+__relative_file__ = os.path.basename(__file__)
+for filename in os.listdir(module_path):
+ if filename.endswith('.py') and filename != __relative_file__:
+ modulename = filename[:-3]
+ try:
+ module = __import__(modulename, global_dict, None, [])
+ except ImportError:
+ continue
diff --git a/product/ZSQLCatalog/SearchText/AdvancedSearchTextDetector.py b/product/ZSQLCatalog/SearchText/AdvancedSearchTextDetector.py
new file mode 100644
index 0000000000000000000000000000000000000000..4bcfe712c62d3e36e8f6c9855a04356fb92430ba
--- /dev/null
+++ b/product/ZSQLCatalog/SearchText/AdvancedSearchTextDetector.py
@@ -0,0 +1,110 @@
+##############################################################################
+#
+# Copyright (c) 2008-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+from lexer import lexer, update_docstrings
+
+# TODO: remove the special OPERATOR case: it does not work when there are both a valid and an invalid operator
+
+class AdvancedSearchTextDetector(lexer):
+
+ def t_OPERATOR(self, t):
+ r'(>=?|<=?|!?=)[ ]*'
+ return t
+
+ def t_LEFT_PARENTHESE(self, t):
+ self.found = True
+ t.type = 'WORD'
+ return t
+
+ def t_STRING(self, t):
+ self.found = True
+ t.type = 'WORD'
+ return t
+
+ def t_COLUMN(self, t):
+ self.found = True
+ t.type = 'WORD'
+ return t
+
+ def t_OR(self, t):
+ self.found = True
+ t.type = 'WORD'
+ return t
+
+ def t_AND(self, t):
+ self.found = True
+ t.type = 'WORD'
+ return t
+
+ def t_NOT(self, t):
+ self.found = True
+ t.type = 'WORD'
+ return t
+
+ def p_search_text(self, p):
+ '''search_text : value
+ | value search_text'''
+ if len(p) == 2:
+ p[0] = p[1]
+ else:
+ p[0] = p[1] or p[2]
+
+ def p_value(self, p):
+ '''value : WORD
+ | OPERATOR WORD'''
+ p[0] = len(p) == 3 and ' ' not in p[1]
+
+ tokens = (
+ 'WORD',
+ 'OPERATOR')
+
+ def real_token(self):
+ return lexer.token(self)
+
+ def token(self):
+ return self.token_list.pop(0)
+
+ def __call__(self, input):
+ self.found = False
+ check_grammar = False
+ self.token_list = token_list = []
+ append = token_list.append
+ self.input(input)
+ while not self.found:
+ token = self.real_token()
+ append(token)
+ if token is None:
+ break
+ if token.type == 'OPERATOR':
+ check_grammar = True
+ if not self.found and check_grammar:
+ self.found = self.parse()
+ return self.found
+
+update_docstrings(AdvancedSearchTextDetector)
+
diff --git a/product/ZSQLCatalog/SearchText/AdvancedSearchTextParser.py b/product/ZSQLCatalog/SearchText/AdvancedSearchTextParser.py
new file mode 100644
index 0000000000000000000000000000000000000000..0192514d5822fc08bc391d27612d6c95ae16d76d
--- /dev/null
+++ b/product/ZSQLCatalog/SearchText/AdvancedSearchTextParser.py
@@ -0,0 +1,229 @@
+##############################################################################
+#
+# Copyright (c) 2008-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+from lexer import lexer, update_docstrings
+try:
+ from Products.ZSQLCatalog.Interface.IAbstractSyntaxNode import INode, IValueNode, ILogicalNode, IColumnNode
+ from Interface.Verify import verifyClass
+except ImportError:
+ INode = None
+ IValueNode = None
+ ILogicalNode = None
+ IColumnNode = None
+ def verifyClass(*args, **kw):
+ pass
+
+class Node(object):
+
+ __implements__ = INode
+
+ def isLeaf(self):
+ return False
+
+ def isColumn(self):
+ return False
+
+ def push(self, logical_operator, node):
+ return LogicalNode(logical_operator, self, node)
+
+verifyClass(INode, Node)
+
+class ValueNode(Node):
+
+ __implements__ = IValueNode
+
+ def __init__(self, value, comparison_operator=''):
+ self.value = value
+ self.comparison_operator = comparison_operator
+
+ def isLeaf(self):
+ return True
+
+ def getComparisonOperator(self):
+ return self.comparison_operator
+
+ def getValue(self):
+ return self.value
+
+ def __repr__(self):
+ return '<%s %r %r>' % (self.__class__.__name__, self.comparison_operator, self.value)
+
+verifyClass(INode, ValueNode)
+verifyClass(IValueNode, ValueNode)
+
+class NotNode(Node):
+
+ __implements__ = ILogicalNode
+
+ def __init__(self, node):
+ self.node = node
+
+ def getLogicalOperator(self):
+ return 'not'
+
+ def getNodeList(self):
+ return [self.node]
+
+ def __repr__(self):
+ return '<%s %r>' % (self.__class__.__name__, self.node)
+
+verifyClass(INode, NotNode)
+verifyClass(ILogicalNode, NotNode)
+
+class LogicalNode(Node):
+
+ __implements__ = ILogicalNode
+
+ def __init__(self, logical_operator, node, other):
+ self.logical_operator = logical_operator
+ self.node_list = []
+ self._push(node)
+ self._push(other)
+
+ def getLogicalOperator(self):
+ return self.logical_operator
+
+ def getNodeList(self):
+ return self.node_list
+
+ def _push(self, node):
+ if isinstance(node, LogicalNode) and node.logical_operator == self.logical_operator:
+ self.node_list.extend(node.node_list)
+ else:
+ self.node_list.append(node)
+
+ def __repr__(self):
+ return '<%s %r %r>' % (self.__class__.__name__, self.logical_operator, self.node_list)
+
+verifyClass(INode, LogicalNode)
+verifyClass(ILogicalNode, LogicalNode)
+
+class ColumnNode(Node):
+
+ __implements__ = IColumnNode
+
+ def __init__(self, column_name, node):
+ self.column_name = column_name
+ self.node = node
+
+ def isColumn(self):
+ return True
+
+ def getColumnName(self):
+ return self.column_name
+
+ def getSubNode(self):
+ return self.node
+
+ def __repr__(self):
+ return '<%s %r %r>' % (self.__class__.__name__, self.column_name, self.node)
+
+verifyClass(INode, ColumnNode)
+verifyClass(IColumnNode, ColumnNode)
+
+class AdvancedSearchTextParser(lexer):
+
+ def p_seach_text(self, p):
+ '''search_text : and_expression
+ | and_expression OR search_text
+ | and_expression search_text'''
+ if len(p) == 2:
+ p[0] = p[1]
+ elif len(p) == 3:
+ p[0] = p[1].push('or', p[2])
+ else:
+ p[0] = p[1].push('or', p[3])
+
+ def p_and_expression(self, p):
+ '''and_expression : boolean_expression
+ | boolean_expression AND and_expression'''
+ if len(p) == 2:
+ p[0] = p[1]
+ else:
+ p[0] = p[1].push('and', p[3])
+
+ def p_boolean_expression(self, p):
+ '''boolean_expression : NOT expression
+ | expression'''
+ if len(p) == 3:
+ p[0] = NotNode(p[2])
+ else:
+ p[0] = p[1]
+
+ def p_expression(self, p):
+ '''expression : LEFT_PARENTHESE search_text RIGHT_PARENTHESE
+ | column
+ | value'''
+ if len(p) == 2:
+ p[0] = p[1]
+ else:
+ p[0] = p[2]
+
+ def p_column(self, p):
+ '''column : COLUMN value_expression'''
+ p[0] = ColumnNode(p[1], p[2])
+
+ def p_value_expression(self, p):
+ '''value_expression : LEFT_PARENTHESE value_or_expression RIGHT_PARENTHESE
+ | value'''
+ if len(p) == 2:
+ p[0] = p[1]
+ else:
+ p[0] = p[2]
+
+ def p_value_or_expression(self, p):
+ '''value_or_expression : value_and_expression
+ | value_and_expression value_or_expression
+ | value_and_expression OR value_or_expression'''
+ if len(p) == 2:
+ p[0] = p[1]
+ elif len(p) == 3:
+ p[0] = p[1].push('or', p[2])
+ else:
+ p[0] = p[1].push('or', p[3])
+
+ def p_value_and_expression(self, p):
+ '''value_and_expression : value_expression
+ | value_expression AND value_and_expression'''
+ if len(p) == 2:
+ p[0] = p[1]
+ else:
+ p[0] = p[1].push('and', p[3])
+
+ def p_value(self, p):
+ '''value : OPERATOR STRING
+ | OPERATOR WORD
+ | STRING
+ | WORD'''
+ if len(p) == 2:
+ p[0] = ValueNode(p[1])
+ else:
+ p[0] = ValueNode(p[2], comparison_operator=p[1])
+
+update_docstrings(AdvancedSearchTextParser)
+
diff --git a/product/ZSQLCatalog/SearchText/SearchTextParser.py b/product/ZSQLCatalog/SearchText/SearchTextParser.py
new file mode 100644
index 0000000000000000000000000000000000000000..a86942abb2d87eb3a7831aadb845ff14c2b7953f
--- /dev/null
+++ b/product/ZSQLCatalog/SearchText/SearchTextParser.py
@@ -0,0 +1,333 @@
+#!/usr/bin/python
+# -*- coding: utf8 -*-
+##############################################################################
+#
+# Copyright (c) 2008-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+import threading
+from AdvancedSearchTextDetector import AdvancedSearchTextDetector
+from AdvancedSearchTextParser import AdvancedSearchTextParser
+from lexer import ParserOrLexerError
+try:
+ from Products.ZSQLCatalog.SQLCatalog import profiler_decorator
+except ImportError:
+ def profiler_decorator(funct):
+ return funct
+
+if __name__ == '__main__':
+ DEBUG = 1
+else:
+ DEBUG = 0
+
+parser_pool = threading.local()
+
+@profiler_decorator
+def getAdvancedSearchTextDetector():
+ try:
+ return parser_pool.advanced_search_text_detector
+ except AttributeError:
+ advanced_search_text_detector = AdvancedSearchTextDetector()
+ advanced_search_text_detector.init(debug=DEBUG)
+ parser_pool.advanced_search_text_detector = advanced_search_text_detector
+ return advanced_search_text_detector
+
+@profiler_decorator
+def getAdvancedSearchTextParser():
+ try:
+ return parser_pool.parser
+ except AttributeError:
+ parser = AdvancedSearchTextParser()
+ parser.init(debug=DEBUG)
+ parser_pool.parser = parser
+ return parser
+
+@profiler_decorator
+def _parse(input, *args, **kw):
+ if getAdvancedSearchTextDetector()(input):
+ result = getAdvancedSearchTextParser()(input, *args, **kw)
+ else:
+ result = None
+ return result
+
+@profiler_decorator
+def parse(*args, **kw):
+ try:
+ result = _parse(*args, **kw)
+ except (KeyboardInterrupt, SystemExit):
+ raise
+ except:
+ result = None
+ return result
+
+if __name__ == '__main__':
+ class Query:
+ def __init__(self, column, value, comparison_operator='='):
+ self.column = column
+ self.comparison_operator = comparison_operator
+ if isinstance(value, (list, tuple)):
+ value = ''.join(value)
+ self.value = value
+
+ def asTuple(self):
+ return (self.column, self.value, self.comparison_operator)
+
+ def __repr__(self):
+ value = self.value
+ if len(value) == 1:
+ value = value[0]
+ return 'Query(%r, %r, %r)' % (self.column, value, self.comparison_operator)
+
+ def __eq__(self, other):
+ if isinstance(other, Query):
+ return self.asTuple() == other.asTuple()
+ else:
+ return False
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ class ComplexQuery:
+ def __init__(self, query_list, operator):
+ self.operator = operator
+ self.query_list = query_list
+
+ def __repr__(self):
+ return 'ComplexQuery(%r, operator=%r)' % (self.query_list, self.operator)
+
+ def __eq__(self, other):
+ if isinstance(other, ComplexQuery):
+ if self.operator != other.operator:
+ return False
+ other_query_list = other.query_list[:]
+ for my_query in self.query_list:
+ for other_index in xrange(len(other_query_list)):
+ other_query = other_query_list[other_index]
+ if my_query == other_query:
+ other_query_list.pop(other_index)
+ break
+ else:
+ return False
+ return len(other_query_list) == 0
+ else:
+ return False
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ check_list = [
+ ('foo', None),
+ ('foo bar', None),
+ ('foo bar', None),
+ ('foo%', None),
+ ('%foo', None),
+ ('%foo%', None),
+ ('foo%bar', None),
+ ('foo% bar', None),
+ ('foo %bar', None),
+ ('foo and bar', None),
+ ('foo or bar', None),
+ ('foo - bar', None),
+ ('foo- bar', None),
+ ('!-1', None),
+ ('->1', None),
+ ('+=1', None),
+ ('jean-paul', None),
+ ('JeanAndPaul', None),
+ ('totoORtata', None),
+ ('NORD', None),
+ ('OR ARGENT', None),
+ ('CUIVRE OR ARGENT', None), # XXX
+ ('title :foo', None),
+ ('-foo', None),
+ ('foo -bar', None),
+ ('+foo -bar', None),
+ ('+1', None),
+ ('-1', None),
+
+ ('foo OR "-" OR bar OR -baz', ComplexQuery([Query(None, 'foo'), Query(None, '-'), Query(None, 'bar'), Query(None, '-baz')], operator='or')),
+ ('foo "-" bar -baz', ComplexQuery([Query(None, 'foo'), Query(None, '-'), Query(None, 'bar'), Query(None, '-baz')], operator='or')),
+ ('title:foo', Query('title', 'foo')),
+ ('title: foo', Query('title', 'foo')),
+ ('title:foo bar', ComplexQuery([Query('title', 'foo'), Query(None, 'bar')], operator='or')),
+ ('title:"foo bar"', Query('title', 'foo bar')),
+ ('"title:foo bar"', Query(None, 'title:foo bar')),
+ ('"foo bar"', Query(None, 'foo bar')),
+ ('"foo bar"', Query(None, 'foo bar')),
+ ('foo AND bar', ComplexQuery([Query(None, 'foo'), Query(None, 'bar')], operator='and')),
+ ('foo OR bar', ComplexQuery([Query(None, 'foo'), Query(None, 'bar')], operator='or')),
+ ('"foo AND bar"', Query(None, 'foo AND bar')),
+ ('"foo and bar"', Query(None, 'foo and bar')),
+ ('"foo OR bar"', Query(None, 'foo OR bar')),
+ ('"foo or bar"', Query(None, 'foo or bar')),
+ ('"foo% bar"', Query(None, 'foo% bar')),
+ ('"foo %bar"', Query(None, 'foo %bar')),
+ ('>1', Query(None, '1', '>')),
+ ('">1"', Query(None, '>1')),
+ ('>a', Query(None, 'a', '>')),
+ ('">a"', Query(None, '>a')),
+ ('>1 0', ComplexQuery([Query(None, '1', '>'), Query(None, '0')], operator='or')),
+ ('>=1', Query(None, '1', '>=')),
+ ('>"=1"', Query(None, '=1', '>')),
+ ('-"1"', ComplexQuery([Query(None, '-'), Query(None, '1')], operator='or')),
+ ('"!-1"', Query(None, '!-1')),
+# (r"a:'tu:\'tu\''", ['a', "tu:'tu'"]),
+ (r'''b:"tu:\'tu\'"''', Query('b', "tu:\\'tu\\'")),
+ (r'''c:"tu:'tu'"''', Query('c', "tu:'tu'")),
+ (r'd:"tu:\"tu\""', Query('d', 'tu:"tu"')),
+ ('toto: tutu tutu', ComplexQuery([Query('toto', 'tutu'), Query(None, 'tutu')], operator='or')),
+ ('(tutu) (toto:tata)', ComplexQuery([Query(None, 'tutu'), Query('toto', 'tata')], operator='or')),
+ ('(tutu) (toto:"tata")', ComplexQuery([Query(None, 'tutu'), Query('toto', 'tata')], operator='or')),
+# ('toto:', ['toto', '']),
+ ('toto:""', Query('toto', '')),
+# ("''", ''),
+ ('""', Query(None, '')),
+ (r'"\""', Query(None, '"')),
+ (r'"\n"', Query(None, '\\n')),
+#ã“ã‚“ã«ã¡ã¯
+ (u'ã‚“', None),
+ (u'(toto:ã‚“) OR (titi:ã‚“)', ComplexQuery([Query('toto', u'ã‚“'), Query('titi', u'ã‚“')], operator='or')),
+ ('ã‚“', None),
+ ('(toto:ã‚“) OR (titi:ã‚“)', ComplexQuery([Query('toto', 'ã‚“'), Query('titi', 'ã‚“')], operator='or')),
+ ('(foo)', Query(None, 'foo')),
+ ('toto:(foo)', Query('toto', 'foo')),
+ ('(foo OR bar)', ComplexQuery([Query(None, 'foo'), Query(None, 'bar')], operator='or')),
+ ('(a AND b) OR (c AND (d OR e))',
+ ComplexQuery([ComplexQuery([Query(None, 'a'), Query(None, 'b')], operator='and'), ComplexQuery([Query(None, 'c'), ComplexQuery([Query(None, 'd'), Query(None, 'e')], operator='or')], operator='and')], operator='or')),
+ ('(foo:"") (bar:baz)', ComplexQuery([Query('foo', ''), Query('bar', 'baz')], operator='or')),
+ ('(foo:"") (OR:bar)', ComplexQuery([Query('foo', ''), Query('OR', 'bar')], operator='or')),
+# ('foo: OR', ['foo', 'or']),
+# ('foo: OR ', ['foo', 'or']),
+# ('(foo:)', ['foo', '']),
+ ('(foo: bar)', Query('foo', 'bar')),
+ ('(a:b) AND (c:d)', ComplexQuery([Query('a', 'b'), Query('c', 'd')], operator='and')),
+ ('a:(b c)', ComplexQuery([Query('a', 'b'), Query('a', 'c')], operator='or')),
+ ('a:(b OR c)', ComplexQuery([Query('a', 'b'), Query('a', 'c')], operator='or')),
+ ('a:(b c d)', ComplexQuery([Query('a', 'b'), Query('a', 'c'), Query('a', 'd')], operator='or')),
+ ('a:(b (c d))', ComplexQuery([Query('a', 'b'), Query('a', 'c'), Query('a', 'd')], operator='or')),
+ ('a:(b OR (c d))', ComplexQuery([Query('a', 'b'), Query('a', 'c'), Query('a', 'd')], operator='or')),
+ ('"JeanANDPaul"', Query(None, 'JeanANDPaul')),
+ ('"Jean" AND "Paul"', ComplexQuery([Query(None, 'Jean'), Query(None, 'Paul')], operator='and')),
+ ('"jean paul" OR "thierry"', ComplexQuery([Query(None, 'jean paul'), Query(None, 'thierry')], operator='or')),
+ ('title:Paul Jean Lili', ComplexQuery([Query('title', 'Paul'), Query(None, 'Jean'), Query(None, 'Lili')], operator='or')),
+ ('toto AND titi OR tutu AND tata OR toto',
+ ComplexQuery([ComplexQuery([Query(None, 'toto'), Query(None, 'titi')], operator='and'), ComplexQuery([Query(None, 'tutu'), Query(None, 'tata')], operator='and'), Query(None, 'toto')], operator='or')),
+ ('toto AND (titi OR tutu) AND tata OR toto',
+ ComplexQuery([ComplexQuery([Query(None, 'toto'), ComplexQuery([Query(None, 'titi'), Query(None, 'tutu')], operator='or'), Query(None, 'tata')], operator='and'), Query(None, 'toto')], operator='or')),
+ ('"OR ARGENT"', Query(None, 'OR ARGENT')),
+ ('1 AND 2 OR 3', ComplexQuery([ComplexQuery([Query(None, '1'), Query(None, '2')], operator='and'), Query(None, '3')], operator='or')),
+ ('1 OR 2 AND 3', ComplexQuery([Query(None, '1'), ComplexQuery([Query(None, '2'), Query(None, '3')], operator='and')], operator='or')),
+ ('1 AND 2 3', ComplexQuery([ComplexQuery([Query(None, '1'), Query(None, '2')], operator='and'), Query(None, '3')], operator='or')),
+ ('1 2 AND 3', ComplexQuery([Query(None, '1'), ComplexQuery([Query(None, '2'), Query(None, '3')], operator='and')], operator='or')),
+ ('10 11 OR 12 13', ComplexQuery([Query(None, '10'), Query(None, '11'), Query(None, '12'), Query(None, '13')], operator='or')),
+ ('((1 AND 2 OR 3) OR (4 AND 5 6) OR (7 8 AND 9) OR (10 11 OR 12 13))',
+ ComplexQuery([ComplexQuery([Query(None, '1'), Query(None, '2')], operator='and'), Query(None, '3'), ComplexQuery([Query(None, '4'), Query(None, '5')], operator='and'), Query(None, '6'), Query(None, '7'), ComplexQuery([Query(None, '8'), Query(None, '9')], operator='and'), Query(None, '10'), Query(None, '11'), Query(None, '12'), Query(None, '13')], operator='or')),
+ ('((titi:foo) AND (toto:bar)) OR ((titi:bar) AND (toto:foo))',
+ ComplexQuery([ComplexQuery([Query('titi', 'foo'), Query('toto', 'bar')], operator='and'), ComplexQuery([Query('titi', 'bar'), Query('toto', 'foo')], operator='and')], operator='or')),
+ ('title:(Paul Jean OR Lili)', ComplexQuery([Query('title', 'Paul'), Query('title', 'Jean'), Query('title', 'Lili')], operator='or')),
+ ('title:Paul Jean OR Lili', ComplexQuery([Query('title', 'Paul'), Query(None, 'Jean'), Query(None, 'Lili')], operator='or')),
+ ]
+
+ def walk(node, key=None):
+ """
+ Recusrively walk given AST and build ComplexQuery & Query instances for each node.
+ """
+ if node.isLeaf():
+ comparison_operator = node.getComparisonOperator()
+ if comparison_operator == '':
+ comparison_operator = '='
+ result = Query(key, node.getValue(), comparison_operator=comparison_operator)
+ elif node.isColumn():
+ result = walk(node.getSubNode(), node.getColumnName())
+ else:
+ query_list = [walk(x, key) for x in node.getNodeList()]
+ operator = node.getLogicalOperator()
+ if operator == 'not' or len(query_list) > 1:
+ result = ComplexQuery(query_list, operator=operator)
+ elif len(query_list) == 1:
+ result = query_list[0]
+ else:
+ result = None
+ return result
+
+ original_parse = _parse
+
+ def parse(input, *args, **kw):
+ """
+ Parse input and walk generated AST.
+ """
+ result = original_parse(input, *args, **kw)
+ if result is not None:
+ #print repr(result)
+ result = walk(result)
+ return result
+
+ success_count = 0
+ for input, expected in check_list:
+ try:
+ result = parse(input)
+ except ParserOrLexerError, message:
+ print "ERROR when checking %r" % (input, )
+ print " crashed with: %s" % (message, )
+ print " instead of producing %r" % (expected, )
+ else:
+ if result != expected:
+ print "ERROR when checking %r:" % (input, )
+ print " produced %r" % (result, )
+ print " instead of %r" % (expected, )
+ else:
+ success_count += 1
+ print '%i/%i checks succeeded.' % (success_count, len(check_list))
+ while 1:
+ try:
+ input = raw_input('catalog> ')
+ except (EOFError, KeyboardInterrupt):
+ break
+ print repr(input)
+ try:
+ try:
+ detector_result = getAdvancedSearchTextDetector()(input)
+ except ParserOrLexerError, message:
+ print ' Detector raise: %r' % (message, )
+ detector_result = False
+ else:
+ print ' Detector: %r' % (detector_result, )
+ if detector_result:
+ print ' LEX:'
+ lexer = getAdvancedSearchTextParser().lexer
+ lexer.input(input)
+ while 1:
+ tok = lexer.token()
+ if not tok: break # No more input
+ print ' %s' % (tok, )
+ print ' YACC:'
+ print ' %r' % (parse(input, debug=2), )
+ else:
+ print ' %r' % (input, )
+ except ParserOrLexerError, message:
+ print message
+ print
+
diff --git a/product/ZSQLCatalog/SearchText/__init__.py b/product/ZSQLCatalog/SearchText/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e45dba662a6d824b94d6e02c9e09defd383bf248
--- /dev/null
+++ b/product/ZSQLCatalog/SearchText/__init__.py
@@ -0,0 +1,2 @@
+from SearchTextParser import parse
+
diff --git a/product/ZSQLCatalog/SearchText/lexer.py b/product/ZSQLCatalog/SearchText/lexer.py
new file mode 100644
index 0000000000000000000000000000000000000000..ad6ac0c1268bd7d7321014ef8fcf5993cba6494d
--- /dev/null
+++ b/product/ZSQLCatalog/SearchText/lexer.py
@@ -0,0 +1,160 @@
+##############################################################################
+#
+# Copyright (c) 2008-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+from ply import lex, yacc
+import os
+import sys
+from cStringIO import StringIO
+
+try:
+ from zLOG import LOG
+except ImportError:
+ def LOG(channel, level, message):
+ print >>sys.stderr, message
+
+module_path = os.path.dirname(os.path.abspath(__file__))
+
+class ParserOrLexerError(Exception):
+ pass
+
+class LexerError(ParserOrLexerError):
+ pass
+
+class ParserError(ParserOrLexerError):
+ pass
+
+class lexer(object):
+ def init(self, **kw):
+ debug = kw.pop('debug', False)
+ # Catch all logs with a cStringIO
+ output = sys.stdout = sys.stderr = StringIO()
+ self.lexer = lex.lex(object=self, **kw)
+ self.parser = yacc.yacc(module=self, debug=debug,
+ debugfile="%s.out" % (self.__class__.__name__, ),
+ tabmodule="%s_parsetab" % (self.__class__.__name__, ),
+ outputdir=module_path)
+ sys.stdout, sys.stderr = sys.__stdout__, sys.__stderr__
+ # Emit all logs with regular Zope logging
+ for line in output.getvalue().split('\n'):
+ if len(line):
+ LOG('lexer', 0, line)
+
+ def t_error(self, t):
+ raise LexerError, 'ERROR: Illegal character %r' % (t.value[0], )
+
+ def p_error(self, p):
+ raise ParserError, 'Syntax error in input: %r' % (p, )
+
+ def input(self, string):
+ self.lexer.input(string)
+
+ def token(self):
+ return self.lexer.token()
+
+ tokens = (
+ 'OR',
+ 'AND',
+ 'NOT',
+ 'COLUMN',
+ 'STRING',
+ 'WORD',
+ 'OPERATOR',
+ 'LEFT_PARENTHESE',
+ 'RIGHT_PARENTHESE')
+
+ t_ignore = ' '
+
+ def t_LEFT_PARENTHESE(self, t):
+ r'\('
+ return t
+
+ def t_RIGHT_PARENTHESE(self, t):
+ r'\)'
+ return t
+
+ def t_OPERATOR(self, t):
+ r'(>=?|<=?|!?=)'
+ return t
+
+ def t_STRING(self, t):
+ r'"(\\.|[^\\"])*"'
+ # Unescape value and strip surrounding quotes
+ value_list = []
+ append = value_list.append
+ escaped = False
+ for char in t.value[1:-1]:
+ if escaped:
+ escaped = False
+ if char != '"':
+ append('\\')
+ else:
+ if char == '\\':
+ escaped = True
+ continue
+ append(char)
+ assert not escaped
+ t.value = ''.join(value_list)
+ return t
+
+ def t_COLUMN(self, t):
+ r'[^><= :\(\)"][^ :\(\)"]*:'
+ t.value = t.value[:-1]
+ return t
+
+ def t_OR(self, t):
+ r'OR'
+ return t
+
+ def t_AND(self, t):
+ r'AND'
+ return t
+
+ def t_NOT(self, t):
+ r'NOT'
+ return t
+
+ def t_WORD(self, t):
+ r'[^><= :\(\)"][^ :\(\)"]*'
+ return t
+
+ def parse(self, *args, **kw):
+ kw['lexer'] = self
+ return self.parser.parse(*args, **kw)
+
+ __call__ = parse
+
+def update_docstrings(klass):
+ for property in dir(klass):
+ if property.startswith('t_'):
+ source = getattr(lexer, property, None)
+ if callable(source):
+ destination = getattr(klass, property)
+ assert callable(destination)
+ if destination.__doc__ is None:
+ destination.im_func.__doc__ = source.__doc__
+
diff --git a/product/ZSQLCatalog/ZSQLCatalog.py b/product/ZSQLCatalog/ZSQLCatalog.py
index ec63a320557c89d1283b7bb7bf8ee2cd5a4f87de..54cb28916b9ea7082cead717cfd226623c367641 100644
--- a/product/ZSQLCatalog/ZSQLCatalog.py
+++ b/product/ZSQLCatalog/ZSQLCatalog.py
@@ -790,8 +790,9 @@ class ZCatalog(Folder, Persistent, Implicit):
except ConflictError:
raise
except:
+ raise
LOG('WARNING ZSQLCatalog', 0, 'wrapObject failed on the object %r' % (obj,), error=sys.exc_info())
- failed_object_list.append(obj)
+ failed_object_list.append(obj) # XXX Strange JPS - why LOG and keep on ?? wrap_obj not defined
# run activity or execute for each archive depending on priority
if len(catalog_dict):
@@ -1043,7 +1044,7 @@ class ZCatalog(Folder, Persistent, Implicit):
security.declarePublic('buildSqlQuery')
buildSqlQuery = buildSQLQuery
- def searchResults(self, REQUEST=None, used=None, sql_catalog_id=None, **kw):
+ def searchResults(self, REQUEST=None, sql_catalog_id=None, **kw):
"""
Search the catalog according to the ZTables search interface.
Search terms can be passed in the REQUEST or as keyword
@@ -1051,18 +1052,18 @@ class ZCatalog(Folder, Persistent, Implicit):
"""
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
- return apply(catalog.searchResults, (REQUEST,used), kw)
+ return apply(catalog.searchResults, (REQUEST, ), kw)
return []
__call__=searchResults
- def countResults(self, REQUEST=None, used=None, sql_catalog_id=None, **kw):
+ def countResults(self, REQUEST=None, sql_catalog_id=None, **kw):
"""
Counts the number of items which satisfy the query defined in kw.
"""
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
- return apply(catalog.countResults, (REQUEST,used), kw)
+ return apply(catalog.countResults, (REQUEST, ), kw)
return []
## this stuff is so the find machinery works
diff --git a/product/ZSQLCatalog/__init__.py b/product/ZSQLCatalog/__init__.py
index 8926814e06ed98e7cdfe9f009d226d701ed71e4a..703d9f14c064bf4dea7854e4eeba99434e679a41 100644
--- a/product/ZSQLCatalog/__init__.py
+++ b/product/ZSQLCatalog/__init__.py
@@ -44,3 +44,6 @@ def initialize(context):
from AccessControl import ModuleSecurityInfo, ClassSecurityInfo
ModuleSecurityInfo('Products.ZSQLCatalog.SQLCatalog').declarePublic(
'ComplexQuery', 'Query', 'NegatedQuery',)
+
+from Query import Query, SimpleQuery
+from SearchKey import SearchKey
\ No newline at end of file
diff --git a/product/ZSQLCatalog/tests/testSQLCatalog.py b/product/ZSQLCatalog/tests/testSQLCatalog.py
new file mode 100644
index 0000000000000000000000000000000000000000..51ab9bc2e6436885e28e140726dd9c3a3e2aa2e1
--- /dev/null
+++ b/product/ZSQLCatalog/tests/testSQLCatalog.py
@@ -0,0 +1,301 @@
+##############################################################################
+#
+# Copyright (c) 2006-2009 Nexedi SA and Contributors. All Rights Reserved.
+# Jerome Perrin <jerome@nexedi.com>
+# Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+import unittest
+import sys
+
+from DateTime import DateTime
+from Products.ZSQLMethods.SQL import SQL as ZSQLMethod
+from Products.CMFCore.Expression import Expression
+
+from Products.ZSQLCatalog.SQLCatalog import Catalog as SQLCatalog
+from Products.ZSQLCatalog.ZSQLCatalog import ZCatalog as ZSQLCatalog
+from Products.ZSQLCatalog.SQLCatalog import Query
+from Products.ZSQLCatalog.SQLCatalog import ComplexQuery
+from Products.ZSQLCatalog.SQLCatalog import SimpleQuery
+from Products.ZSQLCatalog.SQLCatalog import NegatedQuery
+from Products.ZSQLCatalog.Query.Query import Query as _Query
+from Products.ZSQLCatalog.Query.EntireQuery import EntireQuery
+from Products.ZSQLCatalog.Query.RelatedQuery import RelatedQuery
+from DateTime import DateTime
+
+class ReferenceQuery:
+ operator = None
+ column = None
+ value = None
+
+ def __init__(self, *args, **kw):
+ self.operator = kw.pop('operator', None)
+ assert len(args) == 0 or len(kw) == 0
+ self.args = []
+ for arg in args:
+ if isinstance(arg, (tuple, list)):
+ self.args.extend(arg)
+ else:
+ self.args.append(arg)
+ if len(kw) == 1:
+ self.column, self.value = kw.items()[0]
+ elif len(kw) > 1:
+ raise ValueError, 'kw must not have more than one item: %r' % (kw, )
+
+ def __eq__(self, other):
+ if isinstance(other, SimpleQuery):
+ return self.column is not None and \
+ other.getColumn() == self.column and \
+ other.getValue() == self.value and \
+ other.operator == self.operator
+ elif isinstance(other, ComplexQuery):
+ if not (len(other.query_list) == len(self.args) and \
+ other.logical_operator == self.operator):
+ return False
+ other_query_list = other.query_list[:]
+ for subquery in self.args:
+ for other_query_id in xrange(len(other_query_list)):
+ other_query = other_query_list[other_query_id]
+ if subquery == other_query:
+ other_query_list.pop(other_query_id)
+ break
+ else:
+ return False
+ return len(other_query_list) == 0
+ elif isinstance(other, EntireQuery):
+ return len(self.args) == 1 and \
+ self.args[0] == other.query
+ elif isinstance(other, RelatedQuery):
+ return self == other.join_condition
+ elif isinstance(other, Query):
+ return self == other.wrapped_query
+ else:
+ raise TypeError, 'Compared value is not a (known) Query instance: (%s) %r' % (other.__class__.__name__, other)
+
+ def __repr__(self):
+ return '<%s column=%r operator=%r value=%r args=%r>' % \
+ (self.__class__.__name__, self.column, self.operator, self.value, self.args)
+
+class RelatedReferenceQuery:
+ def __init__(self, reference_subquery):
+ self.subquery = reference_subquery
+
+ def __eq__(self, other):
+ return isinstance(other, RelatedQuery) and \
+ self.subquery == other.join_condition
+
+ def __repr__(self):
+ return '<%s %r>' % (self.__class__.__name__, self.subquery)
+
+class DummyCatalog(SQLCatalog):
+ """
+ Mimic a table stucture.
+ """
+
+ sql_catalog_keyword_search_keys = ('keyword', )
+ sql_catalog_datetime_search_keys = ('date', )
+
+ def getColumnMap(self):
+ return {
+ 'uid': ['foo', 'bar'],
+ 'default': ['foo', ],
+ 'keyword': ['foo', ],
+ 'date': ['foo', ],
+ 'other_uid': ['bar', ]
+ }
+
+ def getSQLCatalogRelatedKeyList(self, key_list):
+ return [
+ 'related_default | bar,foo/default/z_related_table',
+ 'related_keyword | bar,foo/keyword/z_related_table',
+ 'related_date | bar,foo/date/z_related_table'
+ ]
+
+ def z_related_table(self, *args, **kw):
+ assert kw.get('src__', False)
+ assert 'query_table' in kw
+ assert 'table_0' in kw
+ assert 'table_1' in kw
+ assert len(kw) == 4
+ return '%(table_0)s.uid = %(query_table)s.uid AND %(table_0)s.other_uid = %(table_1)s' % kw
+
+class TestSQLCatalog(unittest.TestCase):
+ def setUp(self):
+ self._catalog = DummyCatalog('dummy_catalog')
+
+ def assertCatalogRaises(self, exception, kw):
+ self.assertRaises(exception, self._catalog, src__=1, query_table='foo', **kw)
+
+ def catalog(self, reference_tree, kw, check_search_text=True):
+ reference_param_dict = self._catalog._queryResults(query_table='foo', **kw)
+ query = self._catalog.buildQuery(kw)
+ self.assertEqual(reference_tree, query)
+ search_text = query.asSearchTextExpression(self._catalog)
+ if check_search_text:
+ # XXX: sould "keyword" be always used for search text searches ?
+ search_text_param_dict = self._catalog._queryResults(query_table='foo', keyword=search_text)
+ self.assertEqual(reference_param_dict, search_text_param_dict,
+ 'Query: %r\nSearchText: %r\nReference: %r\nSecond rendering: %r' % \
+ (query, search_text, reference_param_dict, search_text_param_dict))
+
+ def test_001_DefaultKey(self):
+ for column in ('default', 'related_default'):
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='=', default='a'), operator='and'),
+ {column: 'a'})
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='like', default='%a'), operator='and'),
+ {column: '%a'})
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='<', default='a'), operator='and'),
+ {column: '<a'})
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='<', default='path'), operator='and'),
+ {column: {'query': 'path', 'range': 'max'}})
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='in', default=['a', 'b']), operator='and'),
+ {column: ['a', 'b']})
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='in', default=['a', 'b']), operator='and'),
+ {column: ['=a', '=b']})
+
+ def test_002_keyOverride(self):
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='=', default='%a'), operator='and'),
+ {'default': {'query': '%a', 'key': 'ExactMatch'}},
+ check_search_text=False)
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='=', default='<a'), operator='and'),
+ {'default': {'query': '<a', 'key': 'ExactMatch'}},
+ check_search_text=False)
+
+ def test_003_DateTimeKey(self):
+ for column in ('date', 'related_date'):
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='>=', date=DateTime('2008/10/01 12:10:21')), operator='and'),
+ {column: {'query': '>2008/10/01 12:10:20', 'format': '%y/%m/%d'}})
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='>=', date=DateTime('2008/10/01 12:10:21 CEST')), operator='and'),
+ {column: {'query': '>2008/10/01 12:10:20 CEST', 'format': '%y/%m/%d'}})
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='>=', date=DateTime('2008/10/01 12:10:21 CET')), operator='and'),
+ {column: {'query': '>2008/10/01 12:10:20 CET', 'format': '%y/%m/%d'}})
+ self.catalog(ReferenceQuery(ReferenceQuery(
+ ReferenceQuery(operator='>=', date=DateTime('2008/10/01 UTC')),
+ ReferenceQuery(operator='<', date=DateTime('2008/10/02 UTC'))
+ , operator='and'), operator='and'),
+ {column: '2008/10/01 UTC'})
+ self.catalog(ReferenceQuery(ReferenceQuery(
+ ReferenceQuery(operator='>=', date=DateTime('2008/01/01 UTC')),
+ ReferenceQuery(operator='<', date=DateTime('2009/01/01 UTC'))
+ , operator='and'), operator='and'),
+ {column: '2008 UTC'})
+ self.catalog(ReferenceQuery(ReferenceQuery(
+ ReferenceQuery(operator='>=', date=DateTime('2008/01/01 UTC')),
+ ReferenceQuery(operator='<', date=DateTime('2008/02/01 UTC'))
+ , operator='and'), operator='and'),
+ {column: '2008/01 UTC'})
+ self.catalog(ReferenceQuery(ReferenceQuery(
+ ReferenceQuery(operator='>=', date=DateTime('2008/10/01 UTC')),
+ ReferenceQuery(operator='<', date=DateTime('2008/10/02 UTC'))
+ , operator='and'), operator='and'),
+ {column: {'type': 'date', 'query': '10/01/2008 UTC', 'format': '%m/%d/%Y'}})
+ self.catalog(ReferenceQuery(ReferenceQuery(
+ ReferenceQuery(operator='>=', date=DateTime('2008/10/01 UTC')),
+ ReferenceQuery(operator='<', date=DateTime('2008/10/02 UTC'))
+ , operator='and'), operator='and'),
+ {column: {'type': 'date', 'query': '01/10/2008 UTC', 'format': '%d/%m/%Y'}})
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='in', date=[DateTime('2008/01/10 UTC'), DateTime('2008/01/09 UTC')]), operator='and'),
+ {column: {'query': ['2008/01/10 UTC', '2008/01/09 UTC'], 'operator': 'in'}},
+ check_search_text=False)
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='>', date=DateTime('2008/01/10 UTC')), operator='and'),
+ {column: {'query': '2008/01/10 UTC', 'range': 'nlt'}},
+ check_search_text=False)
+
+ def test_004_KeywordKey(self):
+ for column in ('keyword', 'related_keyword'):
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='like', keyword='%a%'), operator='and'),
+ {column: 'a'})
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='like', keyword='%a'), operator='and'),
+ {column: '%a'})
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='<', keyword='a'), operator='and'),
+ {column: '<a'})
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='<', keyword='path'), operator='and'),
+ {column: {'query': 'path', 'range': 'max'}})
+ self.catalog(ReferenceQuery(ReferenceQuery(
+ ReferenceQuery(operator='like', keyword='%a%'),
+ ReferenceQuery(operator='like', keyword='%b%')
+ , operator='or'), operator='and'),
+ {column: ['a', 'b']})
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='in', keyword=['a', 'b']), operator='and'),
+ {column: ['=a', '=b']})
+ self.catalog(ReferenceQuery(ReferenceQuery(
+ ReferenceQuery(operator='like', keyword='%a%'),
+ ReferenceQuery(operator='<', keyword='b')
+ , operator='or'), operator='and'),
+ {column: ['a', '<b']})
+ self.catalog(ReferenceQuery(ReferenceQuery(
+ ReferenceQuery(operator='like', keyword='%a%'),
+ ReferenceQuery(operator='like', keyword='%b')
+ , operator='or'), operator='and'),
+ {column: ['a', '%b']})
+
+ def test_005_SearchText(self):
+ self.catalog(ReferenceQuery(ReferenceQuery(ReferenceQuery(operator='like', keyword='%=a%'), ReferenceQuery(operator='like', keyword='%=b%'), operator='or'), operator='and'),
+ {'keyword': '"=a" OR "=b"'})
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='in', keyword=['a', 'b']), operator='and'),
+ {'keyword': '="a" OR ="b"'})
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='in', keyword=['a', 'b']), operator='and'),
+ {'keyword': '=a OR =b'})
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='in', keyword=['a', 'b', 'c']), operator='and'),
+ {'keyword': '=a OR =b OR =c'})
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='like', keyword='%a%'), operator='and'),
+ {'keyword': 'keyword:a'})
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='=', default='a'), operator='and'),
+ {'keyword': 'default:a'})
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='like', keyword='%a b%'), operator='and'),
+ {'keyword': 'a b'})
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='like', keyword='%=a OR =b%'), operator='and'),
+ {'keyword': '"=a OR =b"'})
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='=', keyword='=a OR =b'), operator='and'),
+ {'keyword': '="=a OR =b"'})
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='<', keyword='=a OR =b'), operator='and'),
+ {'keyword': '<"=a OR =b"'})
+ self.catalog(ReferenceQuery(ReferenceQuery(operator='like', keyword='%"a" OR "b"%'), operator='and'),
+ {'keyword': '"\\"a\\" OR \\"b\\""'})
+
+ def test_006_testRelatedKey_with_multiple_join(self):
+ # The name of catalog parameter does not matter at all
+ # ComplexQuery(ComplexQuery(AutoQuery(RelatedQuery(SimpleQuery())), AutoQuery(RelatedQuery(SimpleQuery()))))
+ # 'AutoQuery' doesn't need any ReferenceQuery equivalent.
+ self.catalog(ReferenceQuery(ReferenceQuery(
+ ReferenceQuery(RelatedReferenceQuery(ReferenceQuery(operator='=', default='a')), operator='and'),
+ ReferenceQuery(RelatedReferenceQuery(ReferenceQuery(operator='=', default='b')), operator='and')
+ , operator='and'), operator='and'),
+ {'query': ComplexQuery(Query(related_default='a'), Query(related_default='b'))})
+
+##return catalog(title=Query(title='a', operator='not'))
+#return catalog(title={'query': 'a', 'operator': 'not'})
+#return catalog(title={'query': ['a', 'b'], 'operator': 'not'})
+#return context.portal_catalog(source_title="toto", source_description="tutu", src__=1)
+#print catalog(query=ComplexQuery(Query(title='1'), ComplexQuery(Query(portal_type='Foo') ,Query(portal_type='Bar'), operator='or'), operator='and'))
+#print catalog(title={'query': ('path', 2), 'operator': 'and'}, exception=TypeError)
+#print catalog(sort_on=[('source_title', )], check_search_text=False)
+#print catalog(query=ComplexQuery(Query(source_title='foo'), Query(source_title='bar')), sort_on=[('source_title', ), ('source_title_1', )], check_search_text=False)
+
+def test_suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(TestSQLCatalog))
+ return suite
+
diff --git a/product/ZSQLCatalog/tests/testSearchKeys.py b/product/ZSQLCatalog/tests/testSearchKeys.py
index e75ff300ad4733c596f1278135216807833cc5e5..c5048f2ccb742dc6a502fe3262e1202b108e9be9 100644
--- a/product/ZSQLCatalog/tests/testSearchKeys.py
+++ b/product/ZSQLCatalog/tests/testSearchKeys.py
@@ -28,6 +28,7 @@
import unittest
from DateTime import DateTime
+from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
from Products.ZSQLCatalog.SearchKey.DefaultKey import DefaultKey
from Products.ZSQLCatalog.SearchKey.RawKey import RawKey
from Products.ZSQLCatalog.SearchKey.KeyWordKey import KeyWordKey
@@ -37,19 +38,20 @@ from Products.ZSQLCatalog.SearchKey.FloatKey import FloatKey
from Products.ZSQLCatalog.SQLCatalog import getSearchKeyInstance
from Products.ZSQLCatalog.SearchKey.ScriptableKey import ScriptableKey, KeyMappingKey
-class TestSearchKeyLexer(unittest.TestCase):
+class TestSearchKeyLexer(ERP5TypeTestCase):
"""Test search keys
"""
run_all_test = 1
quiet = 0
-
+
def compare(self, search_key_class, search_value, expected_token_types):
""" """
key = getSearchKeyInstance(search_key_class)
- tokens = key.tokenize(search_value)
+ tokens = key.tokenize(search_value)
token_types = [x.type for x in tokens]
- self.assertEqual(token_types, list(expected_token_types))
+ self.assertSameSet(token_types, expected_token_types)
+
def test_01ProperPoolInitialization(self, quiet=quiet, run=run_all_test):
""" Check that search key pool is properly initialized """
if not run: return
@@ -82,7 +84,6 @@ size/Child/34"""
self.compare(DefaultKey,
'S\xc3\xa9bastien or !="Doe John1" and Doe',
('WORD', 'OR', 'NOT', 'WORDSET', 'AND', 'WORD',))
- self.compare(DefaultKey, '.', ('WORD',))
def test_03KeyWordKey(self, quiet=quiet, run=run_all_test):
""" Check lexer for KeyWordKey."""
@@ -101,37 +102,29 @@ size/Child/34"""
self.compare(KeyWordKey, '<=John% and >="JOHN John"',
('LESSTHANEQUAL', 'KEYWORD', 'AND',
'GREATERTHANEQUAL', 'WORDSET',))
- self.compare(KeyWordKey, '=John% and >="JOHN John"',
- ('EXPLICITEQUALLITYWORD', 'AND',
+ self.compare(KeyWordKey, '=John% and >="JOHN John"',
+ ('EXPLICITEQUALLITYWORD', 'KEYWORD', 'AND',
'GREATERTHANEQUAL', 'WORDSET',))
- self.compare(KeyWordKey, '.', ('WORD',))
def test_04DateTimeKey(self, quiet=quiet, run=run_all_test):
""" Check lexer for DateTimeKey."""
if not run: return
- self.compare(DateTimeKey, '2007.12.23', ('DATE',))
- self.compare(DateTimeKey,
+ self.compare(DateTimeKey, '2007.12.23', ('DATE',))
+ self.compare(DateTimeKey,
'=2007.12.23 22:00:00 Universal or =23/12/2007 10:10 and !=2009-12-12',
('EQUAL', 'DATE', 'OR', 'EQUAL', 'DATE', 'AND', 'NOT', 'DATE',))
- self.compare(DateTimeKey,
+ self.compare(DateTimeKey,
'>=2007.12.23 22:00:00 GMT+02 or <=23/12/2007 and >2009/12/12 and <2009-11-11',
('GREATERTHANEQUAL', 'DATE', 'OR', 'LESSTHANEQUAL', 'DATE',
'AND', 'GREATERTHAN', 'DATE', 'AND', 'LESSTHAN', 'DATE'))
-
+
def test_05FullTextKey(self, quiet=quiet, run=run_all_test):
- """ Check lexer for FullTextKey."""
+ """ Check lexer for FullTextKey."""
if not run: return
- self.compare(FullTextKey, 'John', ('WORD',))
- self.compare(FullTextKey, 'John Doe', ('WORD', 'WORD',))
- self.compare(FullTextKey, '+John -Doe',
- ('PLUS', 'WORD', 'MINUS', 'WORD',))
- self.compare(FullTextKey, 'John*', ('WORD', 'ASTERISK'))
- self.compare(FullTextKey, '+John*', ('PLUS', 'WORD', 'ASTERISK'))
- self.compare(FullTextKey, '.', ('WORD',))
- self.compare(FullTextKey, '"John Doe"', ('DOUBLEQUOTE', 'WORD', 'WORD', 'DOUBLEQUOTE'))
- self.compare(FullTextKey, '+apple +(>turnover <strudel)',
- ('PLUS', 'WORD', 'PLUS', 'LEFTPARENTHES', 'GREATERTHAN', 'WORD',
- 'LESSTHAN', 'WORD', 'RIGHTPARENTHES',))
+ self.compare(FullTextKey, 'John Doe',
+ ('WORD', 'WORD',))
+ self.compare(FullTextKey, '+John -Doe',
+ ('PLUS', 'WORD', 'MINUS', 'WORD',))
def test_06ScriptableKey(self, quiet=quiet, run=run_all_test):
""" Check lexer for ScriptableKey."""
@@ -143,7 +136,7 @@ size/Child/34"""
'John Doe OR creation_date>=2005/12/12',
('WORD', 'WORD', 'OR', 'KEYMAPPING',))
-class TestSearchKeyQuery(unittest.TestCase):
+class TestSearchKeyQuery(ERP5TypeTestCase):
"""Test search keys query generation
"""
run_all_test = 1
@@ -206,12 +199,6 @@ class TestSearchKeyQuery(unittest.TestCase):
'%John and !=Doe%',
"((((title = '%John') AND (title != 'Doe%'))))",
[])
- # special chars
- self.compare(DefaultKey,
- 'title',
- '.',
- "((((title = '.'))))",
- [])
def test_02KeyWordKey(self, quiet=quiet, run=run_all_test):
""" Check DefaultKey query generation"""
@@ -241,12 +228,6 @@ class TestSearchKeyQuery(unittest.TestCase):
'%John Doe% or =Doe John',
"((((title LIKE '%John Doe%'))) OR (((title = 'Doe John'))))",
[])
- # special chars
- self.compare(KeyWordKey,
- 'title',
- '.',
- "((((title LIKE '%.%'))))",
- [])
def test_03DateTimeKey(self, quiet=quiet, run=run_all_test):
""" Check DefaultKey query generation"""
@@ -368,18 +349,7 @@ class TestSearchKeyQuery(unittest.TestCase):
'1ab521ty',
"delivery.stock = '1ab521ty'",
[])
- # special chars
- self.compare(RawKey,
- 'delivery.stock',
- '.',
- "delivery.stock = '.'",
- [])
- #None values
- self.compare(RawKey,
- 'title',
- None,
- "title is NULL",
- [])
+
def test_06FullTextKey(self, quiet=quiet, run=run_all_test):
""" Check FullTextKey query generation"""
if not run: return
@@ -389,28 +359,6 @@ class TestSearchKeyQuery(unittest.TestCase):
"MATCH full_text.SearchableText AGAINST ('john' )",
["MATCH full_text.SearchableText AGAINST ('john' ) AS full_text_SearchableText_relevance",
"MATCH full_text.SearchableText AGAINST ('john' ) AS SearchableText_relevance"])
-
- # special chars
- self.compare(FullTextKey,
- 'full_text.SearchableText',
- '.',
- "MATCH full_text.SearchableText AGAINST ('.' )",
- ["MATCH full_text.SearchableText AGAINST ('.' ) AS full_text_SearchableText_relevance",
- "MATCH full_text.SearchableText AGAINST ('.' ) AS SearchableText_relevance"])
- #Boolean Mode
- self.compare(FullTextKey,
- 'full_text.SearchableText',
- 'john stuart mill',
- "MATCH full_text.SearchableText AGAINST ('+john +stuart +mill' IN BOOLEAN MODE)",
- ["MATCH full_text.SearchableText AGAINST ('+john +stuart +mill' IN BOOLEAN MODE) AS full_text_SearchableText_relevance",
- "MATCH full_text.SearchableText AGAINST ('+john +stuart +mill' IN BOOLEAN MODE) AS SearchableText_relevance"])
-
- self.compare(FullTextKey,
- 'full_text.SearchableText',
- 'John*',
- "MATCH full_text.SearchableText AGAINST ('John*' IN BOOLEAN MODE)",
- ["MATCH full_text.SearchableText AGAINST ('John*' IN BOOLEAN MODE) AS full_text_SearchableText_relevance",
- "MATCH full_text.SearchableText AGAINST ('John*' IN BOOLEAN MODE) AS SearchableText_relevance"])
def test_07ScriptableKey(self, quiet=quiet, run=run_all_test):
""" Check ScriptableKey query generation"""
diff --git a/product/ZSQLCatalog/tests/testZSQLCatalog.py b/product/ZSQLCatalog/tests/testZSQLCatalog.py
index d534c8d6cc3082fd70c1ba10d487221e11605bdb..8619e4d20a9d1a714a607a9edca152f5e9111f6a 100644
--- a/product/ZSQLCatalog/tests/testZSQLCatalog.py
+++ b/product/ZSQLCatalog/tests/testZSQLCatalog.py
@@ -95,369 +95,9 @@ class TestSQLCatalog(unittest.TestCase):
self.assertFalse(
self._catalog.isPortalTypeSelected('not_exists', 'Selected'))
-
-class TestQuery(unittest.TestCase):
- """Test SQL bits generated from Queries
- """
- def testSimpleQuery(self):
- q = Query(title='Foo')
- self.assertEquals(
- dict(where_expression="((((title = 'Foo'))))",
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=[],
- datetime_search_keys = [],
- full_text_search_keys=[]))
-
- def testQueryMultipleKeys(self):
- # using multiple keys is invalid and raises
- # KeyError: 'Query must have only one key'
- self.assertRaises(KeyError, Query, title='Foo', reference='bar')
-
- def testNoneQuery(self):
- q = Query(title=None)
- self.assertEquals(
- dict(where_expression="title is NULL",
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=[],
- datetime_search_keys = [],
- full_text_search_keys=[]))
-
- def testEmptyQueryNotIgnoreEmptyString(self):
- q = Query(title='')
- # if you want to search with an empty string, pass ignore_empty_string=0 to
- # asSQLExpression. XXX not to __init__ ?
- self.assertEquals(
- dict(where_expression="title = ''",
- select_expression_list=[]),
- q.asSQLExpression(ignore_empty_string=0,
- keyword_search_keys=[],
- datetime_search_keys = [],
- full_text_search_keys=[]))
-
- def testEmptyQuery(self):
- q = Query(title='')
- # query are true by default
- self.assertEquals(
- dict(where_expression="1",
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=[],
- datetime_search_keys = [],
- full_text_search_keys=[]))
-
- def testMultiValuedQuery(self):
- q = Query(title=['Foo', 'Bar'])
- self.assertEquals(
- dict(where_expression="(((((title = 'Foo')))) OR ((((title = 'Bar')))))",
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=[],
- datetime_search_keys = [],
- full_text_search_keys=[]))
-
- def testINQuery(self):
- q = Query(title=['Foo', 'Bar'], operator='IN')
- self.assertEquals(
- dict(where_expression="title IN ('Foo', 'Bar')",
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=[],
- datetime_search_keys = [],
- full_text_search_keys=[]))
-
- def testEmptyINQuery(self):
- q = Query(title=[], operator='IN')
- self.assertEquals(
- dict(where_expression="0",
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=[],
- datetime_search_keys = [],
- full_text_search_keys=[]))
-
- def testMinQuery(self):
- q = Query(title='Foo', range='min')
- self.assertEquals(
- dict(where_expression="title >= 'Foo'",
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=[],
- datetime_search_keys = [],
- full_text_search_keys=[]))
-
- def testMaxQuery(self):
- q = Query(title='Foo', range='max')
- self.assertEquals(
- dict(where_expression="title < 'Foo'",
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=[],
- datetime_search_keys = [],
- full_text_search_keys=[]))
-
- # format
- def testDateFormat(self):
- date = DateTime(2001, 02, 03)
- q = Query(date=date, format='%Y/%m/%d', type='date')
- self.assertEquals(
- dict(where_expression=
- "((((date >= '%s' AND date < '%s'))))" \
- %(date.toZone('UTC').ISO(), (date + 1).toZone('UTC').ISO()),
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=[],
- datetime_search_keys = [],
- full_text_search_keys=[]))
-
- # full text
- def testSimpleQueryFullText(self):
- q = Query(title='Foo')
- self.assertEquals(dict(where_expression="MATCH title AGAINST ('Foo' )",
- select_expression_list=
- ["MATCH title AGAINST ('Foo' ) AS title_relevance"]),
- q.asSQLExpression(keyword_search_keys=[],
- datetime_search_keys = [],
- full_text_search_keys=['title']))
-
- def testSimpleQueryFullTextSearchMode(self):
- q = Query(title='Foo',
- search_mode='in_boolean_mode')
- self.assertEquals(dict(
- where_expression="MATCH title AGAINST ('Foo' IN BOOLEAN MODE)",
- select_expression_list=
- ["MATCH title AGAINST ('Foo' IN BOOLEAN MODE) AS title_relevance"]),
- q.asSQLExpression(keyword_search_keys=[],
- datetime_search_keys = [],
- full_text_search_keys=['title']))
-
- def testSimpleQueryFullAutomaticTextSearchMode(self):
- q = Query(title='Foo*',)
- self.assertEquals(dict(
- where_expression="MATCH title AGAINST ('Foo*' IN BOOLEAN MODE)",
- select_expression_list=
- ["MATCH title AGAINST ('Foo*' IN BOOLEAN MODE) AS title_relevance"]),
- q.asSQLExpression(full_text_search_keys=['title']))
-
- def testSimpleQueryFullTextStat__(self):
- # stat__ is an internal implementation artifact to prevent adding
- # select_expression for countFolder
- q = Query(title='Foo')
- self.assertEquals(dict(
- where_expression="MATCH title AGAINST ('Foo' )",
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=[],
- datetime_search_keys = [],
- full_text_search_keys=['title'],
- stat__=1))
-
- def testSimpleQueryKeywordSearchKey(self):
- q = Query(title='Foo')
- self.assertEquals(dict(where_expression="((((title LIKE '%Foo%'))))",
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=['title'],
- datetime_search_keys = [],
- full_text_search_keys=[]))
-
- def testQueryKeywordSearchKeyWithPercent(self):
- q = Query(title='Fo%oo')
- self.assertEquals(dict(where_expression="((((title LIKE 'Fo%oo'))))",
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=['title'],))
-
- def testQueryKeywordSearchKeyWithPercentAndOnlyOneLetter(self):
- q = Query(title='F%o')
- self.assertEquals(dict(where_expression="((((title LIKE 'F%o'))))",
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=['title']))
-
- def testQueryKeywordSearchKeyWithPercentOnly(self):
- q = Query(title='%')
- self.assertEquals(dict(where_expression="((((title LIKE '%'))))",
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=['title'],))
-
- def testQueryKeywordSearchKeyWithMinus(self):
- q = Query(title='F-o')
- self.assertEquals(dict(where_expression="((((title LIKE '%F-o%'))))",
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=['title'],
- datetime_search_keys = [],
- full_text_search_keys=[]))
-
- def testQueryKeywordSearchKeyWithSpace(self):
- q = Query(title='F o')
- self.assertEquals(dict(where_expression="((((title LIKE '%F o%'))))",
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=['title'],
- datetime_search_keys = [],
- full_text_search_keys=[]))
-
- def testQueryKeywordSearchKeyWithPercentAtTheEnd(self):
- q = Query(title='F%')
- self.assertEquals(dict(where_expression="((((title LIKE 'F%'))))",
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=['title'],))
- q = Query(title='Fo%')
- self.assertEquals(dict(where_expression="((((title LIKE 'Fo%'))))",
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=['title'],))
-
- def testQueryKeywordSearchKeyWithPercentAtTheBeginning(self):
- q = Query(title='%o')
- self.assertEquals(dict(where_expression="((((title LIKE '%o'))))",
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=['title'],))
- q = Query(title='%oo')
- self.assertEquals(dict(where_expression="((((title LIKE '%oo'))))",
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=['title'],))
-
- def testNegatedQuery(self):
- q1 = Query(title='Foo')
- q = NegatedQuery(q1)
- self.assertEquals(
- dict(where_expression="(NOT (((((title = 'Foo'))))))",
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=[],
- datetime_search_keys = [],
- full_text_search_keys=[]))
-
- # complex queries
- def testSimpleComplexQuery(self):
- q1 = Query(title='Foo')
- q2 = Query(reference='Bar')
- q = ComplexQuery(q1, q2)
- self.assertEquals(
- dict(where_expression="((((((title = 'Foo'))))) AND (((((reference = 'Bar'))))))",
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=[],
- datetime_search_keys = [],
- full_text_search_keys=[]))
-
- def testNegatedComplexQuery(self):
- q1 = Query(title='Foo')
- q2 = Query(reference='Bar')
- q3 = ComplexQuery(q1, q2)
- q = NegatedQuery(q3)
- self.assertEquals(
- # maybe too many parents here
- dict(where_expression="(NOT (((((((title = 'Foo'))))) AND (((((reference = 'Bar'))))))))",
- select_expression_list=[]),
- q.asSQLExpression(keyword_search_keys=[],
- datetime_search_keys = [],
- full_text_search_keys=[]))
-
-
- # forced keys
- def testSimpleQueryForcedKeywordSearchKey(self):
- q = Query(title='Foo', key='Keyword')
- self.assertEquals("((((title LIKE '%Foo%'))))",
- q.asSQLExpression(keyword_search_keys=[],
- datetime_search_keys = [],
- full_text_search_keys=[])['where_expression'])
-
- def testSimpleQueryForcedFullText(self):
- q = Query(title='Foo', key='FullText')
- self.assertEquals("MATCH title AGAINST ('Foo' )",
- q.asSQLExpression(keyword_search_keys=[],
- datetime_search_keys = [],
- full_text_search_keys=[])['where_expression'])
-
- def testSimpleQueryForcedExactMatch(self):
- q = Query(title='Foo', key='ExactMatch')
- self.assertEquals("title = 'Foo'",
- q.asSQLExpression(keyword_search_keys=['title'],
- datetime_search_keys = [],
- full_text_search_keys=[])['where_expression'])
-
- def testSimpleQueryForcedExactMatchOR(self):
- q = Query(title='Foo% OR %?ar', key='ExactMatch')
- self.assertEquals("title = 'Foo% OR %?ar'",
- q.asSQLExpression(keyword_search_keys=['title'],
- datetime_search_keys = [],
- full_text_search_keys=[])['where_expression'])
-
- def testQuotedStringDefaultKey(self):
- q = Query(title='Foo d\'Ba')
- self.assertEquals(
- dict(where_expression="((((title = 'Foo d''Ba'))))",
- select_expression_list=[]),
- q.asSQLExpression())
-
- def testQuotedStringKeywordKey(self):
- q = Query(title='Foo d\'Ba', key='Keyword')
- self.assertEquals(
- dict(where_expression="((((title LIKE '%Foo d''Ba%'))))",
- select_expression_list=[]),
- q.asSQLExpression())
-
- def testQuotedStringExactMatch(self):
- q = Query(title='Foo d\'Ba', key='ExactMatch')
- self.assertEquals(
- dict(where_expression="title = 'Foo d''Ba'",
- select_expression_list=[]),
- q.asSQLExpression())
-
- def testQuotedStringFullTextKey(self):
- q = Query(title='Foo d\'Ba', type='fulltext')
- self.assertEquals(
- dict(where_expression="MATCH title AGAINST ('+Foo +d''Ba' IN BOOLEAN MODE)",
- select_expression_list=["MATCH title AGAINST ('+Foo +d''Ba' IN BOOLEAN MODE)"
- " AS title_relevance"]),
- q.asSQLExpression())
-
- def testQuotedStringListKeywordKey(self):
- q = Query(title=('Foo d\'Ba',), key='Keyword')
- self.assertEquals(
- dict(where_expression="((((title LIKE '%Foo d''Ba%'))))",
- select_expression_list=[]),
- q.asSQLExpression())
-
- def testQuotedStringListExactMatch(self):
- q = Query(title=('Foo d\'Ba',), key='ExactMatch')
- self.assertEquals(
- dict(where_expression="title = 'Foo d''Ba'",
- select_expression_list=[]),
- q.asSQLExpression())
-
-
- def testQuotedStringDateKey(self):
- q = Query(title='Foo d\'Ba', type='date')
- self.assertEquals(
- # I don't know exactly what we should expect here.
- dict(where_expression="1",
- select_expression_list=[]),
- q.asSQLExpression())
-
- def testQuotedStringFloatKey(self):
- q = Query(title='Foo d\'Ba', type='float')
- self.assertEquals(
- # I don't know exactly what we should expect here.
- # At least it's safe.
- dict(where_expression="1",
- select_expression_list=[]),
- q.asSQLExpression())
-
- def testQuotedStringIntKey(self):
- q = Query(title='Foo d\'Ba', type='int')
- self.assertEquals(
- dict(where_expression="((((title = 'Foo d''Ba'))))",
- select_expression_list=[]),
- q.asSQLExpression())
-
- def testListValuesInQuery(self):
- q = Query(title=('Foo', 'Bar'))
- self.assertEquals(
- dict(where_expression="(((((title = 'Foo')))) OR ((((title = 'Bar')))))",
- select_expression_list=[]),
- q.asSQLExpression())
-
- def testListValuesInQueryWithKey(self):
- q = Query(title=('Foo', 'Bar'), key='Keyword')
- self.assertEquals(
- dict(where_expression=
- "(((((title LIKE '%Foo%')))) OR ((((title LIKE '%Bar%')))))",
- select_expression_list=[]),
- q.asSQLExpression())
-
-
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestSQLCatalog))
suite.addTest(unittest.makeSuite(TestZSQLCatalog))
- suite.addTest(unittest.makeSuite(TestQuery))
return suite