Commit 05efe1ba authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki

Revert "Add auto_extend_select_list argument in buildSQLQuery() and use alias...

Revert "Add auto_extend_select_list argument in buildSQLQuery() and use alias in group_by_expression and order_by_expression."
Revert "Support group_by_list and order_by_list in WebSection_getDocumentValueList."
Revert "Support order_by_list in Resource_zGetMovementHistoryList."

No need to extend SQLCatalog parameter to support group_by_list and order_by_list in sub-queries.
parent d1a6246e
......@@ -140,11 +140,6 @@ try:\n
kw[\'effective_date\'] = effective_date\n
\n
if not all_versions:\n
group_by_list = list(kw.get(\'group_by_list\', []))\n
if all_languages:\n
kw[\'group_by_list\'] = group_by_list + [\'reference\', \'language\']\n
else:\n
kw[\'group_by_list\'] = group_by_list + [\'reference\',]\n
return current_section.WebSection_zGetDocumentValueList(language=language,\n
all_languages=all_languages,\n
src__=src__,\n
......
......@@ -64,7 +64,9 @@ kw</string> </value>
<key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[
<dtml-let query="buildSQLQuery(query=portal_catalog.getSecurityQuery(**kw), auto_extend_select_list=True, **kw)"\n
<dtml-let query="buildSQLQuery(query=portal_catalog.getSecurityQuery(**kw), **kw)"\n
is_translation_order_by="query[\'order_by_expression\'] is not None and\n
\'translated_message\' in query[\'order_by_expression\']"\n
selection_domain="kw.get(\'selection_domain\', None)"\n
selection_report="kw.get(\'selection_report\', None)">\n
\n
......@@ -74,6 +76,12 @@ kw</string> </value>
this method should be refactored to use catalog only.\n
\n
The subquery is named catalog to prevent use another LEFT JOIN.\n
\n
XXX This query heavily depends on MySQL implementation, \n
as it seems that the "GROUP BY reference" returns the first line of the subselect \n
(which is the expected one due to "ORDER BY priority DESC").\n
But, without any explicit reference of this behaviour in MySQL\n
documentation, wrong results may be returned with another MySQL version.\n
</dtml-comment>\n
\n
SELECT\n
......@@ -94,7 +102,9 @@ kw</string> </value>
WHEN \'en\' THEN \'2\'\n
ELSE \'1\' END,\n
my_versioning.version) AS priority\n
<dtml-if "query[\'select_expression\']">,<dtml-var "query[\'select_expression\']"></dtml-if>\n
<dtml-if is_translation_order_by>\n
, related_translation_intermediate.translated_message\n
</dtml-if>\n
FROM\n
<dtml-in prefix="table" expr="query[\'from_table_list\']">\n
<dtml-var table_item> AS <dtml-var table_key>,\n
......@@ -106,6 +116,9 @@ kw</string> </value>
<dtml-var "portal_selections.buildSQLJoinExpressionFromDomainSelection(selection_report)">,\n
</dtml-if>\n
versioning AS my_versioning\n
<dtml-if is_translation_order_by>\n
, translation AS related_translation_intermediate\n
</dtml-if>\n
WHERE\n
my_versioning.uid = catalog.uid\n
<dtml-if "query[\'where_expression\']">\n
......@@ -121,17 +134,34 @@ kw</string> </value>
<dtml-else>\n
AND my_versioning.language IN (<dtml-sqlvar language type="string">, \'\')\n
</dtml-if>\n
\n
<dtml-if is_translation_order_by>\n
AND catalog.validation_state = related_translation_intermediate.original_message\n
</dtml-if>\n
\n
ORDER BY\n
priority DESC\n
\n
) AS catalog\n
<dtml-if "\'versioning\' in query[\'order_by_expression\'] or all_languages">\n
, versioning\n
WHERE catalog.uid = versioning.uid\n
</dtml-if>\n
\n
<dtml-if "query[\'group_by_expression\']">\n
GROUP BY <dtml-var "query[\'group_by_expression\']">\n
<dtml-if all_languages>\n
GROUP BY reference, language\n
<dtml-else>\n
GROUP BY reference\n
</dtml-if>\n
\n
ORDER BY <dtml-var "query[\'order_by_expression\'] or \'priority DESC\'">\n
<dtml-if "query[\'order_by_expression\']">\n
ORDER BY\n
<dtml-if is_translation_order_by>\n
translated_message\n
<dtml-else>\n
<dtml-var "query[\'order_by_expression\']">\n
</dtml-if>\n
</dtml-if>\n
\n
<dtml-if "query[\'limit_expression\']">\n
LIMIT <dtml-var "query[\'limit_expression\']">\n
......
......@@ -2015,7 +2015,6 @@ class SimulationTool(BaseTool):
"""
kw['movement_list_mode'] = 1
kw.update(self._getDefaultGroupByParameters(**kw))
kw['auto_extend_select_list'] = True
sql_kw = self._generateSQLKeywordDict(**kw)
return self.Resource_zGetMovementHistoryList(
......
......@@ -419,7 +419,6 @@
<key> <string>arguments_src</string> </key>
<value> <string>from_expression\r\n
from_table_list:list\r\n
select_expression\r\n
where_expression\r\n
order_by_expression\r\n
group_by_expression\r\n
......@@ -513,9 +512,6 @@ SELECT\n
stock.payment_request_uid AS payment_request_uid,\n
stock.node_uid AS node_uid,\n
stock.section_uid AS section_uid\n
<dtml-if select_expression>\n
,<dtml-var select_expression>\n
</dtml-if>\n
FROM\n
<dtml-if from_expression>\n
<dtml-var from_expression>\n
......@@ -627,10 +623,6 @@ LIMIT\n
</dtml-if>\n
\n
) AS q1\n
<dtml-if order_by_expression>\n
ORDER BY\n
<dtml-var order_by_expression>\n
</dtml-if>\n
]]></string> </value>
......
......@@ -35,6 +35,7 @@ from DateTime import DateTime
from Products.ERP5.tests.testAccounting import AccountingTestCase
from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5ReportTestCase
from Products.ERP5Type.tests.utils import todo_erp5
from Products.ERP5Type.UnrestrictedMethod import UnrestrictedMethod
class TestAccountingReports(AccountingTestCase, ERP5ReportTestCase):
......@@ -798,9 +799,12 @@ class TestAccountingReports(AccountingTestCase, ERP5ReportTestCase):
dict(source_value=account_module.goods_sales,
source_credit=300.0)))
@todo_erp5
def test_Resource_zGetMovementHistoryList(self):
# Check if Resource_zGetMovementHistoryList works fine with derived_merge optimizer.
# see https://bugs.launchpad.net/maria/+bug/985828
# TODO: Fix Resource_zGetMovementHistoryList so that we don't need to workaround
# new behaviour of MariaDB.
# Indeed, https://bugs.launchpad.net/maria/+bug/985828 has been marked
# as WONTFIX.
q = self.portal.erp5_sql_connection.manage_test
q("SET optimizer_switch = 'derived_merge=on'")
try:
......
......@@ -80,10 +80,10 @@ class TestI18NSearch(ERP5TypeTestCase):
self.assertEqual(result[0].getPath(), self.person1.getPath())
# check sort on fulltext column
self.assertTrue('ORDER BY\n `full_text`.`SearchableText` ASC' in self.portal.portal_catalog(SearchableText='Faure', sort_on=(('SearchableText', 'ascending'),), src__=1))
self.assertFalse('ORDER BY\n MATCH' in self.portal.portal_catalog(SearchableText='Faure', sort_on=(('SearchableText', 'ascending'),), src__=1))
# check sort on fulltext search score
self.assertTrue('ORDER BY\n full_text_SearchableText__score__ ASC' in self.portal.portal_catalog(SearchableText='Faure', sort_on=(('SearchableText__score__', 'ascending'),), src__=1))
self.assertTrue('ORDER BY\n MATCH' in self.portal.portal_catalog(SearchableText='Faure', sort_on=(('SearchableText__score__', 'ascending'),), src__=1))
def test_catalog_full_text_title(self):
# check if 'é' == 'e' collation works
......@@ -108,23 +108,18 @@ class TestI18NSearch(ERP5TypeTestCase):
self.assertEqual(result[0].getPath(), self.person3.getPath())
# check sort on fulltext column
self.assertFalse('ORDER BY\n catalog_full_text_title__score__ ASC' in self.portal.portal_catalog(**{
self.assertFalse('ORDER BY\n MATCH' in self.portal.portal_catalog(**{
'catalog_full_text.title':'Faure',
'sort_on':(('catalog_full_text.title', 'ascending'),),
'src__':1
}))
# check sort on fulltext search score
self.assertTrue('ORDER BY\n catalog_full_text_title__score__' in self.portal.portal_catalog(**{
self.assertFalse('ORDER BY\n MATCH' in self.portal.portal_catalog(**{
'catalog_full_text.title':'Faure',
'sort_on':(('catalog_full_text.title__score__', 'ascending'),),
'src__':1
}))
self.assertTrue('ORDER BY\n catalog_full_text_title__score__' in self.portal.portal_catalog(**{
'catalog_full_text.title':'Faure',
'sort_on':(('title__score__', 'ascending'),),
'src__':1
}))
@expectedFailure
def test_full_text_title(self):
......@@ -153,10 +148,10 @@ class TestI18NSearch(ERP5TypeTestCase):
self.assertTrue('MATCH' in self.portal.portal_catalog(destination_title='Faure', src__=1))
# check sort on fulltext column
self.assertTrue('ORDER BY\n `catalog`.`title` ASC' in self.portal.portal_catalog(title='Faure', sort_on=(('title', 'ascending'),), src__=1))
self.assertFalse('ORDER BY\n MATCH' in self.portal.portal_catalog(title='Faure', sort_on=(('title', 'ascending'),), src__=1))
# check sort on fulltext search score
self.assertTrue('ORDER BY\n catalog_full_text_title__score__' in self.portal.portal_catalog(title='Faure', sort_on=(('title__score__', 'ascending'),), src__=1))
self.assertTrue('ORDER BY\n MATCH' in self.portal.portal_catalog(title='Faure', sort_on=(('title__score__', 'ascending'),), src__=1))
def test_suite():
suite = unittest.TestSuite()
......
......@@ -453,9 +453,6 @@ class ColumnMap(object):
def asSQLColumn(self, raw_column, group=DEFAULT_GROUP_ID):
if self.catalog_table_name is None or raw_column in self.column_ignore_set or \
'.' in raw_column or '*' in raw_column:
if raw_column.endswith('__score__'):
result = raw_column.replace('.', '_')
else:
result = raw_column
else:
if raw_column.endswith('__score__'):
......@@ -467,10 +464,7 @@ class ColumnMap(object):
if group is DEFAULT_GROUP_ID:
group, column = self.related_key_dict.get(column, (group, raw_column))
alias = self.table_alias_dict[(group, self.column_map[(group, column)])]
if column_suffix:
result = '%s_%s%s' % (alias, column, column_suffix)
else:
result = '`%s`.`%s`' % (alias, column)
result = '`%s`.`%s%s`' % (alias, column, column_suffix)
if function is not None:
result = '%s(%s)' % (function, result)
return result
......
......@@ -116,15 +116,16 @@ class MatchComparisonOperator(MonovaluedComparisonOperator):
}
select_dict = {}
if not only_group_columns:
select_dict['%s__score__' % column.replace('`', '').replace('.', '_')] = match_string
select_dict['%s__score__' % column.replace('`', '').rsplit('.', 1)[-1]] = match_string
# Support sort on the relevance by using (column)__score__ key.
order_by_dict = {
'%s__score__' % column.replace('`', '').replace('.', '_'): match_string
'`%s__score__`' % '`.`'.join([x.strip('`') for x in column.split('.')]): match_string,
}
return SQLExpression(
self,
select_dict=select_dict,
order_by_dict=order_by_dict,
where_expression=match_string,
order_by_dict=order_by_dict,
can_merge_select_dict=True,
)
......
......@@ -61,7 +61,6 @@ class EntireQuery(object):
left_join_list=(),
limit=None,
catalog_table_name=None,
auto_extend_select_list=False,
extra_column_list=(),
from_expression=None,
order_by_override_list=None,
......@@ -77,7 +76,6 @@ class EntireQuery(object):
self.extra_column_list = list(extra_column_list)
self.from_expression = from_expression
self.implicit_join = implicit_join
self.auto_extend_select_list = auto_extend_select_list
def asSearchTextExpression(self, sql_catalog):
return self.query.asSearchTextExpression(sql_catalog)
......@@ -213,7 +211,6 @@ class EntireQuery(object):
order_by_list=self.order_by_list,
group_by_list=self.group_by_list,
select_dict=self.final_select_dict,
auto_extend_select_list=self.auto_extend_select_list,
limit=self.limit,
where_expression_operator='and',
sql_expression_list=self.sql_expression_list)
......
......@@ -2350,8 +2350,7 @@ class Catalog(Folder,
return order_by_list
def buildEntireQuery(self, kw, query_table='catalog', ignore_empty_string=1,
limit=None, auto_extend_select_list=False,
extra_column_list=()):
limit=None, extra_column_list=()):
group_by_list = kw.pop('group_by_list', kw.pop('group_by', kw.pop('group_by_expression', ())))
if isinstance(group_by_list, basestring):
group_by_list = [x.strip() for x in group_by_list.split(',')]
......@@ -2418,21 +2417,18 @@ class Catalog(Folder,
implicit_join=implicit_join,
limit=limit,
catalog_table_name=query_table,
auto_extend_select_list=auto_extend_select_list,
extra_column_list=extra_column_list,
from_expression=from_expression)
def buildSQLQuery(self, query_table='catalog', REQUEST=None,
ignore_empty_string=1, only_group_columns=False,
limit=None, auto_extend_select_list=False,
extra_column_list=(),
limit=None, extra_column_list=(),
**kw):
return self.buildEntireQuery(
kw,
query_table=query_table,
ignore_empty_string=ignore_empty_string,
limit=limit,
auto_extend_select_list=auto_extend_select_list,
extra_column_list=extra_column_list,
).asSQLExpression(
self,
......
......@@ -94,7 +94,6 @@ class SQLExpression(object):
where_expression_operator=None,
sql_expression_list=(),
select_dict=None,
auto_extend_select_list=False,
limit=None,
from_expression=None,
can_merge_select_dict=False):
......@@ -121,7 +120,6 @@ class SQLExpression(object):
sql_expression_list = [x for x in sql_expression_list if x is not None]
self.sql_expression_list = list(sql_expression_list)
self.select_dict = defaultDict(select_dict)
self.auto_extend_select_list = auto_extend_select_list
if limit is None:
self.limit = ()
elif isinstance(limit, (list, tuple)):
......@@ -135,17 +133,6 @@ class SQLExpression(object):
warnings.warn("Providing a 'from_expression' is deprecated.",
DeprecationWarning)
self.from_expression = from_expression
self._select_dict = self._getSelectDict()[0]
if self.auto_extend_select_list:
select_column_set = {y for x, y in self._select_dict.iteritems()}
extend_column_set = set(self.group_by_list).union(
{x[0] for x in self.order_by_list})
for i in extend_column_set.difference(select_column_set):
# '__score__' suffix alias is already added in select_dict by
# MatchComparisonOperator.
if '__score__' not in i:
self._select_dict['%s__ext__' % i.replace('`', '').replace('.', '_')] = i
self._reversed_select_dict = {y: x for x, y in self._select_dict.iteritems()}
def getTableAliasDict(self):
"""
......@@ -249,8 +236,9 @@ class SQLExpression(object):
append = result.append
order_by_dict = self._getOrderByDict()
for (column, direction, cast) in self.getOrderByList():
if column.endswith('__score__') and column not in order_by_dict:
continue
expression = conflictSafeGet(order_by_dict, column, str(column))
expression = self._reversed_select_dict.get(expression, expression)
if cast not in (None, ''):
expression = 'CAST(%s AS %s)' % (expression, cast)
if direction is not None:
......@@ -316,7 +304,7 @@ class SQLExpression(object):
If there are nested SQLExpression, it merges (union of sets) them with
local value.
"""
result = {self._reversed_select_dict.get(x, x) for x in self.group_by_list}
result = set(self.group_by_list)
for sql_expression in self.sql_expression_list:
result.update(sql_expression.getGroupByset())
return result
......@@ -375,7 +363,7 @@ class SQLExpression(object):
checks that they don't alias different columns with the same name. If
they do, it raises a ValueError.
"""
return self._select_dict
return self._getSelectDict()[0]
def getSelectExpression(self):
"""
......
......@@ -45,7 +45,6 @@ class IEntireQuery(Interface):
def __init__(query, order_by_list=None, group_by_list=None,
select_dict=None, limit=None, catalog_table_name=None,
auto_extend_select_list=False,
extra_column_list=None, from_expression=None,
order_by_override_list=None):
"""
......@@ -68,11 +67,6 @@ class IEntireQuery(Interface):
See SQLExpression.
catalog_table_name (string)
Name of the table to use as a catalog.
auto_extend_select_list (boolean)
If True, select_list is automatically extended to have columns
used in group_by_list and order_by_list. It is useful when use
select_expression in inner query and use group_by_expression or
order_by_expression in outer query.
Deprecated parameters.
extra_column_list (list of string)
......
......@@ -64,8 +64,7 @@ class ISearchKeyCatalog(Interface):
"""
def buildEntireQuery(kw, query_table='catalog', ignore_empty_string=1,
limit=None, auto_extend_select_list=False,
extra_column_list=None):
limit=None, extra_column_list=None):
"""
Construct and return an instance of EntireQuery class from given
parameters by calling buildQuery.
......@@ -96,11 +95,6 @@ class ISearchKeyCatalog(Interface):
- type cast (see SQL documentation of 'CAST')
Sort will happen on given parameter name (its column if it's a column
name, corresponding virtual column otherwise - as for related keys).
auto_extend_select_list (boolean)
If True, select_list is automatically extended to have columns
used in group_by_list and order_by_list. It is useful when use
select_expression in inner query and use group_by_expression or
order_by_expression in outer query.
Extra parameters are passed through to buildQuery.
Backward compatibility parameters:
......@@ -146,8 +140,7 @@ class ISearchKeyCatalog(Interface):
def buildSQLQuery(query_table='catalog', REQUEST=None,
ignore_empty_string=1, only_group_columns=False,
limit=None, auto_extend_select_list=False,
extra_column_list=(),
limit=None, extra_column_list=None,
**kw):
"""
Return an SQLExpression-generated dictionary (see
......
......@@ -61,7 +61,6 @@ class ISQLExpression(Interface):
where_expression_operator=None,
sql_expression_list=None,
select_dict=None,
auto_extend_select_list=False,
limit=None,
from_expression=None):
"""
......@@ -101,11 +100,6 @@ class ISQLExpression(Interface):
Key is column alias.
Value is column name, or Null. If it is Null, the alias will also be
used as column name.
auto_extend_select_list (boolean)
If True, select_list is automatically extended to have columns
used in group_by_list and order_by_list. It is useful when use
select_expression in inner query and use group_by_expression or
order_by_expression in outer query.
limit (1-tuple, 2-tuple, other)
First item is the number of lines expected, second one if given is the
offset of limited result list within the unlimited result list.
......
......@@ -728,27 +728,27 @@ class TestSQLCatalog(ERP5TypeTestCase):
order_by_expression = sql_expression.getOrderByExpression()
self.assertNotEqual(order_by_expression, '')
# ... and not sort by relevance
self.assertEqual('`foo`.`fulltext`', order_by_expression)
self.assertFalse('MATCH' in order_by_expression, order_by_expression)
# order_by_list on fulltext column + '__score__, resulting "ORDER BY" must be non-empty.
sql_expression = self.asSQLExpression({'fulltext': 'foo',
'order_by_list': [('fulltext__score__', ), ]})
order_by_expression = sql_expression.getOrderByExpression()
self.assertNotEqual(order_by_expression, '')
# ... and must sort by relevance
self.assertEqual('foo_fulltext__score__', order_by_expression)
self.assertTrue('MATCH' in order_by_expression, order_by_expression)
# ordering on fulltext column with sort order specified must preserve
# sorting by relevance.
for direction in ('ASC', 'DESC'):
sql_expression = self.asSQLExpression({'fulltext': 'foo',
'order_by_list': [('fulltext__score__', direction), ]})
order_by_expression = sql_expression.getOrderByExpression()
self.assertEqual('foo_fulltext__score__ %s' % direction, order_by_expression)
self.assertTrue('MATCH' in order_by_expression, (order_by_expression, direction))
# Providing a None cast should work too
for direction in ('ASC', 'DESC'):
sql_expression = self.asSQLExpression({'fulltext': 'foo',
'order_by_list': [('fulltext__score__', direction, None), ]})
order_by_expression = sql_expression.getOrderByExpression()
self.assertEqual('foo_fulltext__score__ %s' % direction, order_by_expression)
self.assertTrue('MATCH' in order_by_expression, (order_by_expression, direction))
def test_logicalOperators(self):
self.catalog(ReferenceQuery(ReferenceQuery(operator='=', default='AN ORB'),
......@@ -759,42 +759,6 @@ class TestSQLCatalog(ERP5TypeTestCase):
operator='and'),
{'default': 'AN OR ORB'})
def test_auto_extend_select_list(self):
# by default select_list is not automatically extended by
# order_by_list or group_by_list.
sql_expression = self.asSQLExpression({
'order_by_list': [('default',),]})
select_dict = sql_expression.getSelectDict()
self.assertEqual({}, select_dict)
sql_expression = self.asSQLExpression({
'group_by_list': ['default',]})
select_dict = sql_expression.getSelectDict()
self.assertEqual({}, select_dict)
# select_list is extended if auto_extend_select_list is enabled.
sql_expression = self.asSQLExpression({
'order_by_list': [('default',),]},
auto_extend_select_list=True)
select_dict = sql_expression.getSelectDict()
self.assertEqual({'foo_default__ext__': '`foo`.`default`'}, select_dict)
sql_expression = self.asSQLExpression({
'group_by_list': ['default',]},
auto_extend_select_list=True)
select_dict = sql_expression.getSelectDict()
self.assertEqual({'foo_default__ext__': '`foo`.`default`'}, select_dict)
# fulltext score is automatically added in select_dict even if
# auto_extend_select_list is not enabled.
sql_expression = self.asSQLExpression({
'fulltext': 'foo',
'order_by_list': [('fulltext__score__',),]})
select_dict = sql_expression.getSelectDict()
self.assertEqual(['foo_fulltext__score__'], select_dict.keys())
sql_expression = self.asSQLExpression({
'fulltext': 'foo',
'order_by_list': [('fulltext__score__',),]},
auto_extend_select_list=True)
select_dict = sql_expression.getSelectDict()
self.assertEqual(['foo_fulltext__score__'], select_dict.keys())
def _searchTextInDictQuery(self, column):
self.catalog(ReferenceQuery(ReferenceQuery(
ReferenceQuery(operator='>=', date=DateTime('2001/08/11')),
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment