Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
erp5
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Hamza
erp5
Commits
9cdf7a81
Commit
9cdf7a81
authored
Dec 14, 2011
by
Leonardo Rochael Almeida
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch 'catalog_join'
parents
07cbd374
d2a9088a
Changes
11
Show whitespace changes
Inline
Side-by-side
Showing
11 changed files
with
904 additions
and
115 deletions
+904
-115
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_related_grand_parent.xml
...rtal_catalog/erp5_mysql_innodb/z_related_grand_parent.xml
+7
-6
product/ERP5Catalog/tests/testERP5Catalog.py
product/ERP5Catalog/tests/testERP5Catalog.py
+194
-13
product/ERP5Type/Core/Predicate.py
product/ERP5Type/Core/Predicate.py
+3
-0
product/ZSQLCatalog/ColumnMap.py
product/ZSQLCatalog/ColumnMap.py
+217
-22
product/ZSQLCatalog/Query/EntireQuery.py
product/ZSQLCatalog/Query/EntireQuery.py
+65
-24
product/ZSQLCatalog/SQLCatalog.py
product/ZSQLCatalog/SQLCatalog.py
+10
-0
product/ZSQLCatalog/SQLExpression.py
product/ZSQLCatalog/SQLExpression.py
+16
-11
product/ZSQLCatalog/SearchKey/RelatedKey.py
product/ZSQLCatalog/SearchKey/RelatedKey.py
+107
-23
product/ZSQLCatalog/TableDefinition.py
product/ZSQLCatalog/TableDefinition.py
+283
-0
product/ZSQLCatalog/interfaces/column_map.py
product/ZSQLCatalog/interfaces/column_map.py
+0
-15
product/ZSQLCatalog/tests/testSQLCatalog.py
product/ZSQLCatalog/tests/testSQLCatalog.py
+2
-1
No files found.
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_related_grand_parent.xml
View file @
9cdf7a81
...
...
@@ -14,8 +14,10 @@
</item>
<item>
<key>
<string>
arguments_src
</string>
</key>
<value>
<string>
table_0\n
table_1
</string>
</value>
<value>
<string>
table_0\r\n
table_1\r\n
RELATED_QUERY_SEPARATOR=" AND "\r\n
query_table="catalog"
</string>
</value>
</item>
<item>
<key>
<string>
cache_time_
</string>
</key>
...
...
@@ -55,10 +57,9 @@ table_1</string> </value>
<key>
<string>
src
</string>
</key>
<value>
<string
encoding=
"cdata"
>
<![CDATA[
<dtml-var table_0>
.uid = catalog.parent_uid\n
AND
<dtml-var
table_1
>
.uid =
<dtml-var
table_0
>
.parent_uid\n
\n
<dtml-var table_1>
.uid =
<dtml-var
table_0
>
.parent_uid\n
<dtml-var
RELATED_QUERY_SEPARATOR
>
\n
<dtml-var
table_0
>
.uid =
<dtml-var
query_table
>
.parent_uid
]]>
</string>
</value>
</item>
...
...
product/ERP5Catalog/tests/testERP5Catalog.py
View file @
9cdf7a81
...
...
@@ -104,6 +104,12 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor):
new_erp5_deferred_sql_connection
=
'erp5_sql_deferred_connection2'
new_catalog_id
=
'erp5_mysql_innodb2'
__cleanups
=
()
def
_addCleanup
(
self
,
callable
):
self
.
__cleanups
+=
(
callable
,)
return
callable
def
afterSetUp
(
self
):
uf
=
self
.
getPortal
().
acl_users
uf
.
_doAddUser
(
self
.
username
,
''
,
[
'Manager'
],
[])
...
...
@@ -128,6 +134,8 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor):
self
.
portal
.
manage_delObjects
([
self
.
new_erp5_deferred_sql_connection
])
if
self
.
new_catalog_id
in
self
.
portal
.
portal_catalog
.
objectIds
():
self
.
portal
.
portal_catalog
.
manage_delObjects
([
self
.
new_catalog_id
])
for
cleanup
in
self
.
__cleanups
:
cleanup
(
self
)
transaction
.
commit
()
self
.
tic
()
...
...
@@ -707,7 +715,6 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor):
default_parametrs
[
'where_expression'
]
=
""
default_parametrs
[
'order_by_expression'
]
=
None
#import pdb; pdb.set_trace()
# check that we retrieve our 2 organisations by default.
kw
=
default_parametrs
.
copy
()
kw
.
update
(
portal_catalog
.
buildSQLQuery
(
...
...
@@ -955,14 +962,13 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor):
(
'catalog.title DESC'
,
'`catalog`.`title` DESC'
))
def
test_26_SortOnUnknownKeys
(
self
,
quiet
=
quiet
,
run
=
run_all_test
):
if
not
run
:
return
if
not
run
:
return
if
not
quiet
:
message
=
'Test Sort On Unknow Keys'
ZopeTestCase
.
_print
(
'
\
n
%s '
%
message
)
LOG
(
'Testing... '
,
0
,
message
)
self
.
assertEquals
(
''
,
self
.
getCatalogTool
().
buildSQLQuery
(
self
.
getCatalogTool
().
buildSQLQuery
(
select_list
=
(
'uid'
,
'path'
),
sort_on
=
((
'ignored'
,
'ascending'
),))[
'order_by_expression'
])
def
test_27_SortOnAmbigousKeys
(
self
,
quiet
=
quiet
,
run
=
run_all_test
):
...
...
@@ -1028,6 +1034,7 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor):
'sort_on parameter must be taken into account even if related key '
'is not a parameter of the current query'
)
def
_makeOrganisation
(
self
,
**
kw
):
"""Creates an Organisation in it's default module and reindex it.
By default, it creates a group/nexedi category, and make the organisation a
...
...
@@ -1332,11 +1339,13 @@ class TestERP5Catalog(ERP5TypeTestCase, LogInterceptor):
# catalog those objects
transaction
.
commit
()
self
.
tic
()
catalog_tool
=
self
.
getCatalogTool
()
self
.
assertEquals
([
ob
],
[
x
.
getObject
()
for
x
in
self
.
getCatalogTool
()(
portal_type
=
'Organisation'
,
SearchableText
=
'title'
)])
self
.
assertEquals
(
1
,
self
.
getCatalogTool
().
countResults
(
portal_type
=
'Organisation'
,
SearchableText
=
'title'
)[
0
][
0
])
[
x
.
getObject
()
for
x
in
catalog_tool
(
portal_type
=
'Organisation'
,
SearchableText
=
'title'
)])
self
.
assertEquals
(
1
,
catalog_tool
.
countResults
(
portal_type
=
'Organisation'
,
SearchableText
=
'title'
)[
0
][
0
])
# 'different' is found in more than 50% of records
# MySQL ignores such a word, but Tritonn does not ignore.
...
...
@@ -4279,11 +4288,13 @@ VALUES
self
.
assertEqual
(
catalog
.
countResults
(
parent_uid
=
module_uid
)[
0
][
0
],
module_len
)
self
.
assertEquals
(
catalog
.
countResults
(
from_expression
=
{
'catalog'
:
'(SELECT sub_catalog.* FROM catalog AS sub_catalog'
\
' WHERE sub_catalog.parent_uid=%i)'
\
from_expression
=
{
'catalog'
:
'(SELECT sub_catalog.* FROM catalog AS sub_catalog'
' WHERE sub_catalog.parent_uid=%i)'
' AS catalog'
%
(
module_uid
,
),
})[
0
][
0
],
module_len
)
}
count
=
catalog
.
countResults
(
from_expression
=
from_expression
)[
0
][
0
]
self
.
assertEqual
(
count
,
module_len
)
def
test_getParentUid
(
self
,
quiet
=
quiet
):
from
Products.ERP5.Document.Assignment
import
Assignment
...
...
@@ -4328,6 +4339,176 @@ VALUES
result
=
connector
.
manage_test
(
'select 1 as foo;'
)
self
.
assertEquals
(
1
,
result
[
0
].
foo
)
def
_createSomeGroupCategories
(
self
):
portal_category
=
self
.
getCategoryTool
()
group_category
=
portal_category
.
group
group_data_map
=
dict
(
nexedi
=
(
'Nexedi'
,
'Nexedi Group'
),
tiolive
=
(
'TIOLive'
,
'TioLive Group'
),)
existing_group_id_list
=
group_category
.
objectIds
()
for
group_id
,
(
title
,
description
)
in
group_data_map
.
items
():
if
group_id
in
existing_group_id_list
:
group
=
group_category
[
group_id
]
else
:
group
=
group_category
.
newContent
(
id
=
group_id
)
group
.
edit
(
title
=
title
,
description
=
description
)
def
test_SelectDictWithDynamicRelatedKey
(
self
,
quiet
=
quiet
,
run
=
run_all_test
):
if
not
run
:
return
if
not
quiet
:
message
=
'Select Dict With Dynamic Related Key'
ZopeTestCase
.
_print
(
'
\
n
%s '
%
message
)
LOG
(
'Testing... '
,
0
,
message
)
self
.
_createSomeGroupCategories
()
# Create some orgs associated with varying association with the
# groups created above.
module
=
self
.
portal
.
getDefaultModule
(
'Organisation'
)
# org1 has no groups
org1
=
module
.
newContent
(
portal_type
=
'Organisation'
,
title
=
'org1'
)
# org2 has group nexedi
org2
=
module
.
newContent
(
portal_type
=
'Organisation'
,
title
=
'org2'
)
org2
.
setGroupList
([
'nexedi'
])
# org3 has group tiolive
org3
=
module
.
newContent
(
portal_type
=
'Organisation'
,
title
=
'org3'
)
org3
.
setGroupList
([
'tiolive'
])
# org4 has both groups
org4
=
module
.
newContent
(
portal_type
=
'Organisation'
,
title
=
'org4'
)
org4
.
setGroupList
([
'nexedi'
,
'tiolive'
])
# check associations are correct
actual_group_title_map
=
dict
((
org
.
getTitle
(),
sorted
(
org
.
getGroupTitleList
()))
for
org
in
(
org1
,
org2
,
org3
,
org4
))
expected_group_title_map
=
dict
(
org1
=
[],
org2
=
[
'Nexedi'
],
org3
=
[
'TIOLive'
],
org4
=
[
'Nexedi'
,
'TIOLive'
])
self
.
assertEquals
(
actual_group_title_map
,
expected_group_title_map
)
# Flush message queue
transaction
.
commit
()
self
.
tic
()
# we will restrict our search to orgs with these ids to be resilient
# to preexisting orgs:
org_id_list
=
sorted
(
org
.
getId
()
for
org
in
(
org1
,
org2
,
org3
,
org4
))
# and we'll sort on title to make the output predictable
search_kw
=
dict
(
id
=
org_id_list
,
sort_on
=
'title'
)
# Try to get the organisations with the group title Nexedi to make sure
# searching works correctly
organisation_list
=
[
x
.
getObject
()
for
x
in
module
.
searchFolder
(
strict_group_title
=
'Nexedi'
,
**
search_kw
)]
self
.
assertEquals
(
organisation_list
,
[
org2
,
org4
])
# Now lets fetch the titles of groups of the above orgs using select_dict.
search_kw
.
update
(
select_dict
=
dict
(
strict_group_title
=
None
))
records
=
module
.
searchFolder
(
**
search_kw
)
# by default the catalog only returns items containing the
# relationship we asked for (group). Besides, some entries will
# appear many times, according to the number of relationships each
# catalog entry has in that related key.
results
=
[(
rec
.
title
,
rec
.
strict_group_title
or
'-None-'
)
for
rec
in
records
]
self
.
assertEquals
(
sorted
(
results
),
[(
'org2'
,
'Nexedi'
),
(
'org3'
,
'TIOLive'
),
(
'org4'
,
'Nexedi'
),
(
'org4'
,
'TIOLive'
)])
# But if we demand a left-join on that column, then we'll have all
# orgs we created. They'll still be repeated according to their
# relationships, though.
search_kw
.
update
(
left_join_list
=
(
'strict_group_title'
,))
records
=
module
.
searchFolder
(
**
search_kw
)
results
=
[(
rec
.
title
,
rec
.
strict_group_title
or
'-None-'
)
for
rec
in
records
]
self
.
assertEquals
(
sorted
(
results
),
[(
'org1'
,
'-None-'
),
(
'org2'
,
'Nexedi'
),
(
'org3'
,
'TIOLive'
),
(
'org4'
,
'Nexedi'
),
(
'org4'
,
'TIOLive'
)])
# To get only one of each org, we need to group by one of the
# catalog keys.
# Note that this relies on a non-standard behaviour
# of MySQL: If a selected column is not present in the GROUP BY
# clause, only the first ocurrence is taken. Other databases,
# like Oracle, assume that selected columns are either GROUPed BY
# or are inside an aggregation function (COUNT, SUM, GROUP_CONCAT,
# ...), and consider the query to be in error otherwise.
search_kw
.
update
(
group_by_list
=
(
'uid'
,))
organisation_list
=
[
x
.
getObject
()
for
x
in
module
.
searchFolder
(
**
search_kw
)]
self
.
assertEquals
(
organisation_list
,
[
org1
,
org2
,
org3
,
org4
])
def
test_BackwardCompatibilityWithOldMethods
(
self
,
quiet
=
quiet
,
run
=
run_all_test
):
if
not
run
:
return
if
not
quiet
:
message
=
'Dealing with RelatedKey methods missing the proper separator'
ZopeTestCase
.
_print
(
'
\
n
%s '
%
message
)
LOG
(
'Testing... '
,
0
,
message
)
module
=
self
.
getOrganisationModule
()
org_a
=
self
.
_makeOrganisation
(
title
=
'abc'
,
default_address_city
=
'abc'
)
org_a
.
setReference
(
org_a
.
getId
())
# sometimes the module itself is not indexed yet...
module
.
reindexObject
()
# Flush message queue
transaction
.
commit
()
self
.
tic
()
# make a query to fetch the address of the organisation above by
# querying, among other things, the grand_parent
query
=
dict
(
grand_parent_portal_type
=
"Organisation Module"
,
parent_reference
=
org_a
.
getReference
())
catalog
=
self
.
getCatalogTool
()
# check the query works normally
self
.
assertEqual
([
x
.
getObject
()
for
x
in
catalog
.
searchResults
(
**
query
)],
[
org_a
.
default_address
])
# even if we do a left_join
query_lj
=
query
.
copy
()
query_lj
.
update
(
left_join_list
=
(
'grand_parent_portal_type'
,))
self
.
assertEqual
([
x
.
getObject
()
for
x
in
catalog
.
searchResults
(
**
query_lj
)],
[
org_a
.
default_address
])
# now turn the z_related_grand_parent into an old-style method, without
# RELATED_QUERY_SEPARATOR
method
=
catalog
.
getSQLCatalog
().
z_related_grand_parent
old_src
=
method
.
src
@
self
.
_addCleanup
def
cleanGrandParentMethod
(
self
):
method
.
manage_edit
(
method
.
title
,
method
.
connection_id
,
method
.
arguments_src
,
old_src
)
src
=
old_src
.
replace
(
'<dtml-var RELATED_QUERY_SEPARATOR>'
,
' AND '
)
method
.
manage_edit
(
method
.
title
,
method
.
connection_id
,
method
.
arguments_src
,
src
)
# check that it still works
self
.
assertEqual
([
x
.
getObject
()
for
x
in
catalog
.
searchResults
(
**
query
)],
[
org_a
.
default_address
])
# now try to do a left-join on grand_parent_portal_type which
# shouldn't work
self
.
assertRaises
(
RuntimeError
,
lambda
:
catalog
.
searchResults
(
**
query_lj
))
# Neither should it work if a left-join is attempted in a column
# that has proper related-key rendering, but is present in the
# same query as a column that hasn't, as the whole query is
# converted into implicit inner joins.
self
.
tic
()
query_lj
.
update
(
left_join_list
=
(
'strict_group_title'
,),
select_dict
=
(
'strict_group_title'
,))
self
.
assertRaises
(
RuntimeError
,
lambda
:
catalog
.
searchResults
(
**
query_lj
))
# though it should work on queries that don't use the broken related-key
del
query_lj
[
'grand_parent_portal_type'
]
self
.
assertEqual
([
x
.
getObject
()
for
x
in
catalog
.
searchResults
(
**
query_lj
)],
[
org_a
.
default_address
])
def
test_suite
():
suite
=
unittest
.
TestSuite
()
suite
.
addTest
(
unittest
.
makeSuite
(
TestERP5Catalog
))
...
...
product/ERP5Type/Core/Predicate.py
View file @
9cdf7a81
...
...
@@ -329,7 +329,10 @@ class Predicate(XMLObject):
catalog_kw
[
'where_expression'
]
=
SQLQuery
(
sql_text
)
else
:
catalog_kw
[
'where_expression'
]
=
''
# force implicit join
catalog_kw
[
'implicit_join'
]
=
True
sql_query
=
portal_catalog
.
buildSQLQuery
(
**
catalog_kw
)
# XXX from_table_list is None most of the time after the explicit_join work
for
alias
,
table
in
sql_query
[
'from_table_list'
]:
if
from_table_dict
.
has_key
(
alias
):
raise
KeyError
,
"The same table is used twice for an identity criterion and for a membership criterion"
...
...
product/ZSQLCatalog/ColumnMap.py
View file @
9cdf7a81
...
...
@@ -27,11 +27,18 @@
#
##############################################################################
import
re
import
itertools
from
zLOG
import
LOG
,
WARNING
,
INFO
from
interfaces.column_map
import
IColumnMap
from
zope.interface.verify
import
verifyClass
from
zope.interface
import
implements
from
SQLCatalog
import
profiler_decorator
from
Products.ZSQLCatalog.interfaces.column_map
import
IColumnMap
from
Products.ZSQLCatalog.SQLCatalog
import
profiler_decorator
from
Products.ZSQLCatalog.TableDefinition
import
(
PlaceHolderTableDefinition
,
TableAlias
,
InnerJoin
,
LeftJoin
)
DEFAULT_GROUP_ID
=
None
...
...
@@ -43,12 +50,18 @@ MAPPING_TRACE = False
# currently, it's not possible because related_key_dict is indexed by related key name, which makes 'source_title_1' lookup fail. It should be indexed by group (probably).
# TODO: rename all "related_key" references into "virtual_column"
re_sql_as
=
re
.
compile
(
"
\
s+AS
\
s[^)]+$"
,
re
.
IGNORECASE
|
re
.
MULTILINE
)
class
ColumnMap
(
object
):
implements
(
IColumnMap
)
@
profiler_decorator
def
__init__
(
self
,
catalog_table_name
=
None
):
def
__init__
(
self
,
catalog_table_name
=
None
,
table_override_map
=
None
,
left_join_list
=
None
,
implicit_join
=
False
):
self
.
catalog_table_name
=
catalog_table_name
# Key: group
# Value: set of column names
...
...
@@ -81,15 +94,28 @@ class ColumnMap(object):
self
.
raw_column_dict
=
{}
# Entries: column name
self
.
column_ignore_set
=
set
()
self
.
join_table_set
=
set
()
self
.
straight_join_table_list
=
[]
self
.
left_join_table_list
=
[]
self
.
join_table_map
=
dict
()
# BBB: Remove join_query_list and its uses when all RelatedKey
# methods have been converted to properly return each Join
# condition separately, and all uses of catalog's from_expression
# have been removed.
self
.
join_query_list
=
[]
self
.
table_override_map
=
table_override_map
or
{}
self
.
table_definition
=
PlaceHolderTableDefinition
()
# We need to keep track of the original definition to do inner joins on it
self
.
_inner_table_definition
=
self
.
table_definition
self
.
left_join_list
=
left_join_list
self
.
implicit_join
=
implicit_join
assert
not
(
self
.
implicit_join
and
self
.
left_join_list
),
(
"Cannot do left_joins while forcing implicit join"
)
@
profiler_decorator
def
registerColumn
(
self
,
raw_column
,
group
=
DEFAULT_GROUP_ID
,
simple_query
=
None
):
assert
' as '
not
in
raw_column
.
lower
()
# Sanitize input: extract column from raw column (might contain COUNT, ...).
# XXX This is not enough to parse something like:
# GROUP_CONCAT(DISTINCT foo ORDER BY bar)
if
'('
in
raw_column
:
function
,
column
=
raw_column
.
split
(
'('
)
column
=
column
.
strip
()
...
...
@@ -120,7 +146,7 @@ class ColumnMap(object):
# When a column is registered in default group and is explicitely
# mapped to a table, we must mark its table as requiring a join with
# catalog table (unless it's the catalog table, of course).
self
.
_addJoinTable
(
table
,
group
)
self
.
_addJoinTable
ForColumn
(
table
,
table
+
"."
+
column
,
group
)
def
ignoreColumn
(
self
,
column
):
self
.
column_ignore_set
.
add
(
column
)
...
...
@@ -255,7 +281,7 @@ class ColumnMap(object):
# Although the list of tables those columns belong to is known
# earlier (in "build"), mapping them here
# - avoids code duplication (registerTable, resolveColumn,
# _addJoinTable)
# _addJoinTable
ForColumn
)
# - offers user to vote for an unknown table, overriding this
# forced mapping.
use_allowed
=
table_name
==
catalog_table_name
or
\
...
...
@@ -300,10 +326,11 @@ class ColumnMap(object):
self
.
registerTable
(
table_name
,
group
=
group
)
self
.
resolveColumn
(
column_name
,
table_name
,
group
=
group
)
if
table_name
!=
catalog_table_name
:
self
.
_addJoinTable
(
table
_name
,
group
)
self
.
_addJoinTable
ForColumn
(
table_name
,
column
_name
,
group
)
@
profiler_decorator
def
build
(
self
,
sql_catalog
):
join_query_to_build_list
=
[]
catalog_table_name
=
self
.
catalog_table_name
if
catalog_table_name
is
None
:
return
...
...
@@ -319,7 +346,7 @@ class ColumnMap(object):
if
related_key_definition
is
not
None
:
join_query
=
sql_catalog
.
getSearchKey
(
column_name
,
'RelatedKey'
).
buildQuery
(
sql_catalog
=
sql_catalog
,
related_key_definition
=
related_key_definition
)
join_query
.
registerColumnMap
(
sql_catalog
,
self
)
self
.
_addJoinQuery
(
join_query
)
join_query_to_build_list
.
append
(
join_query
)
# List all possible tables, with all used column for each
for
group
,
column_set
in
self
.
registry
.
iteritems
():
...
...
@@ -397,6 +424,20 @@ class ColumnMap(object):
table_alias_number_dict
[
alias_table_name
]
=
table_alias_number
self
.
resolveTable
(
table_name
,
alias
,
group
=
group
)
# now that we have all aliases, calculate missing joins comming from
# non-RelatedKey relationships (like full_text).
self
.
registerCatalog
()
self
.
_calculateMissingJoins
()
# and all left joins that did not come from explicit queries
# (i.e. joins comming from 'sort_on', 'select_dict', etc.)
for
join_query
in
join_query_to_build_list
:
# XXX ugly use of inner attribute of join_query. Please Refactor:
# search_keys don't actually return SQLExpressions, but they add
# join definitions in the column_map
join_query
.
search_key
.
buildSQLExpression
(
sql_catalog
=
sql_catalog
,
column_map
=
self
,
only_group_columns
=
False
,
group
=
join_query
.
group
,)
if
MAPPING_TRACE
:
# Key: group
# Value: 2-tuple
...
...
@@ -441,8 +482,28 @@ class ColumnMap(object):
def
getCatalogTableAlias
(
self
,
group
=
DEFAULT_GROUP_ID
):
return
self
.
table_alias_dict
[(
group
,
self
.
catalog_table_name
)]
def
_isBackwardCompatibilityRequired
(
self
):
return
bool
(
# if they explicitly ask for implicit
self
.
implicit_join
or
# if they don't pass a catalog alias, we cannot do explicit joins
not
self
.
_setMinimalTableDefinition
()
or
# If one or more RelatedKey methods weren't converted, we'll get
# queries for an implicit inner join, so we have to do all joins
# as implicit.
self
.
join_query_list
or
# for now, work in BW compat mode if a table_override
# is passed. It only works for simple subselect
# definitions anyway, and it's being used primarily
# for writing left-joins manually.
self
.
table_override_map
)
def
getTableAliasDict
(
self
):
if
self
.
_isBackwardCompatibilityRequired
():
# BBB: Using implicit joins or explicit from_expression
return
self
.
table_map
.
copy
()
else
:
return
None
@
profiler_decorator
def
resolveColumn
(
self
,
column
,
table_name
,
group
=
DEFAULT_GROUP_ID
):
...
...
@@ -472,16 +533,30 @@ class ColumnMap(object):
def
getTableAlias
(
self
,
table_name
,
group
=
DEFAULT_GROUP_ID
):
return
self
.
table_alias_dict
[(
group
,
table_name
)]
def
_addJoinQuery
(
self
,
query
):
def
_addJoinQueryForColumn
(
self
,
column
,
query
):
# BBB: This is a backward compatibility method that will be
# removed in the future, when all related key methods have been adapted
# to provide all Join conditions separately
if
column
in
self
.
left_join_list
:
raise
RuntimeError
(
'Left Join requested for column: %r, but rendered '
'join query is not compatible and would result in an '
'Implicit Inner Join:
\
n
%s'
%
(
column
,
query
,))
self
.
join_query_list
.
append
(
query
)
def
iterJoinQueryList
(
self
):
return
iter
(
self
.
join_query_list
)
if
self
.
_isBackwardCompatibilityRequired
():
# Return all join queries for implicit join, and all the other
# queries we were using to build explicit joins, but won't be able to.
return
itertools
.
chain
(
self
.
join_query_list
,
self
.
table_definition
.
getJoinConditionQueryList
())
return
[]
@
profiler_decorator
def
_addJoinTable
(
self
,
table_name
,
group
=
DEFAULT_GROUP_ID
):
def
_addJoinTable
ForColumn
(
self
,
table_name
,
column
,
group
=
DEFAULT_GROUP_ID
):
"""
Declare given table as requiring to be joined with catalog table.
Declare given table as requiring to be joined with catalog table
on uid
.
table_name (string)
Table name.
...
...
@@ -497,17 +572,137 @@ class ColumnMap(object):
# Register uid column if it is not already
self
.
registerColumn
(
'uid'
)
self
.
resolveColumn
(
'uid'
,
catalog_table
)
self
.
join_table_
set
.
add
((
group
,
table_name
)
)
self
.
join_table_
map
.
setdefault
((
group
,
table_name
),
set
()).
add
(
column
)
def
getJoinTableAliasList
(
self
):
return
[
self
.
getTableAlias
(
table_name
,
group
=
group
)
for
(
group
,
table_name
)
in
self
.
join_table_set
]
def
getStraightJoinTableList
(
self
):
return
self
.
straight_join_table_list
[:]
def
getLeftJoinTableList
(
self
):
return
self
.
left_join_table_list
[:]
for
(
group
,
table_name
)
in
self
.
join_table_map
.
keys
()]
def
_getTableOverride
(
self
,
table_name
):
# self.table_override_map is a dictionary mapping table names to
# strings containing aliases of arbitrary table definitions
# (including subselects). So we split the alias and discard it
# since we do our own aliasing.
table_override_w_alias
=
self
.
table_override_map
.
get
(
table_name
)
if
table_override_w_alias
is
None
:
return
table_name
# XXX move the cleanup of table alias overrides to EntireQuery
# class or ZSQLCatalog, so we don't need SQL syntax knowledge in
# ColumnMap.
#
# Normalise the AS sql keyword to remove the last
# aliasing in the string if present. E.g.:
#
# '(SELECT sub_catalog.*
# FROM catalog AS sub_catalog
# WHERE sub_catalog.parent_uid=183) AS catalog'
#
# becomes:
#
# '(SELECT sub_catalog.*
# FROM catalog AS sub_catalog
# WHERE sub_catalog.parent_uid=183)'
table_override
,
removed
=
re_sql_as
.
subn
(
''
,
table_override_w_alias
)
assert
removed
<
2
,
(
'More than one table aliasing was removed from %r'
%
table_override_w_alias
)
if
removed
:
LOG
(
'ColumnMap'
,
WARNING
,
'Table overrides should not contain aliasing: %r'
%
table_override
)
return
table_override
def
makeTableAliasDefinition
(
self
,
table_name
,
table_alias
):
"""Make a table alias, giving a change to ColumnMap to override
the original table definition with another expression"""
table_name
=
self
.
_getTableOverride
(
table_name
)
assert
table_name
and
table_alias
,
(
"table_name (%r) and table_alias (%r) "
"must both be defined"
%
(
table_name
,
table_alias
))
return
TableAlias
(
table_name
,
table_alias
)
def
_setMinimalTableDefinition
(
self
):
""" Set a minimal table definition: the main catalog alias
We don't do this at __init__ because we have neither the catalog
table name nor its intended alias at that point.
"""
inner_def
=
self
.
_inner_table_definition
if
inner_def
.
table_definition
is
None
:
try
:
catalog_table_alias
=
self
.
getCatalogTableAlias
()
except
KeyError
:
LOG
(
'ColumnMap'
,
WARNING
,
'_setMinimalTableDefinition called but the main catalog has not '
'yet received an alias!'
)
return
False
inner_def
.
replace
(
self
.
makeTableAliasDefinition
(
self
.
catalog_table_name
,
catalog_table_alias
))
return
True
def
getTableDefinition
(
self
):
if
self
.
_isBackwardCompatibilityRequired
():
# BBB: One of the RelatedKeys registered an implicit join, do
# not return a table definition, self.getTableAliasDict() should
# be used instead
return
None
self
.
table_definition
.
checkTableAliases
()
return
self
.
table_definition
def
addRelatedKeyJoin
(
self
,
column
,
right_side
,
condition
):
""" Wraps the current table_definition in the left-side of a new
join. Use an InnerJoin or a LeftJoin depending on whether the
column is in the left_join_list or not.
"""
# XXX: to fix TestERP5Catalog.test_52_QueryAndTableAlias, create
# here a list of joins and try to merge each new entry into one of
# the pre-existing entries by comparing their right-sides.
#
# XXX 2: This is the place were we could do ordering of inner and left
# joins so as to get better performance. For instance, a quick win is to
# add all inner-joins first, and all left-joins later. We could also decide
# on the order of left-joins based on the order of self.left_join_list or
# even a catalog property/configuration/script.
#
# XXX 3: This is also the place where we could check if explicit
# table aliases should cause some of these table definitions to be
# collapsed into others.
assert
self
.
_setMinimalTableDefinition
()
Join
=
(
column
in
self
.
left_join_list
)
and
LeftJoin
or
InnerJoin
join_definition
=
Join
(
self
.
table_definition
,
right_side
,
condition
=
condition
)
self
.
table_definition
=
join_definition
# def getFinalTableDefinition(self):
# self._calculateMissingJoins()
# return self.getTableDefinition()
def
_calculateMissingJoins
(
self
):
left_join_set
=
set
(
self
.
left_join_list
)
self
.
_setMinimalTableDefinition
()
catalog_table_alias
=
self
.
getCatalogTableAlias
()
for
(
group
,
table_name
),
column_set
in
self
.
join_table_map
.
items
():
# if any of the columns for this implicit join was requested as a
# left-join, then all columns will be subject to a left-join.
# XXX What if one of the columns was an actual query, as opposed to a
# sort column or select_dict? This would cause results in the main
# catalog that don't match the query to be present as well. We expect
# the user which passes a left_join_list to know what he is doing.
if
column_set
.
intersection
(
left_join_set
):
Join
=
LeftJoin
else
:
Join
=
InnerJoin
table_alias
=
self
.
getTableAlias
(
table_name
,
group
=
group
)
table_alias_def
=
self
.
makeTableAliasDefinition
(
table_name
,
table_alias
)
# XXX: perhaps refactor some of the code below to do:
# self._inner_table_definition.addInnerJoin(TableAlias(...),
# condition=(...))
self
.
_inner_table_definition
.
replace
(
Join
(
self
.
_inner_table_definition
.
table_definition
,
table_alias_def
,
# XXX ColumnMap shouldn't have SQL knowledge
condition
=
(
'`%s`.`uid` = `%s`.`uid`'
%
(
table_alias
,
catalog_table_alias
)),
)
)
verifyClass
(
IColumnMap
,
ColumnMap
)
product/ZSQLCatalog/Query/EntireQuery.py
View file @
9cdf7a81
...
...
@@ -28,6 +28,7 @@
#
##############################################################################
import
warnings
from
Products.ZSQLCatalog.SQLExpression
import
SQLExpression
from
Products.ZSQLCatalog.ColumnMap
import
ColumnMap
from
zLOG
import
LOG
...
...
@@ -35,6 +36,7 @@ from Products.ZSQLCatalog.interfaces.entire_query import IEntireQuery
from
zope.interface.verify
import
verifyClass
from
zope.interface
import
implements
from
Products.ZSQLCatalog.SQLCatalog
import
profiler_decorator
from
Products.ZSQLCatalog.TableDefinition
import
LegacyTableDefinition
def
defaultDict
(
value
):
if
value
is
None
:
...
...
@@ -54,19 +56,28 @@ class EntireQuery(object):
column_map
=
None
@
profiler_decorator
def
__init__
(
self
,
query
,
order_by_list
=
(),
group_by_list
=
(),
select_dict
=
None
,
limit
=
None
,
catalog_table_name
=
None
,
extra_column_list
=
(),
from_expression
=
None
,
order_by_override_list
=
None
):
def
__init__
(
self
,
query
,
order_by_list
=
(),
group_by_list
=
(),
select_dict
=
None
,
left_join_list
=
(),
limit
=
None
,
catalog_table_name
=
None
,
extra_column_list
=
(),
from_expression
=
None
,
order_by_override_list
=
None
,
implicit_join
=
False
):
self
.
query
=
query
self
.
order_by_list
=
list
(
order_by_list
)
self
.
order_by_override_set
=
frozenset
(
order_by_override_list
)
self
.
group_by_list
=
list
(
group_by_list
)
self
.
select_dict
=
defaultDict
(
select_dict
)
self
.
left_join_list
=
left_join_list
self
.
limit
=
limit
self
.
catalog_table_name
=
catalog_table_name
self
.
extra_column_list
=
list
(
extra_column_list
)
self
.
from_expression
=
from_expression
self
.
implicit_join
=
implicit_join
def
asSearchTextExpression
(
self
,
sql_catalog
):
return
self
.
query
.
asSearchTextExpression
(
sql_catalog
)
...
...
@@ -78,7 +89,12 @@ class EntireQuery(object):
# XXX: should we provide a way to register column map as a separate
# method or do it here ?
# Column Map was not built yet, do it.
self
.
column_map
=
column_map
=
ColumnMap
(
catalog_table_name
=
self
.
catalog_table_name
)
column_map
=
ColumnMap
(
catalog_table_name
=
self
.
catalog_table_name
,
table_override_map
=
self
.
from_expression
,
left_join_list
=
self
.
left_join_list
,
implicit_join
=
self
.
implicit_join
,
)
self
.
column_map
=
column_map
for
extra_column
in
self
.
extra_column_list
:
table
,
column
=
extra_column
.
replace
(
'`'
,
''
).
split
(
'.'
)
if
table
!=
self
.
catalog_table_name
:
...
...
@@ -145,30 +161,55 @@ class EntireQuery(object):
None
,
)
*
(
3
-
len
(
order_by
)))
self
.
order_by_list
=
new_order_by_list
# generate SQLExpression from query
sql_expression_list
=
[
self
.
query
.
asSQLExpression
(
sql_catalog
,
column_map
,
only_group_columns
)]
# generate join expression based on column_map.getJoinTableAliasList
sql_expression_list
=
[
self
.
query
.
asSQLExpression
(
sql_catalog
,
column_map
,
only_group_columns
)]
append
=
sql_expression_list
.
append
for
join_query
in
column_map
.
iterJoinQueryList
():
append
(
join_query
.
asSQLExpression
(
sql_catalog
,
column_map
,
only_group_columns
))
join_table_list
=
column_map
.
getJoinTableAliasList
()
if
len
(
join_table_list
):
# XXX: Is there any special rule to observe when joining tables ?
# Maybe we could check which column is a primary key instead of
# hardcoding "uid".
where_pattern
=
'`%s`.`uid` = `%%s`.`uid`'
%
\
(
column_map
.
getCatalogTableAlias
(),
)
# XXX: It would cleaner from completeness point of view to use column
# mapper to render column, but makes code much more complex to just do
# a simple text rendering. If there is any reason why we should have
# those column in the mapper, then we should use the clean way.
append
(
SQLExpression
(
self
,
where_expression
=
' AND '
.
join
(
where_pattern
%
(
x
,
)
for
x
in
join_table_list
)))
append
(
join_query
.
asSQLExpression
(
sql_catalog
,
column_map
,
only_group_columns
))
# generate join expression based on column_map.getJoinTableAliasList
# XXX: This is now done by ColumnMap to its table_definition,
# during build()
#
# join_table_list = column_map.getJoinTableAliasList()
# if len(join_table_list):
# # XXX: Is there any special rule to observe when joining tables ?
# # Maybe we could check which column is a primary key instead of
# # hardcoding "uid".
# where_pattern = '`%s`.`uid` = `%%s`.`uid`' % \
# (column_map.getCatalogTableAlias(), )
# # XXX: It would cleaner from completeness point of view to use column
# # mapper to render column, but makes code much more complex to just do
# # a simple text rendering. If there is any reason why we should have
# # those column in the mapper, then we should use the clean way.
# append(SQLExpression(self, where_expression=' AND '.join(
# where_pattern % (x, ) for x in join_table_list
# )))
# BBB self.from_expression forces use of implicit inner join
table_alias_dict
=
column_map
.
getTableAliasDict
()
if
self
.
from_expression
:
warnings
.
warn
(
"Providing a 'from_expression' is deprecated."
,
DeprecationWarning
)
# XXX: perhaps move this code to ColumnMap?
legacy_from_expression
=
self
.
from_expression
from_expression
=
LegacyTableDefinition
(
legacy_from_expression
,
table_alias_dict
)
table_alias_dict
=
None
else
:
from_expression
=
column_map
.
getTableDefinition
()
assert
((
from_expression
is
None
)
!=
(
table_alias_dict
is
None
)),
(
"Got both a from_expression "
"and a table_alias_dict"
)
self
.
sql_expression_list
=
sql_expression_list
# TODO: wrap the table_alias_dict above into a TableDefinition as well,
# even without a legacy_table_definition.
return
SQLExpression
(
self
,
table_alias_dict
=
column_map
.
getTableAliasDict
()
,
from_expression
=
self
.
from_expression
,
table_alias_dict
=
table_alias_dict
,
from_expression
=
from_expression
,
order_by_list
=
self
.
order_by_list
,
group_by_list
=
self
.
group_by_list
,
select_dict
=
self
.
final_select_dict
,
...
...
product/ZSQLCatalog/SQLCatalog.py
View file @
9cdf7a81
...
...
@@ -2291,6 +2291,13 @@ class Catalog(Folder,
select_dict
=
None
elif
isinstance
(
select_dict
,
(
list
,
tuple
)):
select_dict
=
dict
([(
x
,
None
)
for
x
in
select_dict
])
# Handle left_join_list
left_join_list
=
kw
.
pop
(
'left_join_list'
,
())
# Handle implicit_join. It's True by default, as there's a lot of code
# in BT5s and elsewhere that calls buildSQLQuery() expecting implicit
# join. self._queryResults() defaults it to False for those using
# catalog.searchResults(...) or catalog(...) directly.
implicit_join
=
kw
.
pop
(
'implicit_join'
,
True
)
# Handle order_by_list
order_by_list
=
kw
.
pop
(
'order_by_list'
,
None
)
sort_on
=
kw
.
pop
(
'sort_on'
,
None
)
...
...
@@ -2328,6 +2335,8 @@ class Catalog(Folder,
order_by_override_list
=
order_by_override_list
,
group_by_list
=
group_by_list
,
select_dict
=
select_dict
,
left_join_list
=
left_join_list
,
implicit_join
=
implicit_join
,
limit
=
limit
,
catalog_table_name
=
query_table
,
extra_column_list
=
extra_column_list
,
...
...
@@ -2413,6 +2422,7 @@ class Catalog(Folder,
""" Returns a list of brains from a set of constraints on variables """
if
build_sql_query_method
is
None
:
build_sql_query_method
=
self
.
buildSQLQuery
kw
.
setdefault
(
'implicit_join'
,
False
)
query
=
build_sql_query_method
(
REQUEST
=
REQUEST
,
**
kw
)
# XXX: decide if this should be made normal
ENFORCE_SEPARATION
=
True
...
...
product/ZSQLCatalog/SQLExpression.py
View file @
9cdf7a81
...
...
@@ -166,7 +166,7 @@ class SQLExpression(object):
@
profiler_decorator
def
getFromExpression
(
self
):
"""
Returns a
string.
Returns a
TableDefinition stored in one of the from_expressions or None
If there are nested SQLExpression, it checks that they either don't
define any from_expression or the exact same from_expression. Otherwise,
...
...
@@ -175,7 +175,7 @@ class SQLExpression(object):
result
=
self
.
from_expression
for
sql_expression
in
self
.
sql_expression_list
:
from_expression
=
sql_expression
.
getFromExpression
()
if
None
not
in
(
result
,
from_expression
):
if
from_expression
not
in
(
result
,
None
):
message
=
'I don
\
'
t know how to merge from_expressions'
if
DEBUG
:
message
=
message
+
'. I was created by %r, and I am working on %r (%r) out of [%s]'
%
(
...
...
@@ -385,20 +385,25 @@ class SQLExpression(object):
SQL_SELECT_ALIAS_FORMAT
%
(
column
,
alias
)
for
alias
,
column
in
self
.
getSelectDict
().
iteritems
())
@
profiler_decorator
def
asSQLExpressionDict
(
self
):
def
getFromTableList
(
self
):
table_alias_dict
=
self
.
getTableAliasDict
()
if
not
table_alias_dict
:
return
None
from_table_list
=
[]
append
=
from_table_list
.
append
for
alias
,
table
in
table_alias_dict
.
iteritems
():
append
((
SQL_TABLE_FORMAT
%
(
alias
,
),
SQL_TABLE_FORMAT
%
(
table
,
)))
from_expression_dict
=
self
.
getFromExpression
()
if
from_expression_dict
is
not
None
:
from_expression
=
SQL_LIST_SEPARATOR
.
join
(
from_expression_dict
.
get
(
alias
,
'`%s` AS `%s`'
%
(
table
,
alias
))
for
alias
,
table
in
table_alias_dict
.
iteritems
())
else
:
from_expression
=
None
return
from_table_list
@
profiler_decorator
def
asSQLExpressionDict
(
self
):
from_expression
=
self
.
getFromExpression
()
from_table_list
=
self
.
getFromTableList
()
assert
None
in
(
from_expression
,
from_table_list
),
(
"Cannot return both a from_expression "
"and a from_table_list"
)
if
from_expression
is
not
None
:
from_expression
=
from_expression
.
render
()
return
{
'where_expression'
:
self
.
getWhereExpression
(),
'order_by_expression'
:
self
.
getOrderByExpression
(),
...
...
product/ZSQLCatalog/SearchKey/RelatedKey.py
View file @
9cdf7a81
...
...
@@ -37,12 +37,29 @@ from Products.ZSQLCatalog.interfaces.search_key import IRelatedKey
from
zope.interface.verify
import
verifyClass
from
zope.interface
import
implements
from
Products.ZSQLCatalog.SQLCatalog
import
profiler_decorator
from
Products.ZSQLCatalog.TableDefinition
import
TableAlias
,
InnerJoin
,
LeftJoin
from
logging
import
getLogger
log
=
getLogger
(
__name__
)
BACKWARD_COMPATIBILITY
=
True
RELATED_QUERY_SEPARATOR
=
"
\
n
AND -- related query separator
\
n
"
RELATED_KEY_MISMATCH_MESSAGE
=
"
\
A rendered related key must contain the same number of querying
\
conditions as the tables it relates, properly separated by
\
RELATED_QUERY_SEPARATOR.
\
n
\
Offending related key: %r, for column %r, table_alias_list: %r,
\
rendered_related_key:
\
n
%s"
RELATED_KEY_ALIASED_MESSAGE
=
"
\
Support for explicit joins of aliased related keys is not yet implemented.
\
Offending related key: %r, for column %r, table_alias_list: %r"
class
RelatedKey
(
SearchKey
):
"""
This SearchKey handles searche
d
on virtual columns of RelatedKey type.
This SearchKey handles searche
s
on virtual columns of RelatedKey type.
It generates joins required by the virtual column to reach the actual
column to compare, plus a regular query on that column if needed.
"""
...
...
@@ -115,13 +132,23 @@ class RelatedKey(SearchKey):
def
registerColumnMap
(
self
,
column_map
,
table_alias_list
=
None
):
related_column
=
self
.
getColumn
()
group
=
column_map
.
registerRelatedKey
(
related_column
,
self
.
real_column
)
# Each table except last one must be registered to their own group, so that
# the same table can be used multiple time (and aliased multiple times)
# in the same related key. The last one must be register to related key
# "main" group (ie, the value of the "group" variable) to be the same as
# the ta ble used in join_condition.
# Each table except last one must be registered to their own
# group, so that the same table can be used multiple times (and
# aliased multiple times) in the same related key. The last one
# must be registered to the related key "main" group (ie, the
# value of the "group" variable) to be the same as the table used
# in join_condition.
if
table_alias_list
is
not
None
:
assert
len
(
self
.
table_list
)
==
len
(
table_alias_list
)
# XXX-Leo: remove the rest of this 'if' branch after making sure
# that ColumnMap.addRelatedKeyJoin() can handle collapsing
# chains of inner-joins that are subsets of one another based on
# having the same aliases:
msg
=
RELATED_KEY_ALIASED_MESSAGE
%
(
self
.
related_key_id
,
self
.
column
,
table_alias_list
,)
log
.
warning
(
msg
+
"
\
n
\
n
Forcing implicit join..."
)
column_map
.
implicit_join
=
True
for
table_position
in
xrange
(
len
(
self
.
table_list
)
-
1
):
table_name
=
self
.
table_list
[
table_position
]
local_group
=
column_map
.
registerRelatedKeyColumn
(
related_column
,
table_position
,
group
)
...
...
@@ -145,6 +172,23 @@ class RelatedKey(SearchKey):
column_map
.
registerCatalog
()
return
group
def
stitchJoinDefinition
(
self
,
table_alias_list
,
join_query_list
,
column_map
):
alias
,
table
=
table_alias_list
[
-
1
]
right
=
column_map
.
makeTableAliasDefinition
(
table
,
alias
)
if
not
join_query_list
:
# nothing to do, just return the table alias
assert
len
(
table_alias_list
)
==
1
return
right
else
:
# create an InnerJoin of the last element of the alias list with
# a chain of InnerJoins of the rest of the list conditioned on
# the the last element of the join_query_list
left
=
self
.
stitchJoinDefinition
(
table_alias_list
[:
-
1
],
join_query_list
[:
-
1
],
column_map
)
condition
=
join_query_list
[
-
1
]
return
InnerJoin
(
left
,
right
,
condition
)
@
profiler_decorator
def
buildSQLExpression
(
self
,
sql_catalog
,
column_map
,
only_group_columns
,
group
):
"""
...
...
@@ -170,20 +214,26 @@ class RelatedKey(SearchKey):
table_alias_list
=
[(
getTableAlias
(
related_table
,
group
=
getRelatedKeyGroup
(
index
,
group
)),
related_table
)
for
(
index
,
related_table
)
in
enumerate
(
related_table_list
)]
# table alias for destination table
table_alias_list
.
append
((
getTableAlias
(
destination_table
,
group
=
group
),
destination_table
))
table_alias_list
.
append
((
getTableAlias
(
destination_table
,
group
=
group
),
destination_table
))
# map aliases to use in ZSQLMethod.
table_alias_dict
=
dict
((
'table_%s'
%
(
index
,
),
table_alias
[
0
])
for
(
index
,
table_alias
)
in
enumerate
(
table_alias_list
))
table_alias_dict
=
dict
((
'table_%s'
%
(
index
,
),
table_alias
)
for
(
index
,
(
table_alias
,
table_name
))
in
enumerate
(
table_alias_list
))
assert
len
(
table_alias_list
)
==
len
(
table_alias_dict
)
query_table
=
column_map
.
getCatalogTableAlias
()
rendered_related_key
=
related_key
(
query_table
=
column_map
.
getCatalogTableAlias
(),
query_table
=
query_table
,
RELATED_QUERY_SEPARATOR
=
RELATED_QUERY_SEPARATOR
,
src__
=
1
,
**
table_alias_dict
)
join_condition_list
=
rendered_related_key
.
split
(
RELATED_QUERY_SEPARATOR
)
# Important:
#
Former catalog separated join condition from
related query.
#
Previously the catalog separated join condition from the
related query.
# Example:
# ComplexQuery(Query(title="foo"),
# Query(subordination_title="bar")
...
...
@@ -199,19 +249,53 @@ class RelatedKey(SearchKey):
# This was done on purpose, because doing otherwise gives very poor
# performances (on a simple data set, similar query can take *minutes* to
# execute - as of MySQL 5.x).
# Doing the same way as the former catalog is required for backward
# compatibility, until a decent alternative is found (like spliting the
# "OR" expression into ensemblist operations at query level).
# Note that doing this has a side effect on result list, as objects
# lacking a relation will never appear in the result.
#
# Because of this, we never return an SQLExpression here, as it
# would mix join definition with column condition in the body of
# the WHERE clause. Instead we explicitly define a Join to the
# catalog. The ColumnMap defines whether this is an Inner Join or
# a Left Outer Join. Notice that if an Inner Join is decided,
# objects lacking a relationship will never appear in the result.
if
len
(
join_condition_list
)
==
len
(
table_alias_list
):
# Good! we got a compatible method that splits the join
# conditions according to the related tables.
#
# Add a join on this related key, based on the chain of
# inner-joins of the related key tables.
query_table_join_condition
=
join_condition_list
.
pop
()
right_side
=
self
.
stitchJoinDefinition
(
table_alias_list
,
join_condition_list
,
column_map
)
column_map
.
addRelatedKeyJoin
(
self
.
column
,
right_side
=
right_side
,
condition
=
query_table_join_condition
)
else
:
# Method did not render the related key condition with the
# appropriate separators so we could split it
# XXX: Can we try to parse rendered_related_key to select which
# conditions go with each table? Maybe we could still use
# explicit joins this way...
msg
=
RELATED_KEY_MISMATCH_MESSAGE
%
(
self
.
related_key_id
,
self
.
column
,
table_alias_list
,
rendered_related_key
)
if
BACKWARD_COMPATIBILITY
:
# XXX: Calling a private-ish method on column_map.
# This should never happen. It should be removed as soon as an
# alternative exists.
column_map
.
_addJoinQuery
(
SQLQuery
(
rendered_related_key
))
return
None
# BBB: remove this branch of the condition, and the above
# constant, when all zsql_methods have been adapted to return
# the join queries properly separated by the
# RELATED_QUERY_SEPARATOR.
# The rendered related key doesn't have the separators for each
# joined table, so we revert to doing implicit inner joins:
log
.
warning
(
msg
+
"
\
n
\
n
Adding an Implicit Join Condition..."
)
column_map
.
_addJoinQueryForColumn
(
self
.
column
,
SQLQuery
(
rendered_related_key
))
else
:
return
SQLExpression
(
self
,
where_expression
=
rendered_related_key
)
raise
RuntimeError
(
msg
)
return
None
verifyClass
(
IRelatedKey
,
RelatedKey
)
product/ZSQLCatalog/TableDefinition.py
0 → 100644
View file @
9cdf7a81
##############################################################################
#
# Copyright (c) 2010 Nexedi SA and Contributors. All Rights Reserved.
# Leonardo Rochael Almeida <leonardo@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
# TODO:
# * collapse of parentheses around chains of inner-joins
# * indentation on rendering
SQL_LIST_SEPARATOR
=
', '
SQL_SELECT_ALIAS_FORMAT
=
'%s AS `%s`'
from
Products.ZSQLCatalog.Query.SQLQuery
import
SQLQuery
def
escapeTable
(
table
):
return
"`%s`"
%
table
.
replace
(
'`'
,
r'\
`
')
class TableDefinition(object):
"""Base class for all TableDefinition objects. Used for
typechecking (which should become interface checking later) and
for dumping common code (if there is any).
TableDefinition objects describe the table aliasing and joining on
the "FROM" expression of an SQL query. It is supposed to be
decoded by an SQLExpressionObject into a string.
"""
def checkTableAliases(self, current_aliases=None):
"""Check that aliases defined in this table definition don'
t
try
to
alias
different
tables
to
the
same
name
.
Add
all
aliases
defined
to
the
current_aliases
mapping
if
it
is
passed
in
.
"""
if current_aliases is None:
current_aliases = {}
self._checkTableAliases(current_aliases)
def _checkTableAliases(self, current_aliases):
raise NotImplementedError('should be implemented by subclasses')
def render(self):
raise NotImplementedError('should be implemented by subclasses')
def getJoinConditionQueryList(self):
"""
Return
a
list
of
SQLQuery
objects
containing
all
conditions
used
in
this
table
definition
.
This
is
a
deprecated
method
that
is
here
only
to
accomodate
the
fact
that
not
all
RelatedKey
methods
have
been
migrated
.
"""
query_list = []
self._extendJoinConditionQueryList(query_list)
return query_list
def _extendJoinConditionQueryList(self, query_list):
raise NotImplementedError('should be implemented by subclasses')
def getSuperSet(self, other):
"""
Checks
if
this
TableDefinition
is
a
subset
of
the
other
table
definition
or
vice
-
versa
.
Return
which
one
is
the
superset
.
Returns
whichever
is
the
superset
of
the
other
or
None
"""
raise NotImplementedError('should be implemented by subclasses')
class PlaceHolderTableDefinition(TableDefinition):
"""
Table
Definition
that
simply
holds
an
inner
table
definition
and
delegates
to
it
the
rendering
.
This
object
can
be
used
when
you
need
to
change
a
table
definition
in
the
future
,
(
like
replacing
a
simple
table
aliasing
with
an
inner
join
)
but
don
't know who is going to be holding a reference to the
original table definition to replace it.
"""
def __init__(self, table_definition=None):
self.table_definition = table_definition
replace = __init__
def __repr__(self):
return '
<%
s
for
%
r
>
' % (self.__class__.__name__, self.table_definition)
def _checkTableAliases(self, current_aliases):
assert self.table_definition is not None, "table definition wasn'
t
set
"
return self.table_definition._checkTableAliases(current_aliases)
def render(self):
assert self.table_definition is not None, "
table
definition
wasn
't set"
return self.table_definition.render()
def _extendJoinConditionQueryList(self, query_list):
# XXX _extendJoinConditionQueryList
#assert self.table_definition is not None, "table definition wasn'
t
set
"
if self.table_definition is not None:
return self.table_definition._extendJoinConditionQueryList(query_list)
def getSuperSet(self, other):
assert self.table_definition is not None, "
table
definition
wasn
't set"
return self.table_definition.getSuperSet(other)
class TableAlias(TableDefinition):
"""Definition of a table alias as a FROM expression"""
def __init__(self, table, alias=None):
self.table = table
self.alias = alias or table
def _checkTableAliases(self, current_aliases):
#table_name = current_aliases.setdefault(self.alias, self.table)
table_name = current_aliases.get(self.alias)
if table_name is None:
current_aliases[self.alias] = self.table
return
if table_name != self.table:
message = ("Attempted to alias both %r and %r to %r" %
(table_name, self.table, self.alias,))
else:
message = ("Attempted to alias %r to %r more than once" %
(self.table, self.alias,))
raise ValueError(message)
def render(self):
"""Render this table definition into an actual FROM expression"""
return SQL_SELECT_ALIAS_FORMAT % (self.table, self.alias)
def __repr__(self):
return '
<%
s
%
r
AS
%
r
>
' % (self.__class__.__name__, self.table, self.alias)
def _extendJoinConditionQueryList(self, query_list):
pass
def __eq__(self, other):
return (isinstance(other, TableAlias) and
self.table == other.table and
self.alias == other.alias)
def getSuperSet(self, other):
"""A TableAlias is a subset of another table Alias if either:
- the other is an equivalent TableAlias
- the other is an InnerJoin where the left-side is an equivalent TableAlias
"""
if isinstance(other, TableAlias) and self == other:
# we'
re
just
like
the
other
guy
,
we
could
return
self
or
other
return
self
# delegate the rest of the job to InnerJoin
return
other
.
getSuperSet
(
self
)
JOIN_FORMAT
=
"""
(
%(left)s
%(join)s
%(right)s
ON
%(condition)s
)
"""
.
strip
()
class
Join
(
TableDefinition
):
JOIN_TYPE
=
None
def
__init__
(
self
,
left_tabledef
,
right_tabledef
,
condition
):
assert
self
.
JOIN_TYPE
,
(
'Join must be subclassed and self.JOIN_TYPE '
'must be defined.'
)
assert
isinstance
(
left_tabledef
,
(
TableDefinition
,
None
.
__class__
))
assert
isinstance
(
right_tabledef
,
(
TableDefinition
,
None
.
__class__
))
self
.
left_tabledef
=
left_tabledef
self
.
right_tabledef
=
right_tabledef
# perhaps assert condition is an SQLExpression?
self
.
condition
=
condition
def
render
(
self
):
"""Render the join as an actual FROM expression, delegating
the rendering of each table to its own object.
"""
assert
None
not
in
(
self
.
left_tabledef
,
self
.
right_tabledef
,
self
.
condition
)
return
JOIN_FORMAT
%
dict
(
left
=
self
.
left_tabledef
.
render
(),
right
=
self
.
right_tabledef
.
render
(),
join
=
self
.
JOIN_TYPE
,
condition
=
self
.
condition
)
def
_checkTableAliases
(
self
,
current_aliases
):
self
.
left_tabledef
.
_checkTableAliases
(
current_aliases
)
self
.
right_tabledef
.
_checkTableAliases
(
current_aliases
)
def
__repr__
(
self
):
return
'<%s of %r and %r on %r>'
%
(
self
.
__class__
.
__name__
,
self
.
left_tabledef
,
self
.
right_tabledef
,
self
.
condition
)
def
_extendJoinConditionQueryList
(
self
,
query_list
):
self
.
left_tabledef
.
_extendJoinConditionQueryList
(
query_list
)
self
.
right_tabledef
.
_extendJoinConditionQueryList
(
query_list
)
query_list
.
append
(
SQLQuery
(
self
.
condition
))
def
getSuperSet
(
self
,
other
):
return
None
class
InnerJoin
(
Join
):
"""Definition of an inner-join as a FROM expression"""
JOIN_TYPE
=
"INNER JOIN"
def
getSuperSet
(
self
,
other
):
"""This InnerJoin is a superset of another TableDefinition if either:
- other is a TableAlias (or None) equal to our
left_side. I.e. "other" is at the end of it's inner-join chain.
- other is an InnerJoin, and it's left-side is equal to our
left-side (both TableAliases or None), and our right-side is a
super-set of it's right-side.
"""
if
self
.
left_tabledef
==
other
:
# other and left-side are both None or matching TableAliases
return
self
if
(
isinstance
(
other
,
InnerJoin
)
and
self
.
left_tabledef
==
other
.
left_tabledef
):
# our left-sides match. If one of our right sides is a superset of the
# other right side, then we found the superset
sub_superset
=
self
.
right_tabledef
.
getSuperSet
(
other
.
right_tabledef
)
if
sub_superset
is
self
.
right_tabledef
:
return
self
elif
sub_superset
is
other
.
right_tabledef
:
return
other
return
None
return
None
class
LeftJoin
(
InnerJoin
):
"""Definition of a left-join as a FROM expression"""
JOIN_TYPE
=
"LEFT JOIN"
def
_extendJoinConditionQueryList
(
self
,
query_list
):
""" The condition from a left-join cannot be meaningfully
extracted to be used in an implicit Inner Join, as is done when a
query contains a related key that is not formatted to separate the
join conditions for each related table."""
raise
RuntimeError
(
"Attempted to collapse table definition for implicit "
"inner join, but this table definition contains a Left "
"Join: %r"
%
self
)
class
LegacyTableDefinition
(
TableDefinition
):
"""Table Definition used when a from_expression is passed explicitly.
Mostly used for manual left-join definitions. Deprecated
"""
def
__init__
(
self
,
from_expression
,
table_alias_map
):
self
.
from_expression
=
from_expression
self
.
table_alias_map
=
table_alias_map
def
render
(
self
):
from_expression_dict
=
self
.
from_expression
table_alias_map
=
self
.
table_alias_map
from_expression
=
SQL_LIST_SEPARATOR
.
join
(
from_expression_dict
.
get
(
alias
,
'`%s` AS `%s`'
%
(
table
,
alias
))
for
alias
,
table
in
table_alias_map
.
iteritems
())
return
from_expression
product/ZSQLCatalog/interfaces/column_map.py
View file @
9cdf7a81
...
...
@@ -284,18 +284,3 @@ class IColumnMap(Interface):
Return a copy of the table alias list for tables requiring a join with
catalog table.
"""
def
getStraightJoinTableList
():
"""
Returns the list of tables used this search and which
need to be joined with the main table using explicit
indices.
"""
def
getLeftJoinTableList
():
"""
Returns the list of tables used this search and which
need to be LEFT joined with the main table using explicit
indices.
"""
product/ZSQLCatalog/tests/testSQLCatalog.py
View file @
9cdf7a81
...
...
@@ -202,6 +202,7 @@ class DummyCatalog(SQLCatalog):
assert
'query_table'
in
kw
assert
'table_0'
in
kw
assert
'table_1'
in
kw
assert
'AND'
in
kw
.
pop
(
'RELATED_QUERY_SEPARATOR'
)
assert
len
(
kw
)
==
4
return
'%(table_0)s.uid = %(query_table)s.uid AND %(table_0)s.other_uid = %(table_1)s'
%
kw
...
...
@@ -629,7 +630,7 @@ class TestSQLCatalog(unittest.TestCase):
select_dict
=
sql_expression
.
getSelectDict
()
self
.
assertTrue
(
'ambiguous_mapping'
in
select_dict
,
select_dict
)
self
.
assertTrue
(
'bar'
in
select_dict
[
'ambiguous_mapping'
],
select_dict
[
'ambiguous_mapping'
])
# Doted alias: table name must get stripped. This is required to have an
# Dot
t
ed alias: table name must get stripped. This is required to have an
# upgrade path from old ZSQLCatalog versions where pre-mapped columns were
# used in their select_expression. This must only happen in the
# "{column: None}" form, as otherwise it's the user explicitely asking for
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment