DomainTool: Unite inner-join and left-join predicate lookups

Improvement: As proposed by Jean-Paul when this extension was added (2010),
allows mixing left- and inner-join lookups.
Improvement: Now it is not required to specify tested_base_category_list
to enable inner joins: they will automatically be used whenever a base
category set on context matches the preference. If all categories set on
context match the preference, then no left join will be used at all.

product/ERP5/Tool/DomainTool.py

@@ -33,7 +33,7 @@ from Products.ERP5Type.Globals import InitializeClass, DTMLFile
 from Products.ERP5Type import Permissions
 from Products.ERP5 import _dtmldir
 from Products.ERP5Type.Tool.BaseTool import BaseTool
-from Products.ZSQLCatalog.SQLCatalog import SQLQuery, Query, SimpleQuery, ComplexQuery
+from Products.ZSQLCatalog.SQLCatalog import SimpleQuery, ComplexQuery
 from zLOG import LOG
 from DateTime import DateTime
 
@@ -156,58 +156,62 @@ class DomainTool(BaseTool):
           for tested_base_category in tested_base_category_list:
             if portal_categories.get(tested_base_category) is None:
               raise ValueError('Unknown base category: %r' % (tested_base_category, ))
-            extend(getter(tested_base_category, base=1))
-        preferred_predicate_category_list = portal.portal_preferences.getPreferredPredicateCategoryList()
-
-        if (preferred_predicate_category_list and
-            tested_base_category_list is not None and
-            set(preferred_predicate_category_list).issuperset(tested_base_category_list)):
-          # New behavior is enabled only if preferred predicate category is
-          # defined and tested_base_category_list is passed.
-          predicate_category_query_list = []
-          predicate_category_table_name_list = []
-          category_dict = {}
-          for relative_url in category_list:
-            category_value = portal_categories.getCategoryValue(relative_url)
-            base_category_id = portal_categories.getBaseCategoryId(relative_url)
-            base_category_value = portal_categories.getCategoryValue(base_category_id)
-            if not base_category_value in category_dict:
-              category_dict[base_category_value] = []
-            category_dict[base_category_value].append(category_value)
-
-          for base_category_value, category_value_list in category_dict.iteritems():
-            if base_category_value.getId() in preferred_predicate_category_list:
-              table_index = len(predicate_category_query_list)
-              predicate_category_table_name = 'predicate_category_for_domain_tool_%s' % table_index
-              table_alias_list = [('predicate_category', predicate_category_table_name)]
-              predicate_category_query_list.append(
-                  ComplexQuery(
-                      Query(predicate_category_base_category_uid=base_category_value.getUid(), table_alias_list=table_alias_list),
-                      Query(predicate_category_category_strict_membership=1, table_alias_list=table_alias_list),
-                      ComplexQuery(
-                          Query(predicate_category_category_uid=[category_value.getUid() for category_value in category_value_list], table_alias_list=table_alias_list),
-                          Query(predicate_category_category_uid='NULL', table_alias_list=table_alias_list),
-                          logical_operator='OR'),
-                      logical_operator='AND'))
-
-          if not predicate_category_query_list:
-            # Prevent matching everything
-            predicate_category_query_list.append(Query(predicate_category_base_category_uid=0))
-
-          predicate_category_query = ComplexQuery(
-              logical_operator='AND',
-              *predicate_category_query_list)
-          query_list.append(predicate_category_query)
+            tested_category_list = getter(tested_base_category, base=1)
+            if tested_category_list:
+              extend(tested_category_list)
+            else:
+              # Developer requested specific base categories, but context do not
+              # declare one of these. Skipping this criterion risks matching too
+              # many predicates, breaking the system performance-wise. So let
+              # developer know there is an unexpected situation by raising.
+              raise ValueError('%r does not have any %r relation' % (
+                context.getPath(),
+                tested_base_category,
+              ))
+        left_join_list = kw.get('left_join_list', [])[:]
+        inner_join_list = kw.get('inner_join_list', [])[:]
+        if category_list:
+          preferred_predicate_category_list = portal.portal_preferences.getPreferredPredicateCategoryList([])
+          left_join_category_list = []
+          inner_join_category_list = []
+          for category in category_list:
+            if portal_categories.getBaseCategoryId(category) in preferred_predicate_category_list:
+              inner_join_category_list.append(category)
             else:
-          # Traditional behavior
-          category_expression_dict = portal_categories.buildAdvancedSQLSelector(
-                                             category_list or ['NULL'],
-                                             query_table='predicate_category',
-                                             none_sql_value=0,
+              left_join_category_list.append(category)
+          def onMissing(category):
+            # BBB: ValueError would seem more appropriate here, but original code
+            # was raising TypeError - and this is explicitely tested for.
+            raise TypeError('Unknown category: %r' % (category, ))
+          for join_category_list, join_list, predicate_has_no_condition_value in (
+            # Base category is part of preferred predicate categories, predicates
+            # which ignore it are indexed with category_uid=0.
+            (inner_join_category_list, inner_join_list, 0),
+            # Base category is not part of preferred predicate categories,
+            # predicates which ignore it get no predicate_category row inserted
+            # for it, so an SQL NULL appears, translating to None.
+            (left_join_category_list, left_join_list, None),
+          ):
+            category_parameter_kw = portal_catalog.getCategoryParameterDict(
+              join_category_list,
+              category_table='predicate_category',
+              onMissing=onMissing,
             )
-          kw['where_expression'] = SQLQuery(category_expression_dict['where_expression'])
-          kw['from_expression'] = category_expression_dict['from_expression']
-
+            for relation, uid_set in category_parameter_kw.iteritems():
+              join_list.append(relation)
+              uid_set.add(predicate_has_no_condition_value)
+            kw.update(category_parameter_kw)
+        else:
+          # No category to match against, so predicates expecting any relation
+          # would not apply, so we can exclude these.
+          # Note: this relies on a special indexation mechanism for predicate
+          # categories, which inserts a base_category_uid=0 line when indexed
+          # predicate membership_criterion_category_list is empty.
+          base_category_uid_column = 'predicate_category.base_category_uid'
+          kw[base_category_uid_column] = 0
+          inner_join_list.append(base_category_uid_column)
+        kw['left_join_list'] = left_join_list
+        kw['inner_join_list'] = inner_join_list
       if query_list:
         kw['query'] = ComplexQuery(logical_operator='AND', *query_list)
 

product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_related_predicate_category.catalog_keys.xml

-<catalog_method>
-</catalog_method>

product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_related_predicate_category.sql

-<dtml-var table_0>.uid = <dtml-var query_table>.uid

product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_related_predicate_category.xml

-<?xml version="1.0"?>
-<ZopeData>
-  <record id="1" aka="AAAAAAAAAAE=">
-    <pickle>
-      <global name="SQL" module="Products.ZSQLMethods.SQL"/>
-    </pickle>
-    <pickle>
-      <dictionary>
-        <item>
-            <key> <string>arguments_src</string> </key>
-            <value> <string>table_0\r\n
-query_table</string> </value>
-        </item>
-        <item>
-            <key> <string>connection_id</string> </key>
-            <value> <string>erp5_sql_connection</string> </value>
-        </item>
-        <item>
-            <key> <string>id</string> </key>
-            <value> <string>z_related_predicate_category</string> </value>
-        </item>
-        <item>
-            <key> <string>title</string> </key>
-            <value> <string></string> </value>
-        </item>
-      </dictionary>
-    </pickle>
-  </record>
-</ZopeData>

product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogRelatedKeyTemplateItem/related_key_list.xml

@@ -22,9 +22,6 @@
  <key>parent_relative_url | catalog/relative_url/z_related_parent</key>
  <key>parent_string_index | catalog/string_index/z_related_parent</key>
  <key>parent_title | catalog/title/z_related_parent</key>
- <key>predicate_category_base_category_uid | predicate_category/base_category_uid/z_related_predicate_category</key>
- <key>predicate_category_category_strict_membership | predicate_category/category_strict_membership/z_related_predicate_category</key>
- <key>predicate_category_category_uid | predicate_category/category_uid/z_related_predicate_category</key>
  <key>predicate_uid | predicate/uid/z_related_predicate</key>
  <key>related_resource_from_use_category_uid | category,category/category_uid/z_related_resource_from_use</key>
  <key>resourceType | catalog/portal_type/z_related_resource_uid_from_stock</key>

product/ERP5/bootstrap/erp5_mysql_innodb_catalog/bt/template_catalog_related_key_list

@@ -53,9 +53,6 @@ parent_portal_type | catalog/portal_type/z_related_parent
 translated_causality_state | translation/translated_message/z_related_translated_causality_state
 translated_causality_state_title | translation/translated_message/z_related_translated_causality_state_title
 metric_type_uid | measure/metric_type_uid/z_related_metric_type
-predicate_category_base_category_uid | predicate_category/base_category_uid/z_related_predicate_category
-predicate_category_category_strict_membership | predicate_category/category_strict_membership/z_related_predicate_category
-predicate_category_category_uid | predicate_category/category_uid/z_related_predicate_category
 child_aggregate_relative_url | catalog,category,catalog/relative_url/z_related_child_aggregate
 children_reference | catalog/reference/z_related_children
 related_resource_from_use_category_uid | category,category/category_uid/z_related_resource_from_use

product/ERP5/tests/testDomainTool.py

@@ -436,6 +436,8 @@ class TestDomainTool(TestPredicateMixIn):
     self.tic()
     # resource is not in preferred predicate category list, so only inner join is used
     assertUsesLeftJoinAndPredicateItemsMatchingOrderLineEqual(False, [supply1_line1], tested_base_category_list=['source_section', 'destination_section', 'price_currency', 'resource'])
+    # we now cover all categories defined on order_line, so it uses inner-join only even if tested_base_category_list is not specified.
+    assertUsesLeftJoinAndPredicateItemsMatchingOrderLineEqual(False, [supply1_line1])
 
     # unknown base category ids cause an exception, so typos are detected
     self.assertRaises(
@@ -445,6 +447,14 @@ class TestDomainTool(TestPredicateMixIn):
       portal_type='Sale Supply Line',
       tested_base_category_list=['BOOO'],
     )
+    # known Base Categories but for which context has no relation also raise.
+    self.assertRaises(
+      ValueError,
+      searchPredicateList,
+      context=order_line,
+      portal_type='Sale Supply Line',
+      tested_base_category_list=['colour'],
+    )
 
   def test_searchPredicateInvalidCategories(self):
     predicate = self.portal.sale_supply_module.newContent(

[Jérome Perrin]

@vpelletier @tiwariayush when exporting erp5_mysql_innodb I have error:

Error Type: AttributeError
Error Value: Could not find object 'portal_catalog/erp5_mysql_innodb/z_related_predicate_category' during business template processing.

My understanding is that it's because erp5_mysql_innodb/z_related_predicate_category is still listed in bt/template_catalog_method_id_list

[Vincent Pelletier]

Thanks, fixed in f19a3c76 .