Related keys for translated relations

Implement related keys for translated relations (like `source__translated__title`) and change content translation to index categories, by also indexing types whose translation is set in Localizer.

See merge request nexedi/erp5!1292

bt5/erp5_content_translation/SkinTemplateItem/portal_skins/erp5_content_translation/CatalogTool_getContentTranslationDictList.py

@@ -7,7 +7,8 @@ result = []
 def upperCase(text):
   return convertToUpperCase(text.replace('-', '_'))
 
-content_language_list = context.Localizer.get_languages()
+localizer = portal.Localizer
+content_language_list = localizer.get_languages()
 
 for document in document_list:
   portal_type = document.getPortalType()
@@ -23,19 +24,29 @@ for document in document_list:
     if original_method is not None:
       original_text = original_method()
 
+    property_translation_domain = document.getProperty('%s_translation_domain' % property_name)
     for content_language in content_language_list:
       method_name = 'get%s' % (upperCase('%s_translated_%s' %
                                          (content_language, property_name)),)
       translated_text = None
       method = getattr(document, method_name, None)
-      if method is not None and document.getProperty('%s_translation_domain' % property_name) == 'content':
+      if method is not None and property_translation_domain == 'content':
         translated_text = method()
-      else:
+      elif property_translation_domain == 'content_translation':
         translation_method = getattr(document, 'get%s' % upperCase('translated_%s' % property_name), None)
         if original_text is not None and translation_method is not None:
           temporary_translated_text = translation_method(language=content_language)
           if original_text != temporary_translated_text:
             translated_text = temporary_translated_text
+      elif original_text:
+        temporary_translated_text = (localizer.translate(
+            domain=property_translation_domain,
+            msgid=original_text,
+            lang=content_language
+        ) or original_text).encode('utf-8')
+        if original_text != temporary_translated_text:
+          translated_text = temporary_translated_text
+
       temporary_result.append({'uid': uid,
                                'property_name': property_name,
                                'content_language': content_language,

bt5/erp5_content_translation/SkinTemplateItem/portal_skins/erp5_content_translation/ERP5Site_getPortalTypeContentTranslationMapping.py

@@ -3,10 +3,10 @@ from Products.ERP5Type.Cache import CachingMethod
 def getPortalTypeContentTranslationMapping():
   result = {}
   for type_information in context.getPortalObject().portal_types.listTypeInfo():
-    content_translation_domain_property_name_list =\
-      type_information.getContentTranslationDomainPropertyNameList()
-    if content_translation_domain_property_name_list:
-      result[type_information.getId()] = content_translation_domain_property_name_list
+    for property_name, translation_domain in type_information.getPropertyTranslationDomainDict().items():
+      domain_name = translation_domain.getDomainName()
+      if domain_name:
+        result.setdefault(type_information.getId(), []).append(property_name)
   return result
 
 getPortalTypeContentTranslationMapping = CachingMethod(

bt5/erp5_content_translation/TestTemplateItem/portal_components/test.erp5.testContentTranslation.py

@@ -113,7 +113,7 @@ class TestContentTranslation(ERP5TypeTestCase):
     # Low level columns test. This behaviour is not guaranteed. I'm not sure
     # content_translation must be a search table - jerome
     result5 = portal.portal_catalog(property_name='title')
-    self.assertEqual(len(result5), 2)
+    self.assertGreaterEqual(len(result5), 2)
     result6 = portal.portal_catalog(content_language='nob-read')
     self.assertEqual(len(result6), 2)
     result7 = portal.portal_catalog(translated_text='XXX YYY')

bt5/erp5_content_translation/TestTemplateItem/portal_components/test.erp5.testTranslatedRelatedKeys.py

+##############################################################################
+# coding: utf-8
+# Copyright (c) 2002-2020 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+##############################################################################
+
+import mock
+
+from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
+
+
+class TestTranslatedRelatedKeys(ERP5TypeTestCase):
+
+  def getBusinessTemplateList(self):
+    return (
+        'erp5_base',
+        'erp5_content_translation',
+        'erp5_l10n_fr',
+        'erp5_l10n_jp',
+    )
+
+  def afterSetUp(self):
+    # For this test we will use:
+    #  * title as "content translated properties" on Organisation
+    #  * short_title as "content translated properties" on Organisation (because it contain a _,
+    #    to make sure we are supporting properties with _)
+    #  * some "group" categories and their translations in erp5_content
+    #  * two organisations
+    self.portal.portal_types.Organisation.setTranslationDomain(
+        prop_name='title', domain='content_translation')
+    self.portal.portal_types.Organisation.setTranslationDomain(
+        prop_name='short_title', domain='content_translation')
+
+    # clear cache because ERP5Site_getPortalTypeContentTranslationMapping uses
+    # a cache.
+    self.commit()
+    self.portal.portal_caches.clearCacheFactory('erp5_content_long')
+
+    # translations for categories
+    message_catalog = self.portal.Localizer.erp5_content
+    message_catalog.add_language('fr')
+    message_catalog.add_language('ja')
+    message_catalog.gettext('Nexedi', add=True)
+    message_catalog.gettext('Another', add=True)
+    message_catalog.message_edit('Nexedi', 'fr', u'Catégorie Nexedi', '')
+    message_catalog.message_edit('Another', 'fr', u'Autre Catégorie', '')
+    message_catalog.message_edit('Nexedi', 'ja', u'ネクセディカテゴリー', '')
+    message_catalog.message_edit('Another', 'ja', u'別カテゴリー', '')
+
+    # categories (which will be translated using erp5_content)
+    group_base_category = self.portal.portal_categories.group
+    if 'nexedi' not in group_base_category.objectIds():
+      group_base_category.newContent(id='nexedi', title='Nexedi')
+    if 'another' not in group_base_category.objectIds():
+      group_base_category.newContent(id='another', title='Another')
+
+    self.organisation_nexedi = self.portal.organisation_module.newContent(
+        portal_type='Organisation',
+        title='Nexedi',
+        short_title='Nexedi SA',
+        group_value=group_base_category.nexedi,
+    )
+    self.organisation_nexedi.setFrTranslatedTitle("Nexedi")
+    self.organisation_nexedi.setFrTranslatedShortTitle("Nexedi Société Anonyme")
+    self.organisation_nexedi.setJaTranslatedTitle("ネクセディ")
+    self.organisation_nexedi.setJaTranslatedShortTitle("ネクセディ 本社")
+    self.organisation_another = self.portal.organisation_module.newContent(
+        portal_type='Organisation',
+        title='Another not translated Organisation',
+        group_value=group_base_category.another,
+    )
+    self.tic()
+
+  def test_content_translation_search_folder(self):
+    folder = self.portal.person_module.newContent(portal_type='Person')
+    career_nexedi = folder.newContent(
+        portal_type='Career', subordination_value=self.organisation_nexedi)
+    career_another = folder.newContent(
+        portal_type='Career', subordination_value=self.organisation_another)
+    self.tic()
+
+    localizer = self.portal.Localizer
+    with mock.patch.object(
+        localizer,
+        'get_selected_language',
+        return_value='ja',
+    ):
+      self.assertEqual(
+          [
+              x.getObject() for x in folder.searchFolder(
+                  subordination__translated__title='ネクセディ')
+          ], [career_nexedi])
+      self.assertEqual(
+          [
+              x.getObject() for x in folder.searchFolder(
+                  subordination__translated__short_title='ネクセディ 本社')
+          ], [career_nexedi])
+    with mock.patch.object(
+        localizer,
+        'get_selected_language',
+        return_value='fr',
+    ):
+      self.assertEqual(
+          [
+              x.getObject() for x in folder.searchFolder(
+                  subordination__translated__title='Nexedi')
+          ], [career_nexedi])
+    with mock.patch.object(
+        localizer,
+        'get_selected_language',
+        return_value='fr',
+    ):
+      self.assertEqual(
+          [
+              x.getObject() for x in folder.searchFolder(
+                  subordination__translated__short_title='Nexedi Société Anonyme'
+              )
+          ], [career_nexedi])
+
+    # if a translation exist for another language it is not used when language does
+    # not match.
+    with mock.patch.object(
+        localizer,
+        'get_selected_language',
+        return_value='fr',
+    ):
+      self.assertEqual(
+          [
+              x.getObject() for x in folder.searchFolder(
+                  subordination__translated__title='ネクセディ')
+          ], [])
+
+    # if property is not translated, the original propery can be used for searching
+    with mock.patch.object(
+        localizer,
+        'get_selected_language',
+        return_value='anything',
+    ):
+      self.assertEqual(
+          [
+              x.getObject() for x in folder.searchFolder(
+                  subordination__translated__title='Another not translated Organisation'
+              )
+          ], [career_another])
+
+    # if property is translated, but not in the selected language, the original property
+    # can be used for searching
+    with mock.patch.object(
+        localizer,
+        'get_selected_language',
+        return_value='other',
+    ):
+      self.assertEqual(
+          [
+              x.getObject() for x in folder.searchFolder(
+                  subordination__translated__title='Nexedi')
+          ], [career_nexedi])
+
+    # strict
+    with mock.patch.object(
+        localizer,
+        'get_selected_language',
+        return_value='ja',
+    ):
+      self.assertEqual(
+          [
+              x.getObject() for x in folder.searchFolder(
+                  strict__subordination__translated__title='ネクセディ')
+          ], [career_nexedi])
+
+    # sort
+    with mock.patch.object(
+        localizer,
+        'get_selected_language',
+        return_value='ja',
+    ):
+      self.assertEqual(
+          [
+              x.getObject() for x in folder.searchFolder(
+                  uid=(career_nexedi.getUid(), career_another.getUid()),
+                  sort_on=[('subordination__translated__title', 'ASC')])
+          ], [career_another, career_nexedi])
+
+      self.assertEqual(
+          [
+              x.getObject() for x in folder.searchFolder(
+                  uid=(career_nexedi.getUid(), career_another.getUid()),
+                  sort_on=[('subordination__translated__title', 'DESC')])
+          ], [career_nexedi, career_another])
+
+    # select dict
+    with mock.patch.object(
+        localizer,
+        'get_selected_language',
+        return_value='ja',
+    ):
+      self.assertEqual(
+          sorted(
+              [
+                  x.subordination__translated__title
+                  for x in folder.searchFolder(
+                      uid=(career_nexedi.getUid(), career_another.getUid()),
+                      select_dict={'subordination__translated__title': None})
+              ]),
+          [
+              'Another not translated Organisation',
+              # XXX select dict is not really good, because we also select rows
+              # for the original message (translation_language = "")
+              'Nexedi',
+              'ネクセディ',
+          ])
+
+  def test_erp5_content_search_folder(self):
+    localizer = self.portal.Localizer
+    with mock.patch.object(
+        localizer,
+        'get_selected_language',
+        return_value='ja',
+    ):
+      self.assertEqual(
+          [
+              x.getObject()
+              for x in self.portal.organisation_module.searchFolder(
+                  uid=(
+                      self.organisation_nexedi.getUid(),
+                      self.organisation_another.getUid()),
+                  group__translated__title='ネクセディカテゴリー')
+          ], [self.organisation_nexedi])
+
+    with mock.patch.object(
+        localizer,
+        'get_selected_language',
+        return_value='fr',
+    ):
+      self.assertEqual(
+          [
+              x.getObject()
+              for x in self.portal.organisation_module.searchFolder(
+                  uid=(
+                      self.organisation_nexedi.getUid(),
+                      self.organisation_another.getUid()),
+                  group__translated__title='Catégorie Nexedi')
+          ], [self.organisation_nexedi])

bt5/erp5_content_translation/TestTemplateItem/portal_components/test.erp5.testTranslatedRelatedKeys.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Test Component" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_recorded_property_dict</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+        <item>
+            <key> <string>default_reference</string> </key>
+            <value> <string>testTranslatedRelatedKeys</string> </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>test.erp5.testTranslatedRelatedKeys</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Test Component</string> </value>
+        </item>
+        <item>
+            <key> <string>sid</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_error_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_warning_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>version</string> </key>
+            <value> <string>erp5</string> </value>
+        </item>
+        <item>
+            <key> <string>workflow_history</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary/>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>component_validation_workflow</string> </key>
+                    <value>
+                      <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
+                    </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="4" aka="AAAAAAAAAAQ=">
+    <pickle>
+      <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_log</string> </key>
+            <value>
+              <list>
+                <dictionary>
+                  <item>
+                      <key> <string>action</string> </key>
+                      <value> <string>validate</string> </value>
+                  </item>
+                  <item>
+                      <key> <string>validation_state</string> </key>
+                      <value> <string>validated</string> </value>
+                  </item>
+                </dictionary>
+              </list>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_content_translation/bt/template_test_id_list

 test.erp5.testContentTranslation
+test.erp5.testTranslatedRelatedKeys
\ No newline at end of file

bt5/erp5_content_translation/bt/test_dependency_list

 erp5_full_text_mroonga_catalog
+erp5_l10n_fr
+erp5_l10n_ja
\ No newline at end of file

product/ERP5Catalog/CatalogTool.py

@@ -62,6 +62,7 @@ STRICT_METHOD_NAME = 'strict_'
 STRICT_METHOD_NAME_LEN = len(STRICT_METHOD_NAME)
 PARENT_METHOD_NAME = 'parent_'
 PARENT_METHOD_NAME_LEN = len(PARENT_METHOD_NAME)
+TRANSLATED_METHOD_NAME = '_translated_'
 RELATED_DYNAMIC_METHOD_NAME = '_related'
 # Negative as it's used as a slice end offset
 RELATED_DYNAMIC_METHOD_NAME_LEN = -len(RELATED_DYNAMIC_METHOD_NAME)
@@ -294,8 +295,19 @@ class IndexableObjectWrapper(object):
 class RelatedBaseCategory(Method):
     """A Dynamic Method to act as a related key.
     """
-    def __init__(self, id, strict_membership=0, related=0, query_table_column='uid'):
+    def __init__(
+        self,
+        id,
+        strict_membership=0,
+        related=0,
+        query_table_column='uid',
+        translated=False,
+        content_translation_property_name=None,
+    ):
       self._id = id
+      self._translated = translated
+      if translated:
+        self._id = id.split(TRANSLATED_METHOD_NAME)[0]
       if self._id == IGNORE_BASE_CATEGORY_UID:
         base_category_sql = ''
       else:
@@ -327,6 +339,21 @@ class RelatedBaseCategory(Method):
           'query_table_side': query_table_side,
           'query_table_column': query_table_column
       }
+      if translated:
+        self._template = """\
+%(base_category)s%(strict)s%%(content_translation)s.property_name = "%(content_translation_property_name)s"
+AND %(base_category)s%(strict)s%%(content_translation)s.content_language in ("%%(localizer_language)s", "")
+AND %(base_category)s%(strict)s%%(content_translation)s.uid = %%(category_table)s.%(foreign_side)s
+%%(RELATED_QUERY_SEPARATOR)s
+%%(category_table)s.%(query_table_side)s = %%(query_table)s.%(query_table_column)s""" % {
+          'base_category': base_category_sql,
+          'strict': strict,
+          'foreign_side': foreign_side,
+          'query_table_side': query_table_side,
+          'query_table_column': query_table_column,
+          'content_translation_property_name': content_translation_property_name,
+      }
+
       self._monotable_template = """\
 %(base_category)s%(strict)s%%(category_table)s.%(query_table_side)s = %%(query_table)s.%(query_table_column)s""" % {
           'base_category': base_category_sql,
@@ -349,6 +376,9 @@ class RelatedBaseCategory(Method):
         # one invocation to the next.
         format_dict['base_category_uid'] = instance.getPortalObject().portal_categories.\
           _getOb(self._id).getUid()
+      if self._translated:
+        format_dict["content_translation"] = table_1
+        format_dict["localizer_language"] = instance.getPortalObject().Localizer.get_selected_language()
       return (
         self._monotable_template if table_1 is None else self._template
       ) % format_dict
@@ -1039,13 +1069,15 @@ class CatalogTool (UniqueObject, ZCatalog, CMFCoreCatalogTool, ActiveObject):
       by looking at the category tree.
 
       Syntax:
-        [[predicate_][strict_][parent_]_]<base category id>__[related__]<column id>
+        [[predicate_][strict_][parent_]_]<base category id>__[related__][translated__]<column id>
       "predicate": Use predicate_category as relation table, otherwise category table.
       "strict": Match only strict relation members, otherwise match non-strict too.
       "parent": Search for documents whose parent have described relation, otherwise search for their immediate relations.
       <base_category_id>: The id of an existing Base Category document, or "any" to not restrict by relation type.
       "related": Search for reverse relationships, otherwise search for direct relationships.
-      <column_id>: The name of the column to compare values against.
+      "translated": Lookup for property <column_id> in content_translation table,
+      instead of looking it up as a catalog column.
+      <column_id>: The name of the column (or translated property) to compare values against.
 
       Old syntax is supported for backward-compatibility, but will not receive
       further extensions:
@@ -1062,12 +1094,16 @@ class CatalogTool (UniqueObject, ZCatalog, CMFCoreCatalogTool, ActiveObject):
         if '__' in key:
           split_key = key.split('__')
           column_id = split_key.pop()
-          if 'catalog' not in column_map.get(column_id, ()):
-            continue
           base_category_id = split_key.pop()
           related = base_category_id == 'related'
           if related:
             base_category_id = split_key.pop()
+          translated = base_category_id == 'translated'
+          if translated:
+            base_category_id = split_key.pop()
+          elif 'catalog' not in column_map.get(column_id, ()):
+            continue
+
           if split_key:
             flag_string, = split_key
             flag_list = flag_string.split('_')
@@ -1084,6 +1120,7 @@ class CatalogTool (UniqueObject, ZCatalog, CMFCoreCatalogTool, ActiveObject):
           # BBB: legacy related key format
           default_string = 'default_'
           related_string = 'related_'
+          translated = False
           prefix = key
           if prefix.startswith(default_string):
             prefix = prefix[len(default_string):]
@@ -1117,13 +1154,14 @@ class CatalogTool (UniqueObject, ZCatalog, CMFCoreCatalogTool, ActiveObject):
         related_key_list.append(
           key + ' | ' +
           ('predicate_' if flag_bitmap & DYNAMIC_RELATED_KEY_FLAG_PREDICATE else '') + 'category' +
-          ('' if is_uid else ',catalog') +
+          ('' if is_uid else (',content_translation' if translated else ',catalog')) +
           '/' +
-          column_id +
+          ('translated_text' if translated else column_id) +
           '/' + DYNAMIC_METHOD_NAME +
           (STRICT_METHOD_NAME if flag_bitmap & DYNAMIC_RELATED_KEY_FLAG_STRICT else '') +
           (PARENT_METHOD_NAME if flag_bitmap & DYNAMIC_RELATED_KEY_FLAG_PARENT else '') +
           base_category_id +
+          ((TRANSLATED_METHOD_NAME + column_id) if translated else '') +
           (RELATED_DYNAMIC_METHOD_NAME if related else '')
         )
       return related_key_list
@@ -1264,6 +1302,14 @@ class CatalogTool (UniqueObject, ZCatalog, CMFCoreCatalogTool, ActiveObject):
         if base_name.startswith(PARENT_METHOD_NAME):
           base_name = base_name[PARENT_METHOD_NAME_LEN:]
           kw['query_table_column'] = 'parent_uid'
+        if TRANSLATED_METHOD_NAME in base_name:
+          base_name, content_translation_property_name = base_name.split(TRANSLATED_METHOD_NAME, 1)
+          kw['translated'] = True
+          kw['content_translation_property_name'] = content_translation_property_name
+          if '"' in content_translation_property_name:
+            # prevent values which would generate invalid queries
+            return None
+
         method = RelatedBaseCategory(base_name, **kw)
         setattr(self.__class__, name, method)
         # This getattr has 2 purposes: