CMFCategory: add support for ZODB indexing of related documents

product/CMFCategory/CategoryTool.py

@@ -31,6 +31,7 @@
 ERP portal_categories tool.
 """
 from collections import deque
+from BTrees.OOBTree import OOTreeSet
 from OFS.Folder import Folder
 from Products.CMFCore.utils import UniqueObject
 from Products.ERP5Type.Globals import InitializeClass, DTMLFile
@@ -40,9 +41,11 @@ from Acquisition import aq_base, aq_inner
 from Products.ERP5Type import Permissions
 from Products.ERP5Type.Base import Base
 from Products.ERP5Type.Cache import getReadOnlyTransactionCache
+from Products.ERP5Type.TransactionalVariable import getTransactionalVariable
 from Products.CMFCategory import _dtmldir
 from Products.CMFCore.PortalFolder import ContentFilter
 from Products.CMFCategory.Renderer import Renderer
+from Products.CMFCategory.Category import Category, BaseCategory
 from OFS.Traversable import NotFound
 import types
 
@@ -55,6 +58,34 @@ _marker = object()
 class CategoryError( Exception ):
     pass
 
+
+class RelatedIndex(): # persistent.Persistent can be added
+                      # without breaking compatibility
+
+  def __repr__(self):
+    try:
+      contents = ', '.join('%s=%r' % (k, list(v))
+                           for (k, v) in self.__dict__.iteritems())
+    except Exception:
+      contents = '...'
+    return '<%s(%s) at 0x%x>' % (self.__class__.__name__, contents, id(self))
+
+  def __nonzero__(self):
+    return any(self.__dict__.itervalues())
+
+  def add(self, base, relative_url):
+    try:
+      getattr(self, base).add(relative_url)
+    except AttributeError:
+      setattr(self, base, OOTreeSet((relative_url,)))
+
+  def remove(self, base, relative_url):
+    try:
+      getattr(self, base).remove(relative_url)
+    except (AttributeError, KeyError):
+      pass
+
+
 class CategoryTool( UniqueObject, Folder, Base ):
     """
       The CategoryTool object is the placeholder for all methods
@@ -1174,7 +1205,28 @@ class CategoryTool( UniqueObject, Folder, Base ):
 
     security.declareProtected( Permissions.ModifyPortalContent, '_setCategoryList' )
     def _setCategoryList(self, context, value):
-       context.categories = tuple(value)
+      old = set(getattr(aq_base(context), 'categories', ()))
+      context.categories = value = tuple(value)
+      if context.isTempDocument():
+        return
+      value = set(value)
+      relative_url = context.getRelativeUrl()
+      for edit, value in ("remove", old - value), ("add", value - old):
+        for path in value:
+          base = self.getBaseCategoryId(path)
+          try:
+            if self[base].isRelatedLocallyIndexed():
+              path = self._removeDuplicateBaseCategoryIdInCategoryPath(base, path)
+              ob = aq_base(self.unrestrictedTraverse(path))
+              try:
+                related = ob._related_index
+              except AttributeError:
+                if edit is "remove":
+                  continue
+                related = ob._related_index = RelatedIndex()
+              getattr(related, edit)(base, relative_url)
+          except KeyError:
+            pass
 
     security.declareProtected( Permissions.AccessContentsInformation, 'getAcquiredCategoryList' )
     def getAcquiredCategoryList(self, context):
@@ -1287,50 +1339,153 @@ class CategoryTool( UniqueObject, Folder, Base ):
       portal_type = kw.get('portal_type')
 
       if isinstance(portal_type, str):
-        portal_type = [portal_type]
+        portal_type = portal_type,
 
       # Base Category may not be related, besides sub categories
-      if context.getPortalType() == 'Base Category':
-        category_list = [context.getRelativeUrl()]
+      relative_url = context.getRelativeUrl()
+      local_index_dict = {}
+      if isinstance(context, BaseCategory):
+        category_list = relative_url,
       else:
+        category_list = []
         if isinstance(base_category_list, str):
-          base_category_list = [base_category_list]
+          base_category_list = base_category_list,
         elif base_category_list is () or base_category_list is None:
           base_category_list = self.getBaseCategoryList()
-        category_list = []
         for base_category in base_category_list:
-          category_list.append("%s/%s" % (base_category, context.getRelativeUrl()))
-
-      brain_result = self.Base_zSearchRelatedObjectsByCategoryList(
-                           category_list=category_list,
+          if self[base_category].isRelatedLocallyIndexed():
+            category = base_category + '/'
+            local_index_dict[base_category] = '' \
+              if relative_url.startswith(category) else category
+          else:
+            category_list.append("%s/%s" % (base_category, relative_url))
+
+      search = self.getPortalObject().Base_zSearchRelatedObjectsByCategoryList
+      if local_index_dict:
+        # For some base categories, lookup indexes in ZODB.
+        recurse = isinstance(context, Category) and not strict_membership
+        result_dict = {}
+        def check_local():
+          r = set(getattr(related, base_category, ()))
+          r.difference_update(result_dict)
+          for r in r:
+            try:
+              ob = self.unrestrictedTraverse(r)
+              if category in aq_base(ob).categories:
+                result_dict[r] = ob
+                continue
+              # Do not add 'r' to result_dict, because 'ob' may be linked in
+              # another way.
+            except (AttributeError, KeyError):
+              result_dict[r] = None
+            related.remove(base_category, r)
+        tv = getTransactionalVariable().setdefault(
+          'CategoriesTool.getRelatedValueList', {})
+        try:
+          related = aq_base(context)._related_index
+        except AttributeError:
+          related = RelatedIndex()
+        include_self = False
+        for base_category, category in local_index_dict.iteritems():
+          if not category:
+            # Categories are member of themselves.
+            include_self = True
+            result_dict[relative_url] = context
+          category += relative_url
+          if tv.get(category, -1) < recurse:
+            # Update local index with results from catalog for backward
+            # compatibility. But no need to do it several times in the same
+            # transaction.
+            for r in search(category_list=category,
+                            portal_type=None,
+                            strict_membership=strict_membership):
+              r = r.relative_url
+              # relative_url is empty if object is deleted (but not yet
+              # unindexed). Nothing specific to do in such case because
+              # category tool won't match.
+              try:
+                ob = self.unrestrictedTraverse(r)
+                categories = aq_base(ob).categories
+              except (AttributeError, KeyError):
+                result_dict[r] = None
+                continue
+              if category in categories:
+                related.add(base_category, r)
+                result_dict[r] = ob
+              elif recurse:
+                for p in categories:
+                  if p.startswith(category + '/'):
+                    try:
+                      o = self.unrestrictedTraverse(p)
+                      p = aq_base(o)._related_index
+                    except KeyError:
+                      continue
+                    except AttributeError:
+                      p = o._related_index = RelatedIndex()
+                    result_dict[r] = ob
+                    p.add(base_category, r)
+            tv[category] = recurse
+          # Get and check all objects referenced by local index for the base
+          # category that is currently considered.
+          check_local()
+        # Modify context only if it's worth it.
+        if related and not hasattr(aq_base(context), '_related_index'):
+          context._related_index = related
+        # In case of non-strict membership search, include all objects that
+        # are linked to a subobject of context.
+        if recurse:
+          r = [context]
+          while r:
+            for ob in r.pop().objectValues():
+              r.append(ob)
+              relative_url = ob.getRelativeUrl()
+              if include_self:
+                result_dict[relative_url] = ob
+              try:
+                related = aq_base(ob)._related_index
+              except AttributeError:
+                continue
+              for base_category, category in local_index_dict.iteritems():
+                category += relative_url
+                check_local()
+        # Filter out objects that are not of requested portal type.
+        result = [ob for ob in result_dict.itervalues() if ob is not None and (
+          not portal_type or ob.getPortalType() in portal_type)]
+        # Finish with base categories that are only indexed in catalog,
+        # making sure we don't return duplicate values.
+        if category_list:
+          for r in search(category_list=category_list,
                           portal_type=portal_type,
-                           strict_membership=strict_membership)
+                          strict_membership=strict_membership):
+            if r.relative_url not in result_dict:
+              try:
+                result.append(self.unrestrictedTraverse(r.path))
+              except KeyError:
+                pass
 
+      else:
+        # Catalog-only search.
         result = []
+        for r in search(category_list=category_list,
+                        portal_type=portal_type,
+                        strict_membership=strict_membership):
+          try:
+            result.append(self.unrestrictedTraverse(r.path))
+          except KeyError:
+            pass
+
       if checked_permission is None:
-        # No permission to check
-        for b in brain_result:
-          o = b.getObject()
-          if o is not None:
-            result.append(o)
-      else:
+        return result
+
       # Check permissions on object
       if isinstance(checked_permission, str):
-          checked_permission = (checked_permission, )
+        checked_permission = checked_permission,
       checkPermission = self.portal_membership.checkPermission
-          for b in brain_result:
-            obj = b.getObject()
-            if obj is not None:
+      def check(ob):
         for permission in checked_permission:
-                if not checkPermission(permission, obj):
-                  break
-                result.append(obj)
-
-      return result
-      # XXX missing filter and **kw stuff
-      #return self.search_category(category_list=category_list,
-      #                            portal_type=spec)
-      # future implementation with brains, much more efficient
+          if checkPermission(permission, ob):
+            return True
+      return filter(check, result)
 
     security.declareProtected( Permissions.AccessContentsInformation,
                                'getRelatedPropertyList' )

product/CMFCategory/tests/testCMFCategory.py

@@ -60,6 +60,11 @@ class TestCMFCategory(ERP5TypeTestCase):
       ),
     resource = dict(
       ),
+    test0 = dict(
+      ),
+    test1 = dict(
+      contents=('a', ('ab', 'ac', ('acd',))),
+      ),
     )
 
   def getTitle(self):
@@ -104,7 +109,8 @@ class TestCMFCategory(ERP5TypeTestCase):
         acquisition_copy_value=0,
         acquisition_append_value=0,
         acquisition_mask_value=0,
-        acquisition_portal_type_list="python: []")
+        acquisition_portal_type_list="python: []",
+        related_locally_indexed=0)
       edit_kw.update(kw)
       queue = deque(((bc, edit_kw.pop('contents', ())),))
       bc.edit(**edit_kw)
@@ -129,6 +135,7 @@ class TestCMFCategory(ERP5TypeTestCase):
       ti = self.getTypesTool().getTypeInfo(portal_type)
       ti.filter_content_types = 0
       self._original_categories[portal_type] = x = ti.getTypeBaseCategoryList()
+      x += 'test0', 'test1'
       ti._setTypeBaseCategoryList(x + categories)
 
     # Make persons.
@@ -1094,6 +1101,55 @@ class TestCMFCategory(ERP5TypeTestCase):
     self.assertEqual(get(bc.id), list('bab'))
     _set(bc.id, ())
 
+  def test_relatedIndex(self):
+    category_tool = self.getCategoriesTool()
+    newOrganisation = self.getOrganisationModule().newContent
+    organisation = newOrganisation()
+    other_organisation = newOrganisation(destination_value=organisation)
+    person = self.getPersonModule().newContent(test0_value=organisation,
+                                               test1='a/ac/acd')
+    self.tic()
+    get = organisation.getTest0RelatedValueList
+    a = category_tool.test1.a
+    def check():
+      self.assertEqual([person, other_organisation],
+        category_tool.getRelatedValueList(organisation))
+      self.assertEqual([person], get())
+      self.assertEqual([person], get(portal_type='Person'))
+      self.assertEqual([], get(portal_type='Organisation'))
+      self.assertEqual([person], a.getTest1RelatedValueList(
+        portal_type='Person'))
+      self.assertEqual([a], a.getTest1RelatedValueList(
+        strict_membership=True))
+      self.assertEqual([person], a.ac.acd.getTest1RelatedValueList(
+        portal_type='Person', strict_membership=True))
+    category_tool.test0._setRelatedLocallyIndexed(True)
+    category_tool.test1._setRelatedLocallyIndexed(True)
+    check()
+    related_list = sorted(a.getTest1RelatedList())
+    self.assertTrue(person.getRelativeUrl() in related_list)
+    self.assertEqual(related_list, sorted(x.getRelativeUrl()
+      for x in self.portal.portal_catalog(test1_uid=a.getUid())))
+    related = organisation._related_index
+    self.assertTrue(related)
+    self.assertEqual([person.getRelativeUrl()], list(related.test0))
+    person.unindexObject()
+    self.tic()
+    category_tool.test0._setRelatedLocallyIndexed(False)
+    self.assertEqual([], get())
+    category_tool.test0._setRelatedLocallyIndexed(True)
+    check()
+    person.categories = tuple(x for x in person.categories
+                                if not x.startswith('test0/'))
+    self.assertEqual([], get())
+    self.assertFalse(related)
+    self.assertEqual([], list(related.test0))
+    related = a.ac.acd._related_index.test1
+    self.assertEqual(list(related), [person.getRelativeUrl()])
+    person._setTest1Value(a)
+    self.assertEqual(list(related), [])
+
+
 def test_suite():
   suite = unittest.TestSuite()
   suite.addTest(unittest.makeSuite(TestCMFCategory))

product/ERP5/Document/BusinessTemplate.py

@@ -544,7 +544,8 @@ class BaseTemplateItem(Implicit, Persistent):
     klass = obj.__class__
     classname = klass.__name__
 
-    attr_set = set(('_dav_writelocks', '_filepath', '_owner', 'last_id', 'uid',
+    attr_set = set(('_dav_writelocks', '_filepath', '_owner', '_related_index',
+                    'last_id', 'uid',
                     '__ac_local_roles__', '__ac_local_roles_group_id_dict__'))
     if export:
       if not keep_workflow_history:

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/BaseCategory_validateRelatedLocallyIndexed.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>Script_magic</string> </key>
+            <value> <int>3</int> </value>
+        </item>
+        <item>
+            <key> <string>_bind_names</string> </key>
+            <value>
+              <object>
+                <klass>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                </klass>
+                <tuple/>
+                <state>
+                  <dictionary>
+                    <item>
+                        <key> <string>_asgns</string> </key>
+                        <value>
+                          <dictionary>
+                            <item>
+                                <key> <string>name_container</string> </key>
+                                <value> <string>container</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_context</string> </key>
+                                <value> <string>context</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_m_self</string> </key>
+                                <value> <string>script</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_subpath</string> </key>
+                                <value> <string>traverse_subpath</string> </value>
+                            </item>
+                          </dictionary>
+                        </value>
+                    </item>
+                  </dictionary>
+                </state>
+              </object>
+            </value>
+        </item>
+        <item>
+            <key> <string>_body</string> </key>
+            <value> <string>return not (value and (\n
+  request.other[\'field_my_acquisition_object_id_list\'] or\n
+  request.other[\'field_my_acquisition_base_category_list\']))\n
+</string> </value>
+        </item>
+        <item>
+            <key> <string>_params</string> </key>
+            <value> <string>value, request</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>BaseCategory_validateRelatedLocallyIndexed</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/BaseCategory_view.xml

@@ -105,6 +105,7 @@
                     <key> <string>right</string> </key>
                     <value>
                       <list>
+                        <string>my_related_locally_indexed</string>
                         <string>my_acquisition_copy_value</string>
                         <string>my_acquisition_mask_value</string>
                         <string>my_acquisition_append_value</string>

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/BaseCategory_view/my_related_locally_indexed.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>delegated_list</string> </key>
+            <value>
+              <list>
+                <string>external_validator</string>
+                <string>title</string>
+              </list>
+            </value>
+        </item>
+        <item>
+            <key> <string>delegated_message_list</string> </key>
+            <value>
+              <list>
+                <string>external_validator_failed</string>
+              </list>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>my_related_locally_indexed</string> </value>
+        </item>
+        <item>
+            <key> <string>message_values</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>external_validator_failed</string> </key>
+                    <value> <string>Local index is incompatible with category acquision.</string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+        <item>
+            <key> <string>overrides</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>field_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+                <item>
+                    <key> <string>form_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+                <item>
+                    <key> <string>target</string> </key>
+                    <value> <string></string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+        <item>
+            <key> <string>tales</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>field_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+                <item>
+                    <key> <string>form_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+                <item>
+                    <key> <string>target</string> </key>
+                    <value> <string></string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+        <item>
+            <key> <string>values</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>external_validator</string> </key>
+                    <value>
+                      <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+                    </value>
+                </item>
+                <item>
+                    <key> <string>field_id</string> </key>
+                    <value> <string>my_checkbox</string> </value>
+                </item>
+                <item>
+                    <key> <string>form_id</string> </key>
+                    <value> <string>Base_viewFieldLibrary</string> </value>
+                </item>
+                <item>
+                    <key> <string>target</string> </key>
+                    <value> <string>Click to edit the target</string> </value>
+                </item>
+                <item>
+                    <key> <string>title</string> </key>
+                    <value> <string>Index Related Documents Locally</string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="Method" module="Products.Formulator.MethodField"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>method_name</string> </key>
+            <value> <string>BaseCategory_validateRelatedLocallyIndexed</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Base_zSearchRelatedObjectsByCategoryList.xml

@@ -58,15 +58,11 @@ order_by_expression</string> </value>
             <key> <string>src</string> </key>
             <value> <string encoding="cdata"><![CDATA[
 
-SELECT DISTINCT catalog.uid, catalog.path, portal_type\n
+SELECT DISTINCT catalog.uid, path, relative_url, portal_type\n
 FROM catalog, category\n
 WHERE catalog.uid = category.uid\n
   <dtml-if portal_type>\n
-    AND\n
-    (<dtml-in portal_type>\n
-      <dtml-unless sequence-start> OR </dtml-unless>\n
-        catalog.portal_type=\'<dtml-var sequence-item>\'\n
-    </dtml-in>)\n
+    AND <dtml-sqltest portal_type type="string" multiple>\n
   </dtml-if>\n
   AND (<dtml-var "portal_categories.buildSQLSelector(category_list)">)\n
   <dtml-if strict_membership>\n

product/ERP5/bootstrap/erp5_property_sheets/PropertySheetTemplateItem/portal_property_sheets/BaseCategory/related_locally_indexed_property.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Standard Property" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_local_properties</string> </key>
+            <value>
+              <tuple>
+                <dictionary>
+                  <item>
+                      <key> <string>id</string> </key>
+                      <value> <string>mode</string> </value>
+                  </item>
+                  <item>
+                      <key> <string>type</string> </key>
+                      <value> <string>string</string> </value>
+                  </item>
+                </dictionary>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>categories</string> </key>
+            <value>
+              <tuple>
+                <string>elementary_type/boolean</string>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value> <string>Determines if related values should be indexed on target documents (i.e. in ZODB) in addition to catalog.\n
+This is incompatible with category acquisition.</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>related_locally_indexed_property</string> </value>
+        </item>
+        <item>
+            <key> <string>mode</string> </key>
+            <value> <string>w</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Standard Property</string> </value>
+        </item>
+        <item>
+            <key> <string>property_default</string> </key>
+            <value> <string>python: 0</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>