Merge support for ZODB indexing of category related documents

product/CMFCategory/CategoryTool.py

@@ -30,7 +30,8 @@
 """\
 ERP portal_categories tool.
 """
-
+from collections import deque
+from BTrees.OOBTree import OOTreeSet
 from OFS.Folder import Folder
 from Products.CMFCore.utils import UniqueObject
 from Products.ERP5Type.Globals import InitializeClass, DTMLFile
@@ -40,9 +41,11 @@ from Acquisition import aq_base, aq_inner
 from Products.ERP5Type import Permissions
 from Products.ERP5Type.Base import Base
 from Products.ERP5Type.Cache import getReadOnlyTransactionCache
+from Products.ERP5Type.TransactionalVariable import getTransactionalVariable
 from Products.CMFCategory import _dtmldir
 from Products.CMFCore.PortalFolder import ContentFilter
 from Products.CMFCategory.Renderer import Renderer
+from Products.CMFCategory.Category import Category, BaseCategory
 from OFS.Traversable import NotFound
 import types
 
@@ -55,6 +58,34 @@ _marker = object()
 class CategoryError( Exception ):
     pass
 
+
+class RelatedIndex(): # persistent.Persistent can be added
+                      # without breaking compatibility
+
+  def __repr__(self):
+    try:
+      contents = ', '.join('%s=%r' % (k, list(v))
+                           for (k, v) in self.__dict__.iteritems())
+    except Exception:
+      contents = '...'
+    return '<%s(%s) at 0x%x>' % (self.__class__.__name__, contents, id(self))
+
+  def __nonzero__(self):
+    return any(self.__dict__.itervalues())
+
+  def add(self, base, relative_url):
+    try:
+      getattr(self, base).add(relative_url)
+    except AttributeError:
+      setattr(self, base, OOTreeSet((relative_url,)))
+
+  def remove(self, base, relative_url):
+    try:
+      getattr(self, base).remove(relative_url)
+    except (AttributeError, KeyError):
+      pass
+
+
 class CategoryTool( UniqueObject, Folder, Base ):
     """
       The CategoryTool object is the placeholder for all methods
@@ -308,38 +339,34 @@ class CategoryTool( UniqueObject, Folder, Base ):
         such as site/group/a/b/c/b1/c1 where b and b1 are both children
         categories of a.
 
-        relative_url -- a single relative url of a list of
-                        relative urls
+        relative_url -- a single relative url or a list of relative urls
 
         strict       -- if set to 1, only return uids of parents, not
                         relative_url
       """
-      uid_dict = {}
+      uid_set = set()
       if isinstance(relative_url, str):
         relative_url = (relative_url,)
       for path in relative_url:
         try:
           o = self.getCategoryValue(path, base_category=base_category)
           if o is not None:
+            if base_category is None:
               my_base_category = self.getBaseCategoryId(path)
-            bo = self.get(my_base_category, None)
-            if bo is not None:
-              bo_uid = int(bo.getUid())
-              uid_dict[(int(o.uid), bo_uid, 1)] = 1 # Strict Membership
-              if o.meta_type == 'CMF Category' or o.meta_type == 'CMF Base Category':
+            else:
+              my_base_category = base_category
+            bo_uid = self[my_base_category].getUid()
+            uid_set.add((o.getUid(), bo_uid, 1)) # Strict Membership
+            if not strict:
+              while o.portal_type == 'Category':
                 # This goes up in the category tree
                 # XXX we should also go up in some other cases....
                 # ie. when some documents act as categories
-                if not strict:
-                  while o.meta_type == 'CMF Category':
                 o = o.aq_parent # We want acquisition here without aq_inner
-                    uid_dict[(int(o.uid), bo_uid, 0)] = 1 # Non Strict Membership
+                uid_set.add((o.getUid(), bo_uid, 0)) # Non Strict Membership
         except (KeyError, AttributeError):
           LOG('WARNING: CategoriesTool',0, 'Unable to find uid for %s' % path)
-      return uid_dict.keys()
-
-    security.declareProtected(Permissions.AccessContentsInformation, 'getUids')
-    getUids = getCategoryParentUidList
+      return list(uid_set) # cast to list for <dtml-in>
 
     security.declareProtected(Permissions.AccessContentsInformation, 'getCategoryChildUidList')
     def getCategoryChildUidList(self, relative_url, base_category = None, strict=0):
@@ -631,90 +658,58 @@ class CategoryTool( UniqueObject, Folder, Base ):
           portal_type = (portal_type,)
         spec = portal_type
 
-      self._cleanupCategories(context)
-
       if isinstance(category_list, str):
         category_list = (category_list, )
       elif category_list is None:
         category_list = ()
-      elif isinstance(category_list, (list, tuple)):
-        pass
-      else:
-        __traceback_info__ = (base_category_list, category_list)
-        raise TypeError('Category must be of string, tuple of string '
-                        'or list of string type.')
 
       if isinstance(base_category_list, str):
         base_category_list = (base_category_list, )
 
-      # Build the ckecked_permission filter
       if checked_permission is not None:
         checkPermission = self.portal_membership.checkPermission
-        def permissionFilter(obj):
-          if checkPermission(checked_permission, obj):
-            return 0
-          else:
-            return 1
 
-      new_category_list = []
-      default_dict = {}
-      spec_len = len(spec)
+      new_category_list = deque()
+      default_base_category_set = set()
+      default_category_set = set()
       for path in self._getCategoryList(context):
         my_base_id = self.getBaseCategoryId(path)
-        if my_base_id not in base_category_list:
-          # Keep each membership which is not in the
-          # specified list of base_category ids
-          new_category_list.append(path)
-        else:
-          keep_it = 0
-          if spec_len != 0 or (checked_permission is not None):
+        if my_base_id in base_category_list:
+          if spec or checked_permission is not None:
             obj = self.unrestrictedTraverse(path, None)
             if obj is not None:
-              if spec_len != 0:
               # If spec is (), then we should keep nothing
               # Everything will be replaced
               # If spec is not (), Only keep this if not in our spec
-                  my_type = obj.portal_type
-                  keep_it = (my_type not in spec)
-              if (not keep_it) and (checked_permission is not None):
-                keep_it = permissionFilter(obj)
-
-          if keep_it:
+              if (spec and obj.portal_type not in spec) or not (
+                  checked_permission is None or
+                  checkPermission(checked_permission, obj)):
                 new_category_list.append(path)
-          elif keep_default:
-            # We must remember the default value
-            # for each replaced category
-            if not default_dict.has_key(my_base_id):
-              default_dict[my_base_id] = path
-      # We now create a list of default category values
-      default_new_category_list = []
-      for path in default_dict.values():
-        if base or len(base_category_list) > 1:
-          if path in category_list:
-            default_new_category_list.append(path)
+                continue
+          # We must remember the default value for each replaced category
+          if keep_default and my_base_id not in default_base_category_set:
+            default_base_category_set.add(my_base_id)
+            default_category_set.add(path)
         else:
-          if path[len(base_category_list[0])+1:] in category_list:
-            default_new_category_list.append(path)
+          # Keep each membership which is not in the
+          # specified list of base_category ids
+          new_category_list.append(path)
       # Before we append new category values (except default values)
       # We must make sure however that multiple links are possible
-      default_path_found = {}
+      base = '' if base or len(base_category_list) > 1 \
+        else base_category_list[0] + '/'
       for path in category_list:
-        if not path in ('', None):
-          if base or len(base_category_list) > 1:
-            # Only keep path which are member of base_category_list
-            if self.getBaseCategoryId(path) in base_category_list:
-              if path not in default_new_category_list or default_path_found.has_key(path):
-                default_path_found[path] = 1
-                new_category_list.append(path)
+        if path not in ('', None):
+          if base:
+            path = base + path
+          elif self.getBaseCategoryId(path) not in base_category_list:
+            continue
+          if path in default_category_set:
+            default_category_set.remove(path)
+            new_category_list.appendleft(path)
           else:
-            new_path = '%s/%s' % (base_category_list[0], path)
-            if new_path not in default_new_category_list:
-              new_category_list.append(new_path)
-#       LOG("CategoryTool, setCategoryMembership", 0 ,
-#           'new_category_list: %s' % str(new_category_list))
-#       LOG("CategoryTool, setCategoryMembership", 0 ,
-#           'default_new_category_list: %s' % str(default_new_category_list))
-      self._setCategoryList(context, tuple(default_new_category_list + new_category_list))
+            new_category_list.append(path)
+      self._setCategoryList(context, new_category_list)
 
 
     security.declareProtected( Permissions.AccessContentsInformation, 'setDefaultCategoryMembership' )
@@ -739,7 +734,6 @@ class CategoryTool( UniqueObject, Folder, Base ):
                                         to filter the object on
 
       """
-      self._cleanupCategories(context)
       if isinstance(default_category, (tuple, list)):
         default_category = default_category[0]
       category_list = self.getCategoryMembershipList(context, base_category,
@@ -1182,35 +1176,28 @@ class CategoryTool( UniqueObject, Folder, Base ):
       """
       if getattr(aq_base(context), 'isCategory', 0):
         return context.isAcquiredMemberOf(category)
-      for c in self._getAcquiredCategoryList(context):
+      for c in self.getAcquiredCategoryList(context):
         if c.find(category) >= 0:
           return 1
       return 0
 
     security.declareProtected( Permissions.AccessContentsInformation, 'getCategoryList' )
     def getCategoryList(self, context):
-      self._cleanupCategories(context)
-      return self._getCategoryList(context)
-
-    security.declareProtected( Permissions.AccessContentsInformation, '_getCategoryList' )
-    def _getCategoryList(self, context):
-      if getattr(aq_base(context), 'categories', _marker) is not _marker:
-        if isinstance(context.categories, tuple):
-          result = list(context.categories)
-        elif isinstance(context.categories, list):
-          result = context.categories
-        else:
-          result = []
+      result = getattr(aq_base(context), 'categories', None)
+      if result is not None:
+        result = list(result)
       elif isinstance(context, dict):
-        result = list(context.get('categories', []))
+        return list(context.get('categories', ()))
       else:
         result = []
       if getattr(context, 'isCategory', 0):
         category_url = context.getRelativeUrl()
         if category_url not in result:
-          result.append(context.getRelativeUrl()) # Pure category is member of itself
+          result.append(category_url) # Pure category is member of itself
       return result
 
+    _getCategoryList = getCategoryList
+
     security.declareProtected( Permissions.ModifyPortalContent, 'setCategoryList' )
     def setCategoryList(self, context, value):
        self._setCategoryList(context, value)
@@ -1218,45 +1205,39 @@ class CategoryTool( UniqueObject, Folder, Base ):
 
     security.declareProtected( Permissions.ModifyPortalContent, '_setCategoryList' )
     def _setCategoryList(self, context, value):
-       context.categories = tuple(value)
+      old = set(getattr(aq_base(context), 'categories', ()))
+      context.categories = value = tuple(value)
+      if context.isTempDocument():
+        return
+      value = set(value)
+      relative_url = context.getRelativeUrl()
+      for edit, value in ("remove", old - value), ("add", value - old):
+        for path in value:
+          base = self.getBaseCategoryId(path)
+          try:
+            if self[base].isRelatedLocallyIndexed():
+              path = self._removeDuplicateBaseCategoryIdInCategoryPath(base, path)
+              ob = aq_base(self.unrestrictedTraverse(path))
+              try:
+                related = ob._related_index
+              except AttributeError:
+                if edit is "remove":
+                  continue
+                related = ob._related_index = RelatedIndex()
+              getattr(related, edit)(base, relative_url)
+          except KeyError:
+            pass
 
     security.declareProtected( Permissions.AccessContentsInformation, 'getAcquiredCategoryList' )
     def getAcquiredCategoryList(self, context):
-      """
-        Returns the list of acquired categories
-      """
-      self._cleanupCategories(context)
-      return self._getAcquiredCategoryList(context)
-
-    security.declareProtected( Permissions.AccessContentsInformation, '_getAcquiredCategoryList' )
-    def _getAcquiredCategoryList(self, context):
       result = self.getAcquiredCategoryMembershipList(context,
                      base_category = self.getBaseCategoryList(context=context))
-      append = result.append
-      non_acquired = self._getCategoryList(context)
-      for c in non_acquired:
+      for c in self._getCategoryList(context):
         # Make sure all local categories are considered
         if c not in result:
-          append(c)
-      if getattr(context, 'isCategory', 0):
-        append(context.getRelativeUrl()) # Pure category is member of itself
+          result.append(c)
       return result
 
-    security.declareProtected( Permissions.ModifyPortalContent, '_cleanupCategories' )
-    def _cleanupCategories(self, context):
-      # Make sure _cleanupCategories does not modify objects each time it is called
-      # or we get many conflicts
-      requires_update = 0
-      categories = []
-      append = categories.append
-      if getattr(context, 'categories', _marker) is not _marker:
-        for cat in self._getCategoryList(context):
-          if isinstance(cat, str):
-            append(cat)
-          else:
-            requires_update = 1
-      if requires_update: self.setCategoryList(context, tuple(categories))
-
     # Catalog related methods
     def updateRelatedCategory(self, category, previous_category_url, new_category_url):
       new_category = re.sub('^%s$' %
@@ -1349,7 +1330,6 @@ class CategoryTool( UniqueObject, Folder, Base ):
     security.declareProtected( Permissions.AccessContentsInformation,
                                'getRelatedValueList' )
     def getRelatedValueList(self, context, base_category_list=None,
-                            spec=(), filter=None, base=1, 
                             checked_permission=None, **kw):
       """
         This methods returns the list of objects related to the context
@@ -1359,65 +1339,158 @@ class CategoryTool( UniqueObject, Folder, Base ):
       portal_type = kw.get('portal_type')
 
       if isinstance(portal_type, str):
-        portal_type = [portal_type]
-      if spec is (): 
-        # We do not want to care about spec
-        spec = None
+        portal_type = portal_type,
 
       # Base Category may not be related, besides sub categories
-      if context.getPortalType() == 'Base Category':
-        category_list = [context.getRelativeUrl()]
+      relative_url = context.getRelativeUrl()
+      local_index_dict = {}
+      if isinstance(context, BaseCategory):
+        category_list = relative_url,
       else:
+        category_list = []
         if isinstance(base_category_list, str):
-          base_category_list = [base_category_list]
+          base_category_list = base_category_list,
         elif base_category_list is () or base_category_list is None:
           base_category_list = self.getBaseCategoryList()
-        category_list = []
         for base_category in base_category_list:
-          category_list.append("%s/%s" % (base_category, context.getRelativeUrl()))
-
-      sql_kw = {}
-      for sql_key in ('limit', 'order_by_expression'): # XXX-JPS it would be better to use Catalog API
-        if sql_key in kw:
-          sql_kw[sql_key] = kw[sql_key]
-
-      brain_result = self.Base_zSearchRelatedObjectsByCategoryList(
-                           category_list=category_list,
+          if self[base_category].isRelatedLocallyIndexed():
+            category = base_category + '/'
+            local_index_dict[base_category] = '' \
+              if relative_url.startswith(category) else category
+          else:
+            category_list.append("%s/%s" % (base_category, relative_url))
+
+      search = self.getPortalObject().Base_zSearchRelatedObjectsByCategoryList
+      if local_index_dict:
+        # For some base categories, lookup indexes in ZODB.
+        recurse = isinstance(context, Category) and not strict_membership
+        result_dict = {}
+        def check_local():
+          r = set(getattr(related, base_category, ()))
+          r.difference_update(result_dict)
+          for r in r:
+            try:
+              ob = self.unrestrictedTraverse(r)
+              if category in aq_base(ob).categories:
+                result_dict[r] = ob
+                continue
+              # Do not add 'r' to result_dict, because 'ob' may be linked in
+              # another way.
+            except (AttributeError, KeyError):
+              result_dict[r] = None
+            related.remove(base_category, r)
+        tv = getTransactionalVariable().setdefault(
+          'CategoriesTool.getRelatedValueList', {})
+        try:
+          related = aq_base(context)._related_index
+        except AttributeError:
+          related = RelatedIndex()
+        include_self = False
+        for base_category, category in local_index_dict.iteritems():
+          if not category:
+            # Categories are member of themselves.
+            include_self = True
+            result_dict[relative_url] = context
+          category += relative_url
+          if tv.get(category, -1) < recurse:
+            # Update local index with results from catalog for backward
+            # compatibility. But no need to do it several times in the same
+            # transaction.
+            for r in search(category_list=category,
+                            portal_type=None,
+                            strict_membership=strict_membership):
+              r = r.relative_url
+              # relative_url is empty if object is deleted (but not yet
+              # unindexed). Nothing specific to do in such case because
+              # category tool won't match.
+              try:
+                ob = self.unrestrictedTraverse(r)
+                categories = aq_base(ob).categories
+              except (AttributeError, KeyError):
+                result_dict[r] = None
+                continue
+              if category in categories:
+                related.add(base_category, r)
+                result_dict[r] = ob
+              elif recurse:
+                for p in categories:
+                  if p.startswith(category + '/'):
+                    try:
+                      o = self.unrestrictedTraverse(p)
+                      p = aq_base(o)._related_index
+                    except KeyError:
+                      continue
+                    except AttributeError:
+                      p = o._related_index = RelatedIndex()
+                    result_dict[r] = ob
+                    p.add(base_category, r)
+            tv[category] = recurse
+          # Get and check all objects referenced by local index for the base
+          # category that is currently considered.
+          check_local()
+        # Modify context only if it's worth it.
+        if related and not hasattr(aq_base(context), '_related_index'):
+          context._related_index = related
+        # In case of non-strict membership search, include all objects that
+        # are linked to a subobject of context.
+        if recurse:
+          r = [context]
+          while r:
+            for ob in r.pop().objectValues():
+              r.append(ob)
+              relative_url = ob.getRelativeUrl()
+              if include_self:
+                result_dict[relative_url] = ob
+              try:
+                related = aq_base(ob)._related_index
+              except AttributeError:
+                continue
+              for base_category, category in local_index_dict.iteritems():
+                category += relative_url
+                check_local()
+        # Filter out objects that are not of requested portal type.
+        result = [ob for ob in result_dict.itervalues() if ob is not None and (
+          not portal_type or ob.getPortalType() in portal_type)]
+        # Finish with base categories that are only indexed in catalog,
+        # making sure we don't return duplicate values.
+        if category_list:
+          for r in search(category_list=category_list,
                           portal_type=portal_type,
-                           strict_membership=strict_membership,
-                           **sql_kw)
+                          strict_membership=strict_membership):
+            if r.relative_url not in result_dict:
+              try:
+                result.append(self.unrestrictedTraverse(r.path))
+              except KeyError:
+                pass
 
+      else:
+        # Catalog-only search.
         result = []
+        for r in search(category_list=category_list,
+                        portal_type=portal_type,
+                        strict_membership=strict_membership):
+          try:
+            result.append(self.unrestrictedTraverse(r.path))
+          except KeyError:
+            pass
+
       if checked_permission is None:
-        # No permission to check
-        for b in brain_result:
-          o = b.getObject()
-          if o is not None:
-            result.append(o)
-      else:
+        return result
+
       # Check permissions on object
       if isinstance(checked_permission, str):
-          checked_permission = (checked_permission, )
+        checked_permission = checked_permission,
       checkPermission = self.portal_membership.checkPermission
-          for b in brain_result:
-            obj = b.getObject()
-            if obj is not None:
+      def check(ob):
         for permission in checked_permission:
-                if not checkPermission(permission, obj):
-                  break
-                result.append(obj)
-
-      return result
-      # XXX missing filter and **kw stuff
-      #return self.search_category(category_list=category_list,
-      #                            portal_type=spec)
-      # future implementation with brains, much more efficient
+          if checkPermission(permission, ob):
+            return True
+      return filter(check, result)
 
     security.declareProtected( Permissions.AccessContentsInformation,
                                'getRelatedPropertyList' )
     def getRelatedPropertyList(self, context, base_category_list=None,
-                               property_name=None, spec=(), 
-                               filter=None, base=1, 
+                               property_name=None,
                                checked_permission=None, **kw):
       """
         This methods returns the list of property_name on  objects
@@ -1426,8 +1499,7 @@ class CategoryTool( UniqueObject, Folder, Base ):
       result = []
       for o in self.getRelatedValueList(
                           context=context,
-                          base_category_list=base_category_list, spec=spec,
-                          filter=filter, base=base, 
+                          base_category_list=base_category_list,
                           checked_permission=checked_permission, **kw):
         result.append(o.getProperty(property_name, None))
       return result

product/CMFCategory/tests/testCMFCategory.py

@@ -26,6 +26,7 @@
 #
 ##############################################################################
 
+from collections import deque
 import unittest
 
 from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
@@ -42,6 +43,30 @@ class TestCMFCategory(ERP5TypeTestCase):
   region2 = 'europe/west/germany'
   region_list = [region1, region2]
 
+  category_dict = dict(
+    region = dict(
+      acquisition_base_category_list=('subordination','parent'),
+      acquisition_portal_type_list="python: ['Address', 'Organisation', 'Person']",
+      acquisition_mask_value=1,
+      acquisition_object_id_list=['default_address'],
+      contents=('europe', ('west', ('france', 'germany'))),
+      ),
+    subordination = dict(
+      acquisition_portal_type_list="python: ['Career', 'Organisation']",
+      acquisition_object_id_list=['default_career'],
+      ),
+    gender = dict(
+      fallback_base_category_list=['subordination'],
+      ),
+    resource = dict(
+      ),
+    test0 = dict(
+      ),
+    test1 = dict(
+      contents=('a', ('ab', 'ac', ('acd',))),
+      ),
+    )
+
   def getTitle(self):
     return "CMFCategory"
 
@@ -74,16 +99,44 @@ class TestCMFCategory(ERP5TypeTestCase):
     portal = self.portal
     self.validateRules()
 
+    portal_categories = self.getCategoriesTool()
+    for name, kw in self.category_dict.iteritems():
+      try:
+        bc = portal_categories[name]
+      except KeyError:
+        bc = portal_categories.newContent(name)
+      edit_kw = dict(
+        acquisition_copy_value=0,
+        acquisition_append_value=0,
+        acquisition_mask_value=0,
+        acquisition_portal_type_list="python: []",
+        related_locally_indexed=0)
+      edit_kw.update(kw)
+      queue = deque(((bc, edit_kw.pop('contents', ())),))
+      bc.edit(**edit_kw)
+      while queue:
+        parent, contents = queue.popleft()
+        for x in contents:
+          if type(x) is str:
+            try:
+              category = parent[x]
+            except KeyError:
+              category = parent.newContent(x)
+          else:
+            queue.append((category, x))
+
     # This test creates Person inside Person and Organisation inside
-    # Organisation, so we modifiy type informations to allow anything inside
-    # Person and Organisation (we'll cleanup on teardown)
-    self.getTypesTool().getTypeInfo('Person').filter_content_types = 0
-    organisation_ti = self.getTypesTool().getTypeInfo('Organisation')
-    organisation_ti.filter_content_types = 0
-    # we also enable 'destination' category on organisations
-    self._organisation_categories = organisation_ti.getTypeBaseCategoryList()
-    organisation_ti._setTypeBaseCategoryList(self._organisation_categories
-                                             + ['destination', 'resource'])
+    # Organisation, so we modify type informations to allow anything inside
+    # Person and Organisation (we'll cleanup on teardown).
+    self._original_categories = {}
+    for portal_type, categories in (
+        ('Person', []),
+        ('Organisation', ['destination', 'resource'])):
+      ti = self.getTypesTool().getTypeInfo(portal_type)
+      ti.filter_content_types = 0
+      self._original_categories[portal_type] = x = ti.getTypeBaseCategoryList()
+      x += 'test0', 'test1'
+      ti._setTypeBaseCategoryList(x + categories)
 
     # Make persons.
     person_module = self.getPersonModule()
@@ -109,62 +162,14 @@ class TestCMFCategory(ERP5TypeTestCase):
     sale_packing_list_module = portal.sale_packing_list_module
     if self.id1 not in sale_packing_list_module.objectIds():
         sale_packing_list_module.newContent(id=self.id1)
-    # This set the acquisition for region
-    for bc in ('region', ):
-      if not hasattr(portal_categories, bc):
-        portal_categories.newContent(portal_type='Base Category',id=bc)
-      portal_categories[bc].setAcquisitionBaseCategoryList(('subordination','parent'))
-      portal_categories[bc].setAcquisitionPortalTypeList("python: ['Address', 'Organisation', 'Person']")
-      portal_categories[bc].setAcquisitionMaskValue(1)
-      portal_categories[bc].setAcquisitionCopyValue(0)
-      portal_categories[bc].setAcquisitionAppendValue(0)
-      portal_categories[bc].setAcquisitionObjectIdList(['default_address'])
-      if not 'europe' in portal_categories[bc].objectIds():
-        portal_categories[bc].newContent(id='europe',portal_type='Category')
-      big_region = portal_categories[bc]['europe']
-      # Now we have to include by hand no categories
-      if not 'west' in big_region.objectIds():
-        big_region.newContent(id='west',portal_type='Category')
-      region = big_region['west']
-      if not 'france' in region.objectIds():
-        region.newContent(id='france',portal_type='Category')
-      if not 'germany' in region.objectIds():
-        region.newContent(id='germany',portal_type='Category')
-    for bc in ('subordination', ):
-      if not hasattr(portal_categories, bc):
-        portal_categories.newContent(portal_type='Base Category',id=bc)
-      portal_categories[bc].setAcquisitionPortalTypeList("python: ['Career', 'Organisation']")
-      portal_categories[bc].setAcquisitionMaskValue(0)
-      portal_categories[bc].setAcquisitionCopyValue(0)
-      portal_categories[bc].setAcquisitionAppendValue(0)
-      portal_categories[bc].setAcquisitionObjectIdList(['default_career'])
-    for bc in ('gender', ):
-      if not hasattr(portal_categories, bc):
-        portal_categories.newContent(portal_type='Base Category',id=bc)
-      portal_categories[bc].setAcquisitionPortalTypeList("python: []")
-      portal_categories[bc].setAcquisitionMaskValue(0)
-      portal_categories[bc].setAcquisitionCopyValue(0)
-      portal_categories[bc].setAcquisitionAppendValue(0)
-      portal_categories[bc].setFallbackBaseCategoryList(['subordination'])
-    for bc in ('resource', ):
-      if not hasattr(portal_categories, bc):
-        portal_categories.newContent(portal_type='Base Category',id=bc)
-      portal_categories[bc].setAcquisitionPortalTypeList("python: []")
-      portal_categories[bc].setAcquisitionMaskValue(0)
-      portal_categories[bc].setAcquisitionCopyValue(0)
-      portal_categories[bc].setAcquisitionAppendValue(0)
 
   def beforeTearDown(self):
     """Clean up."""
-    # categories
-    for bc in ('region', 'subordination', 'gender', 'resource'):
-      bc_obj = self.getPortal().portal_categories[bc]
-      bc_obj.manage_delObjects()
     # type informations
-    self.getTypesTool().getTypeInfo('Person').filter_content_types = 1
-    organisation_ti = self.getTypesTool().getTypeInfo('Organisation')
-    organisation_ti.filter_content_types = 1
-    organisation_ti = self._organisation_categories
+    for portal_type, categories in self._original_categories.iteritems():
+      ti = self.getTypesTool().getTypeInfo(portal_type)
+      ti.filter_content_types = 1
+      ti._setTypeBaseCategoryList(categories)
 
   def login(self):
     uf = self.portal.acl_users
@@ -1075,6 +1080,76 @@ class TestCMFCategory(ERP5TypeTestCase):
     # Check indexation
     self.tic()
 
+  def test_setCategoryMemberShip(self):
+    person = self.getPersonModule().newContent(portal_type='Person')
+    category_tool = self.getCategoriesTool()
+    bc = category_tool.newContent()
+    bc.newContent('a')
+    bc.newContent('b')
+    base = (bc.id + '/').__add__
+    def get(*args, **kw):
+      return category_tool.getCategoryMembershipList(person, *args, **kw)
+    def _set(*args, **kw):
+      return category_tool._setCategoryMembership(person, *args, **kw)
+    _set(bc.id, list('aa'))
+    self.assertEqual(get(bc.id), list('aa'))
+    _set(bc.id, list('baa'))
+    self.assertEqual(get(bc.id), list('aba'))
+    _set(bc.id, map(base, 'bb'), 1)
+    self.assertEqual(get(bc.id), list('bb'))
+    _set(bc.id, map(base, 'abb'), 1)
+    self.assertEqual(get(bc.id), list('bab'))
+    _set(bc.id, ())
+
+  def test_relatedIndex(self):
+    category_tool = self.getCategoriesTool()
+    newOrganisation = self.getOrganisationModule().newContent
+    organisation = newOrganisation()
+    other_organisation = newOrganisation(destination_value=organisation)
+    person = self.getPersonModule().newContent(test0_value=organisation,
+                                               test1='a/ac/acd')
+    self.tic()
+    get = organisation.getTest0RelatedValueList
+    a = category_tool.test1.a
+    def check():
+      self.assertEqual([person, other_organisation],
+        category_tool.getRelatedValueList(organisation))
+      self.assertEqual([person], get())
+      self.assertEqual([person], get(portal_type='Person'))
+      self.assertEqual([], get(portal_type='Organisation'))
+      self.assertEqual([person], a.getTest1RelatedValueList(
+        portal_type='Person'))
+      self.assertEqual([a], a.getTest1RelatedValueList(
+        strict_membership=True))
+      self.assertEqual([person], a.ac.acd.getTest1RelatedValueList(
+        portal_type='Person', strict_membership=True))
+    category_tool.test0._setRelatedLocallyIndexed(True)
+    category_tool.test1._setRelatedLocallyIndexed(True)
+    check()
+    related_list = sorted(a.getTest1RelatedList())
+    self.assertTrue(person.getRelativeUrl() in related_list)
+    self.assertEqual(related_list, sorted(x.getRelativeUrl()
+      for x in self.portal.portal_catalog(test1_uid=a.getUid())))
+    related = organisation._related_index
+    self.assertTrue(related)
+    self.assertEqual([person.getRelativeUrl()], list(related.test0))
+    person.unindexObject()
+    self.tic()
+    category_tool.test0._setRelatedLocallyIndexed(False)
+    self.assertEqual([], get())
+    category_tool.test0._setRelatedLocallyIndexed(True)
+    check()
+    person.categories = tuple(x for x in person.categories
+                                if not x.startswith('test0/'))
+    self.assertEqual([], get())
+    self.assertFalse(related)
+    self.assertEqual([], list(related.test0))
+    related = a.ac.acd._related_index.test1
+    self.assertEqual(list(related), [person.getRelativeUrl()])
+    person._setTest1Value(a)
+    self.assertEqual(list(related), [])
+
+
 def test_suite():
   suite = unittest.TestSuite()
   suite.addTest(unittest.makeSuite(TestCMFCategory))

product/ERP5/Document/BusinessTemplate.py

@@ -544,7 +544,8 @@ class BaseTemplateItem(Implicit, Persistent):
     klass = obj.__class__
     classname = klass.__name__
 
-    attr_set = set(('_dav_writelocks', '_filepath', '_owner', 'last_id', 'uid',
+    attr_set = set(('_dav_writelocks', '_filepath', '_owner', '_related_index',
+                    'last_id', 'uid',
                     '__ac_local_roles__', '__ac_local_roles_group_id_dict__'))
     if export:
       if not keep_workflow_history:

product/ERP5/Tool/CategoryTool.py

@@ -40,6 +40,7 @@ from Products.ERP5Type.Core.Folder import OFS_HANDLER
 from Products.ERP5Type.CopySupport import CopyContainer
 from Products.CMFCore.utils import getToolByName
 from Products.ERP5Type.Cache import caching_instance_method
+from Products.ERP5Type.dynamic import portal_type_class
 
 from zLOG import LOG
 
@@ -84,54 +85,10 @@ class CategoryTool(CopyContainer, CMFCategoryTool, BaseTool):
     def hasContent(self,id):
       return id in self.objectIds()
 
-    security.declareProtected(Permissions.AccessContentsInformation, 'getCategoryParentUidList')
-    def getCategoryParentUidList(self, relative_url, base_category = None, strict=0):
-      """
-        Returns the uids of all categories provided in categorie. This
-        method can support relative_url such as site/group/a/b/c which
-        base category is site yet use categories defined in group.
-
-        It is also able to use acquisition to create complex categories
-        such as site/group/a/b/c/b1/c1 where b and b1 are both children
-        categories of a.
-
-        relative_url -- a single relative url of a list of
-                        relative urls
-
-        strict       -- if set to 1, only return uids of parents, not
-                        relative_url
-      """
-      uid_dict = {}
-      if type(relative_url) is type('a'): relative_url = (relative_url,)
-      for path in relative_url:
-        try:
-          o = self.getCategoryValue(path, base_category=base_category)
-          if o is not None:
-            if base_category is None:
-              my_base_category = self.getBaseCategoryId(path)
-            else:
-              my_base_category = base_category
-            bo = getattr(self, my_base_category, None)
-            if bo is not None:
-              bo_uid = bo.getUid()
-              uid_dict[(o.getUid(), bo_uid, 1)] = 1 # Strict membership
-              if o.meta_type == 'ERP5 Category' or o.meta_type == 'ERP5 Base Category' or \
-                o.meta_type == 'CMF Category' or o.meta_type == 'CMF Base Category':
-                # This goes up in the category tree
-                # XXX we should also go up in some other cases....
-                # ie. when some documents act as categories
-                if not strict:
-                  while o.meta_type == 'ERP5 Category' or o.meta_type == 'CMF Category':
-                    o = o.aq_parent # We want acquisition here without aq_inner
-                    uid_dict[(o.getUid(), bo_uid, 0)] = 1 # Non strict
-        except (TypeError, KeyError):
-          LOG('WARNING: CategoriesTool',0, 'Unable to find uid for %s' % path)
-      return uid_dict.keys()
-
-    security.declareProtected(Permissions.AccessContentsInformation, 'getUids')
-    getUids = getCategoryParentUidList
-
-    @caching_instance_method(id='portal_categories.getBaseCategoryDict', cache_factory='erp5_content_long', cache_id_generator=lambda m, *a, **k:m)
+    @caching_instance_method(
+      id='portal_categories.getBaseCategoryDict',
+      cache_factory='erp5_content_long',
+      cache_id_generator=lambda *a: portal_type_class.last_sync)
     def getBaseCategoryDict(self):
       """
         Cached method to which resturns a dict with category names as keys, and None as values.

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/BaseCategory_validateRelatedLocallyIndexed.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>Script_magic</string> </key>
+            <value> <int>3</int> </value>
+        </item>
+        <item>
+            <key> <string>_bind_names</string> </key>
+            <value>
+              <object>
+                <klass>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                </klass>
+                <tuple/>
+                <state>
+                  <dictionary>
+                    <item>
+                        <key> <string>_asgns</string> </key>
+                        <value>
+                          <dictionary>
+                            <item>
+                                <key> <string>name_container</string> </key>
+                                <value> <string>container</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_context</string> </key>
+                                <value> <string>context</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_m_self</string> </key>
+                                <value> <string>script</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_subpath</string> </key>
+                                <value> <string>traverse_subpath</string> </value>
+                            </item>
+                          </dictionary>
+                        </value>
+                    </item>
+                  </dictionary>
+                </state>
+              </object>
+            </value>
+        </item>
+        <item>
+            <key> <string>_body</string> </key>
+            <value> <string>return not (value and (\n
+  request.other[\'field_my_acquisition_object_id_list\'] or\n
+  request.other[\'field_my_acquisition_base_category_list\']))\n
+</string> </value>
+        </item>
+        <item>
+            <key> <string>_params</string> </key>
+            <value> <string>value, request</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>BaseCategory_validateRelatedLocallyIndexed</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/BaseCategory_view.xml

@@ -105,6 +105,7 @@
                     <key> <string>right</string> </key>
                     <value>
                       <list>
+                        <string>my_related_locally_indexed</string>
                         <string>my_acquisition_copy_value</string>
                         <string>my_acquisition_mask_value</string>
                         <string>my_acquisition_append_value</string>

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/BaseCategory_view/my_related_locally_indexed.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>delegated_list</string> </key>
+            <value>
+              <list>
+                <string>external_validator</string>
+                <string>title</string>
+              </list>
+            </value>
+        </item>
+        <item>
+            <key> <string>delegated_message_list</string> </key>
+            <value>
+              <list>
+                <string>external_validator_failed</string>
+              </list>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>my_related_locally_indexed</string> </value>
+        </item>
+        <item>
+            <key> <string>message_values</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>external_validator_failed</string> </key>
+                    <value> <string>Local index is incompatible with category acquision.</string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+        <item>
+            <key> <string>overrides</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>field_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+                <item>
+                    <key> <string>form_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+                <item>
+                    <key> <string>target</string> </key>
+                    <value> <string></string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+        <item>
+            <key> <string>tales</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>field_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+                <item>
+                    <key> <string>form_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+                <item>
+                    <key> <string>target</string> </key>
+                    <value> <string></string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+        <item>
+            <key> <string>values</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>external_validator</string> </key>
+                    <value>
+                      <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+                    </value>
+                </item>
+                <item>
+                    <key> <string>field_id</string> </key>
+                    <value> <string>my_checkbox</string> </value>
+                </item>
+                <item>
+                    <key> <string>form_id</string> </key>
+                    <value> <string>Base_viewFieldLibrary</string> </value>
+                </item>
+                <item>
+                    <key> <string>target</string> </key>
+                    <value> <string>Click to edit the target</string> </value>
+                </item>
+                <item>
+                    <key> <string>title</string> </key>
+                    <value> <string>Index Related Documents Locally</string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="Method" module="Products.Formulator.MethodField"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>method_name</string> </key>
+            <value> <string>BaseCategory_validateRelatedLocallyIndexed</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Base_zSearchRelatedObjectsByCategoryList.xml

@@ -58,15 +58,11 @@ order_by_expression</string> </value>
             <key> <string>src</string> </key>
             <value> <string encoding="cdata"><![CDATA[
 
-SELECT DISTINCT catalog.uid, catalog.path, portal_type\n
+SELECT DISTINCT catalog.uid, path, relative_url, portal_type\n
 FROM catalog, category\n
 WHERE catalog.uid = category.uid\n
   <dtml-if portal_type>\n
-    AND\n
-    (<dtml-in portal_type>\n
-      <dtml-unless sequence-start> OR </dtml-unless>\n
-        catalog.portal_type=\'<dtml-var sequence-item>\'\n
-    </dtml-in>)\n
+    AND <dtml-sqltest portal_type type="string" multiple>\n
   </dtml-if>\n
   AND (<dtml-var "portal_categories.buildSQLSelector(category_list)">)\n
   <dtml-if strict_membership>\n

product/ERP5/bootstrap/erp5_core/bt/revision

-41060
\ No newline at end of file
+41061
\ No newline at end of file

product/ERP5/bootstrap/erp5_property_sheets/PropertySheetTemplateItem/portal_property_sheets/BaseCategory/related_locally_indexed_property.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Standard Property" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_local_properties</string> </key>
+            <value>
+              <tuple>
+                <dictionary>
+                  <item>
+                      <key> <string>id</string> </key>
+                      <value> <string>mode</string> </value>
+                  </item>
+                  <item>
+                      <key> <string>type</string> </key>
+                      <value> <string>string</string> </value>
+                  </item>
+                </dictionary>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>categories</string> </key>
+            <value>
+              <tuple>
+                <string>elementary_type/boolean</string>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value> <string>Determines if related values should be indexed on target documents (i.e. in ZODB) in addition to catalog.\n
+This is incompatible with category acquisition.</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>related_locally_indexed_property</string> </value>
+        </item>
+        <item>
+            <key> <string>mode</string> </key>
+            <value> <string>w</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Standard Property</string> </value>
+        </item>
+        <item>
+            <key> <string>property_default</string> </key>
+            <value> <string>python: 0</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

product/ERP5/bootstrap/erp5_property_sheets/bt/revision

-56
\ No newline at end of file
+57
\ No newline at end of file

product/ERP5/tests/testResource.py

@@ -862,20 +862,19 @@ class TestResource(ERP5TypeTestCase):
     self.tic()
     # Test the cases
     for product, variation, node, base_price in test_case_list:
+      categories = []
       if node is not None:
         self.logMessage("Check product %s with destination section %s" % \
                         (product.getTitle(), node.getTitle()),
                         tab=1)
-        self.assertEquals(base_price,
-             product.getPrice(
-               categories=['destination_section/%s' % node.getRelativeUrl(),
-                           variation]))
+        categories.append('destination_section/' + node.getRelativeUrl())
       else:
         self.logMessage("Check product %s without destination section" % \
                         product.getTitle(),
                         tab=1)
-        self.assertEquals(base_price,
-                          product.getPrice(categories=[variation]))
+      if variation:
+        categories.append(variation)
+      self.assertEqual(base_price, product.getPrice(categories=categories))
   
 
   # The following test tests Movement.getPrice, which is based on the movement

product/ERP5Type/Accessor/Value.py

@@ -48,8 +48,7 @@ class SetSetter(BaseSetter):
     def __call__(self, instance, *args, **kw):
       if self._warning:
         LOG("ERP5Type Deprecated Setter Id:",0, self._id)
-      value = set(args[0])
-      instance._setValue(self._key, value,
+      instance._setValue(self._key, set(args[0]),
                                                 spec=kw.get('spec',()),
                                                 filter=kw.get('filter', None),
                                                 portal_type=kw.get('portal_type',()),
@@ -451,7 +450,7 @@ class UidSetSetter(BaseSetter):
     def __call__(self, instance, *args, **kw):
       if self._warning:
         LOG("ERP5Type Deprecated Getter Id:",0, self._id)
-      instance._setValueUidList(self._key, args[0],
+      instance._setValueUidList(self._key, set(args[0]),
                                                  spec=kw.get('spec',()),
                                                  filter=kw.get('filter', None),
                                                  portal_type=kw.get('portal_type',()),

product/ERP5Type/Base.py

@@ -2013,11 +2013,8 @@ class Base( CopyContainer,
                                        checked_permission=None):
     # We must do an ordered list so we can not use the previous method
     # self._setValue(id, self.portal_catalog.getObjectList(uids), spec=spec)
-    references = []
-    if type(uids) not in (type(()), type([])):
-      uids = [uids]
-    for uid in uids:
-      references.append(self.portal_catalog.getObject(uid))
+    references = map(self.getPortalObject().portal_catalog.getObject,
+                     (uids,) if isinstance(uids, (int, long)) else uids)
     self._setValue(id, references, spec=spec, filter=filter, portal_type=portal_type,
                                    keep_default=keep_default, checked_permission=checked_permission)
 
@@ -2178,9 +2175,6 @@ class Base( CopyContainer,
     """
     return self._getCategoryTool().getAcquiredCategoryList(self)
 
-  def _getAcquiredCategoryList(self):
-    return self._getCategoryTool()._getAcquiredCategoryList(self)
-
   security.declareProtected( Permissions.ModifyPortalContent, 'setCategoryList' )
   def setCategoryList(self, path_list):
     self.portal_categories.setCategoryList(self, path_list)

product/ERP5Type/tests/testERP5Type.py

@@ -722,27 +722,21 @@ class TestERP5Type(PropertySheetTestCase, LogInterceptor):
       self.assertEquals(person.getDefaultRegion(), 'alpha')
       person.setRegionUid(alpha.getUid())
       self.assertEquals(person.getRegion(), 'alpha')
-      person.setRegionUidList([alpha.getUid(), alpha.getUid()])
-      self.assertEquals(person.getRegionList(), ['alpha', 'alpha'])
+      person.setRegionUidList([beta.getUid(), beta.getUid()])
+      self.assertEquals(person.getRegionList(), ['beta', 'beta'])
       person.setRegionUidSet([alpha.getUid(), alpha.getUid()])
-      self.assertEquals(person.getRegionSet(), ['alpha'])
+      self.assertEquals(person.getRegionList(), ['alpha'])
       person.setRegionUidList([alpha.getUid(), beta.getUid(), alpha.getUid()])
       self.assertEquals(person.getRegionList(), ['alpha', 'beta', 'alpha'])
       person.setRegionUidSet([alpha.getUid(), beta.getUid(), alpha.getUid()])
-      result = person.getRegionSet()
-      result.sort()
-      self.assertEquals(result, ['alpha', 'beta'])
+      self.assertEquals(sorted(person.getRegionSet()), ['alpha', 'beta'])
       person.setDefaultRegionUid(beta.getUid())
       self.assertEquals(person.getDefaultRegion(), 'beta')
-      result = person.getRegionSet()
-      result.sort()
-      self.assertEquals(result, ['alpha', 'beta'])
+      self.assertEquals(sorted(person.getRegionSet()), ['alpha', 'beta'])
       self.assertEquals(person.getRegionList(), ['beta', 'alpha'])
       person.setDefaultRegionUid(alpha.getUid())
       self.assertEquals(person.getDefaultRegion(), 'alpha')
-      result = person.getRegionSet()
-      result.sort()
-      self.assertEquals(result, ['alpha', 'beta'])
+      self.assertEquals(sorted(person.getRegionSet()), ['alpha', 'beta'])
       self.assertEquals(person.getRegionList(), ['alpha', 'beta'])
       # Test accessor on documents rather than on categories
       person.setDefaultRegionUid(person.getUid())