Improvements on folder migration

See merge request nexedi/erp5!1394

bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testFolderMigration.py

@@ -481,6 +481,34 @@ class TestFolderMigration(ERP5TypeTestCase, LogInterceptor):
       self.assertEqual(self.folder.isBTree(), False)
       self.assertEqual(self.folder.isHBTree(), True)
 
+    def test_15_checkMigrationWorksIfIdsDontChange(self):
+      """
+      migrate folder using a script that leaves some objects with same ids
+      """
+      # Create some objects
+      self.assertEqual(self.folder.getIdGenerator(), '')
+      self.assertEqual(len(self.folder), 0)
+      obj1 = self.newContent()
+      self.assertEqual(obj1.getId(), '1')
+      obj2 = self.newContent()
+      self.assertEqual(obj2.getId(), '2')
+      obj3 = self.newContent(id='custom-id')
+      self.assertEqual(obj3.getId(), 'custom-id')
+      self.tic()
+      # call migration script Base_generateIdFromCreationDate that only changes int ids
+      self.folder.migrateToHBTree(migration_generate_id_method="Base_generateIdFromCreationDate",
+                                  new_generate_id_method="_generatePerDayId")
+      self.tic()
+      # check object ids
+      from DateTime import DateTime
+      date = DateTime().Date()
+      date = date.replace("/", "")
+      #1 y 2 should have new format id (because old ids were int)
+      self.assertEqual(obj1.getId(), '%s-1' % date)
+      self.assertEqual(obj2.getId(), '%s-2' % date)
+      #3 should have the same old id
+      self.assertEqual(obj3.getId(), 'custom-id')
+
 def test_suite():
   suite = unittest.TestSuite()
   suite.addTest(unittest.makeSuite(TestFolderMigration))

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Base_generateIdFromCreationDate.py

+# retrieve a date from object and return the new id
+if obj is None:
+  obj = context
+
+# don't change custom (non-int) ids
+old_id = obj.getId()
+if not old_id.isdigit():
+  return old_id
+
+date = obj.getCreationDate()
+if date is None:
+  from DateTime import DateTime
+  date = DateTime()
+
+date = date.Date().replace('/', '')
+return "%s-%s" %(date, old_id)

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Base_generateIdFromCreationDate.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>Script_magic</string> </key>
+            <value> <int>3</int> </value>
+        </item>
+        <item>
+            <key> <string>_bind_names</string> </key>
+            <value>
+              <object>
+                <klass>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                </klass>
+                <tuple/>
+                <state>
+                  <dictionary>
+                    <item>
+                        <key> <string>_asgns</string> </key>
+                        <value>
+                          <dictionary>
+                            <item>
+                                <key> <string>name_container</string> </key>
+                                <value> <string>container</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_context</string> </key>
+                                <value> <string>context</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_m_self</string> </key>
+                                <value> <string>script</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_subpath</string> </key>
+                                <value> <string>traverse_subpath</string> </value>
+                            </item>
+                          </dictionary>
+                        </value>
+                    </item>
+                  </dictionary>
+                </state>
+              </object>
+            </value>
+        </item>
+        <item>
+            <key> <string>_params</string> </key>
+            <value> <string>obj=None</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>Base_generateIdFromCreationDate</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/ERP5Site_setNewIdPerBundle.py

@@ -4,5 +4,6 @@ method = getattr(context, method)
 for id_ in id_list:
   ob = folder.get(id_)
   new_id = method(ob)
+  if new_id != id_:
     ob.setDefaultActivateParameterDict(activate_kw)
     ob.setId(new_id)

product/ERP5Type/Core/Folder.py

@@ -795,14 +795,13 @@ class Folder(FolderMixIn, CopyContainer, ObjectManager, Base, OFSFolder2, CMFBTr
     return self._folder_handler == HBTREE_HANDLER
 
   security.declareProtected( Permissions.ManagePortal, 'migrateToHBTree' )
-  def migrateToHBTree(self, migration_generate_id_method=None, new_generate_id_method='_generatePerDayId', REQUEST=None):
+  def migrateToHBTree(self, migration_generate_id_method=None, new_generate_id_method='_generatePerDayId', bundle_count=10, REQUEST=None):
     """
     Function to migrate from a BTree folder to HBTree folder.
     It will first call setId on all folder objects to have right id
     to be used with an hbtreefolder.
     Then it will migrate foder from btree to hbtree.
     """
-    BUNDLE_COUNT = 10
 
     # if folder is already migrated or migration process is in progress
     # do not do anything beside logging
@@ -822,15 +821,15 @@ class Folder(FolderMixIn, CopyContainer, ObjectManager, Base, OFSFolder2, CMFBTr
       tag = "%s/%s/migrate" %(self.getId(),migration_generate_id_method)
       id_list  = list(self.objectIds())
       # set new id by bundle
-      for x in xrange(len(self) / BUNDLE_COUNT):
-        self.activate(activity="SQLQueue", tag=tag).ERP5Site_setNewIdPerBundle(
+      for x in xrange(len(self) / bundle_count):
+        self.activate(activity="SQLQueue", tag=tag, priority=3, serialization_tag='ERP5Site_setNewIdPerBundle').ERP5Site_setNewIdPerBundle(
           self.getPath(),
-          id_list[x*BUNDLE_COUNT:(x+1)*BUNDLE_COUNT],
+          id_list[x*bundle_count:(x+1)*bundle_count],
           migration_generate_id_method, tag)
 
-      remaining_id_count = len(self) % BUNDLE_COUNT
+      remaining_id_count = len(self) % bundle_count
       if remaining_id_count:
-        self.activate(activity="SQLQueue", tag=tag).ERP5Site_setNewIdPerBundle(
+        self.activate(activity="SQLQueue", tag=tag, priority=3, serialization_tag='ERP5Site_setNewIdPerBundle').ERP5Site_setNewIdPerBundle(
           self.getPath(),
           id_list[-remaining_id_count:],
           migration_generate_id_method, tag)