diff --git a/product/ERP5/Document/SQLNonContinuousIncreasingIdGenerator.py b/product/ERP5/Document/SQLNonContinuousIncreasingIdGenerator.py index 37d4ac73d934f779b24344dd4ea7e01e33d76961..d638793565f7c71acb0e11997040fbea657f49eb 100644 --- a/product/ERP5/Document/SQLNonContinuousIncreasingIdGenerator.py +++ b/product/ERP5/Document/SQLNonContinuousIncreasingIdGenerator.py @@ -29,13 +29,13 @@ import zope.interface from Acquisition import aq_base from AccessControl import ClassSecurityInfo -from Products.ERP5Type.Globals import PersistentMapping from Products.ERP5Type import Permissions, PropertySheet, interfaces from Products.ERP5Type.Utils import ScalarMaxConflictResolver from Products.ERP5.Document.IdGenerator import IdGenerator from _mysql_exceptions import ProgrammingError from MySQLdb.constants.ER import NO_SUCH_TABLE from zLOG import LOG, INFO +from BTrees.OOBTree import OOBTree class SQLNonContinuousIncreasingIdGenerator(IdGenerator): """ @@ -135,7 +135,7 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator): id_group_done.append(id_group) # save the last ids which not exist in sql - for id_group in (set(self.last_max_id_dict) - set(id_group_done)): + for id_group in (set(self.last_max_id_dict.keys()) - set(id_group_done)): set_last_id_method(id_group=id_group, last_id=self.last_max_id_dict[id_group].value) @@ -168,7 +168,7 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator): LOG('initialize SQL Generator', INFO, 'Id Generator: %s' % (self,)) # Check the dictionnary if self.last_max_id_dict is None: - self.last_max_id_dict = PersistentMapping() + self.last_max_id_dict = OOBTree() # Create table portal_ids if not exists portal = self.getPortalObject() try: @@ -219,7 +219,7 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator): added here) """ # Remove dictionary - self.last_max_id_dict = PersistentMapping() + self.last_max_id_dict = OOBTree() # Remove and recreate portal_ids table portal = self.getPortalObject() portal.IdTool_zDropTable() @@ -263,9 +263,24 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator): # Update persistent dict if self.getStoredInZodb(): if self.last_max_id_dict is None: - self.last_max_id_dict = PersistentMapping() + self.last_max_id_dict = OOBTree() self.last_max_id_dict.update(new_id_dict) + security.declareProtected(Permissions.ModifyPortalContent, + 'rebuildGeneratorIdDict') + def rebuildGeneratorIdDict(self): + """ + Rebuild generator id dict from SQL table. + + This is usefull when we are migrating the dict structure, or cleanly + rebuild the dict from sql table. This method is opposite of + rebuildSqlTable(). + """ + if not self.getStoredInZodb(): + raise RuntimeError('Please set \"stored in zodb\" flag before rebuild.') + id_dict = self.exportGeneratorIdDict() + self.importGeneratorIdDict(id_dict=id_dict, clear=True) + security.declareProtected(Permissions.ModifyPortalContent, 'rebuildSqlTable') def rebuildSqlTable(self): @@ -319,7 +334,7 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator): portal = self.getPortalObject() last_max_id_dict = self.last_max_id_dict if last_max_id_dict is None: - self.last_max_id_dict = last_max_id_dict = PersistentMapping() + self.last_max_id_dict = last_max_id_dict = OOBTree() last_id_group = None for line in portal.IdTool_zGetValueList(id_group=id_group): last_id_group = id_group = line[0] diff --git a/product/ERP5/Document/ZODBContinuousIncreasingIdGenerator.py b/product/ERP5/Document/ZODBContinuousIncreasingIdGenerator.py index 2d261af5c93b247ff9ea93e91e588e3999e4b36b..e54ec18658f26a9375e0020d9809efc26723222f 100644 --- a/product/ERP5/Document/ZODBContinuousIncreasingIdGenerator.py +++ b/product/ERP5/Document/ZODBContinuousIncreasingIdGenerator.py @@ -28,9 +28,9 @@ import zope.interface from AccessControl import ClassSecurityInfo -from Products.ERP5Type.Globals import PersistentMapping from Products.ERP5Type import Permissions, interfaces from Products.ERP5.Document.IdGenerator import IdGenerator +from BTrees.OOBTree import OOBTree from zLOG import LOG, INFO @@ -97,7 +97,7 @@ class ZODBContinuousIncreasingIdGenerator(IdGenerator): """ LOG('initialize ZODB Generator', INFO, 'Id Generator: %s' % (self,)) if getattr(self, 'last_id_dict', None) is None: - self.last_id_dict = PersistentMapping() + self.last_id_dict = OOBTree() # XXX compatiblity code below, dump the old dictionnaries portal_ids = getattr(self, 'portal_ids', None) @@ -124,7 +124,7 @@ class ZODBContinuousIncreasingIdGenerator(IdGenerator): added here) """ # Remove dictionary - self.last_id_dict = PersistentMapping() + self.last_id_dict = OOBTree() security.declareProtected(Permissions.ModifyPortalContent, 'exportGeneratorIdDict') @@ -149,3 +149,17 @@ class ZODBContinuousIncreasingIdGenerator(IdGenerator): if not isinstance(value, int): raise TypeError, 'the value given in dictionary is not a integer' self.last_id_dict.update(id_dict) + + security.declareProtected(Permissions.ModifyPortalContent, + 'rebuildGeneratorIdDict') + def rebuildGeneratorIdDict(self): + """ + Rebuild generator id dict. + In fact, export it, clear it and import it into new dict. + This is mostly intendted to use when we are migrating the id dict + structure. + """ + id_dict = self.exportGeneratorIdDict() + self.importGeneratorIdDict(id_dict=id_dict, clear=True) + + diff --git a/product/ERP5/tests/testIdTool.py b/product/ERP5/tests/testIdTool.py index 5d70362cbd4407c03f2886505fea6ff642272ed6..bb31b994d1c31ee149974e38dda29cc2c3a72c8d 100644 --- a/product/ERP5/tests/testIdTool.py +++ b/product/ERP5/tests/testIdTool.py @@ -33,6 +33,7 @@ import unittest from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase from Products.ERP5Type.tests.utils import createZODBPythonScript from _mysql_exceptions import ProgrammingError +from BTrees.OOBTree import OOBTree class TestIdTool(ERP5TypeTestCase): @@ -154,7 +155,7 @@ class TestIdTool(ERP5TypeTestCase): zodb_generator = self.getLastGenerator('test_application_zodb') zodb_portal_type = 'ZODB Continuous Increasing Id Generator' self.assertEquals(zodb_generator.getPortalType(), zodb_portal_type) - self.assertEqual(getattr(zodb_generator, 'last_id_dict', {}), {}) + self.assertEqual(len(zodb_generator.last_id_dict), 0) # generate ids self.checkGenerateNewId('test_application_zodb') # check zodb dict @@ -171,7 +172,11 @@ class TestIdTool(ERP5TypeTestCase): sql_generator = self.getLastGenerator('test_application_sql') sql_portal_type = 'SQL Non Continuous Increasing Id Generator' self.assertEquals(sql_generator.getPortalType(), sql_portal_type) - self.assertEquals(getattr(sql_generator, 'last_max_id_dict', {}), {}) + # This assertEquals() make sure that last_max_id_dict property is empty. + # Note that keys(), values() and items() methods of OOBTree do not return + # a list of all the items. The methods return a lazy evaluated object. + # len() method on OOBTree can handle properly even in the situation. + self.assertEquals(len(sql_generator.last_max_id_dict), 0) # retrieve method to recovery the last id in the database last_id_method = getattr(self.portal, 'IdTool_zGetLastId', None) self.assertNotEquals(last_id_method, None) @@ -189,7 +194,7 @@ class TestIdTool(ERP5TypeTestCase): self.assertEquals(sql_generator.last_max_id_dict['c02'].value, 0) self.assertEquals(sql_generator.last_max_id_dict['d02'].value, 21) else: - self.assertEquals(getattr(sql_generator, 'last_max_id_dict', {}), {}) + self.assertEquals(len(sql_generator.last_max_id_dict), 0) def test_02b_generateNewIdWithSQLGeneratorWithoutStorageZODB(self): """ diff --git a/product/ERP5/tests/testIdToolUpgrade.py b/product/ERP5/tests/testIdToolUpgrade.py index e4473a7f75a8b50274086e13340435602c34f9c0..7d0aad57445277d472d96333e1463c596f3e106f 100644 --- a/product/ERP5/tests/testIdToolUpgrade.py +++ b/product/ERP5/tests/testIdToolUpgrade.py @@ -27,12 +27,17 @@ # ############################################################################## +import unittest + from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase from Products.ERP5Type.Globals import PersistentMapping +from Products.ERP5Type.Utils import ScalarMaxConflictResolver from BTrees.Length import Length +from BTrees.OOBTree import OOBTree from zLOG import LOG -class TestIdTool(ERP5TypeTestCase): + +class TestIdToolUpgrade(ERP5TypeTestCase): """ Automatic upgrade of id tool is really sensible to any change. Therefore, make sure that the upgrade is still working even if there changes. @@ -46,6 +51,66 @@ class TestIdTool(ERP5TypeTestCase): """ return "Test Id Tool Upgrade" + def afterSetUp(self): + self.login() + self.portal = self.getPortal() + self.id_tool = self.portal.portal_ids + self.id_tool.initializeGenerator(all=True) + self.createGenerators() + self.tic() + + def beforeTearDown(self): + self.id_tool.clearGenerator(all=True) + + def createGenerators(self): + """ + Initialize some generators for the tests + """ + self.application_sql_generator = self.id_tool.newContent(\ + portal_type='Application Id Generator', + reference='test_application_sql', + version='001') + self.conceptual_sql_generator = self.id_tool.newContent(\ + portal_type='Conceptual Id Generator', + reference='test_non_continuous_increasing', + version='001') + self.sql_generator = self.id_tool.newContent(\ + portal_type='SQL Non Continuous Increasing Id Generator', + reference='test_sql_non_continuous_increasing', + version='001') + self.application_sql_generator.setSpecialiseValue(\ + self.conceptual_sql_generator) + self.conceptual_sql_generator.setSpecialiseValue(self.sql_generator) + + self.application_zodb_generator = self.id_tool.newContent(\ + portal_type='Application Id Generator', + reference='test_application_zodb', + version='001') + self.conceptual_zodb_generator = self.id_tool.newContent(\ + portal_type='Conceptual Id Generator', + reference='test_continuous_increasing', + version='001') + self.zodb_generator = self.id_tool.newContent(\ + portal_type='ZODB Continuous Increasing Id Generator', + reference='test_zodb_continuous_increasing', + version='001') + self.application_zodb_generator.setSpecialiseValue(\ + self.conceptual_zodb_generator) + self.conceptual_zodb_generator.setSpecialiseValue(self.zodb_generator) + + def getLastGenerator(self, id_generator): + """ + Return Last Id Generator + """ + document_generator = self.id_tool.searchFolder(reference=id_generator)[0] + application_generator = document_generator.getLatestVersionValue() + conceptual_generator = application_generator.getSpecialiseValue()\ + .getLatestVersionValue() + last_generator = conceptual_generator.getSpecialiseValue()\ + .getLatestVersionValue() + return last_generator + + def testUpgradeIdToolDicts(self): # With old erp5_core, we have no generators, no IdTool_* zsql methods, # and we have a dictionary stored on id tool @@ -123,3 +188,132 @@ class TestIdTool(ERP5TypeTestCase): generator = generator_list[0] self.assertEquals(generator.last_id_dict['foo'], 4) self.assertEquals(generator.last_id_dict["('bar', 'baz')"], 3) + + + def _setUpLastMaxIdDict(self, id_generator_reference): + def countup(id_generator, id_group, until): + for i in xrange(until + 1): + self.id_tool.generateNewId(id_generator=id_generator_reference, + id_group=id_group) + + countup(id_generator_reference, 'A-01', 2) + countup(id_generator_reference, 'B-01', 1) + var_id = 'C-%04d' + for x in xrange(self.a_lot_of_key): + countup(id_generator_reference, var_id % x, 0) + + def _getLastIdDictName(self, id_generator): + portal_type = id_generator.getPortalType() + if portal_type == 'SQL Non Continuous Increasing Id Generator': + return 'last_max_id_dict' + elif portal_type == 'ZODB Continuous Increasing Id Generator': + return 'last_id_dict' + else: + raise RuntimeError("not expected to test the generator :%s" % portal_type) + + def _getLastIdDict(self, id_generator): + last_id_dict_name = self._getLastIdDictName(id_generator) + return getattr(id_generator, last_id_dict_name) + + def _setLastIdDict(self, id_generator, value): + last_id_dict_name = self._getLastIdDictName(id_generator) + setattr(id_generator, last_id_dict_name, value) + + def _getValueFromLastIdDict(self, last_id_dict, key): + value = last_id_dict[key] + if isinstance(value, int): + # in ZODB Id Generator it is stored in int + return value + elif isinstance(value, ScalarMaxConflictResolver): + return value.value + else: + raise RuntimeError('not expected to test the value: %s' % value) + + def _assertIdGeneratorLastMaxIdDict(self, id_generator): + last_id_dict = self._getLastIdDict(id_generator) + self.assertEquals(2, self._getValueFromLastIdDict(last_id_dict, 'A-01')) + self.assertEquals(1, self._getValueFromLastIdDict(last_id_dict, 'B-01')) + for x in xrange(self.a_lot_of_key): + key = 'C-%04d' % x + self.assertEquals(0, self._getValueFromLastIdDict(last_id_dict, key)) + + # 1(A-01) + 1(B-01) + a_lot_of_key(C-*) + number_of_group_id = self.a_lot_of_key + 2 + self.assertEqual(number_of_group_id, + len(id_generator.exportGeneratorIdDict())) + self.assertEqual(number_of_group_id, len(last_id_dict)) + + + def _checkDataStructureMigration(self, id_generator): + """ First, simulate previous data structure which is using + PersisntentMapping as the storage, then migrate to OOBTree. + Then, migrate the id generator again from OOBTree to OOBtree + just to be sure.""" + id_generator_reference = id_generator.getReference() + + reference_portal_type_dict = { + 'test_sql_non_continuous_increasing':'SQL Non Continuous ' \ + 'Increasing Id Generator', + 'test_zodb_continuous_increasing':'ZODB Continuous ' \ + 'Increasing Id Generator' + } + try: + portal_type = reference_portal_type_dict[id_generator_reference] + self.assertEquals(id_generator.getPortalType(), portal_type) + except: + raise ValueError("reference is not valid: %s" % id_generator_reference) + + self._setLastIdDict(id_generator, PersistentMapping()) # simulate previous + last_id_dict = self._getLastIdDict(id_generator) + + # setUp the data for migration test + self._setUpLastMaxIdDict(id_generator_reference) + + # test migration: PersistentMapping to OOBTree + self.assertTrue(isinstance(last_id_dict, PersistentMapping)) + self._assertIdGeneratorLastMaxIdDict(id_generator) + id_generator.rebuildGeneratorIdDict() # migrate the dict + self._assertIdGeneratorLastMaxIdDict(id_generator) + + # test migration: OOBTree to OOBTree. this changes nothing, just to be sure + last_id_dict = self._getLastIdDict(id_generator) + self.assertTrue(isinstance(last_id_dict, OOBTree)) + self._assertIdGeneratorLastMaxIdDict(id_generator) + id_generator.rebuildGeneratorIdDict() # migrate the dict + self._assertIdGeneratorLastMaxIdDict(id_generator) + + # test migration: SQL to OOBTree + if id_generator.getPortalType() == \ + 'SQL Non Continuous Increasing Id Generator': + self._setLastIdDict(id_generator, OOBTree()) # set empty one + last_id_dict = self._getLastIdDict(id_generator) + assert(len(last_id_dict), 0) # 0 because it is empty + self.assertTrue(isinstance(last_id_dict, OOBTree)) + # migrate the dict totally from sql table in this case + id_generator.rebuildGeneratorIdDict() + self._assertIdGeneratorLastMaxIdDict(id_generator) + + + def testRebuildIdDictFromPersistentMappingToOOBTree(self): + """ + Check migration is working + """ + # this is the amount of keys that is creating in this test + self.a_lot_of_key = 1010 + # check sql id generator migration + id_generator_reference = 'test_application_sql' + id_generator = self.getLastGenerator(id_generator_reference) + id_generator.setStoredInZodb(True) + id_generator.clearGenerator() # clear stored data + self._checkDataStructureMigration(id_generator) + + # check zodb id generator migration + id_generator_reference = 'test_application_zodb' + id_generator = self.getLastGenerator(id_generator_reference) + id_generator.clearGenerator() # clear stored data + self._checkDataStructureMigration(id_generator) + +def test_suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(TestIdToolUpgrade)) + return suite