Merge remote-tracking branch 'origin/master' into t

* origin/master:
  client: kill .supportsTransactionalUndo()
  client: for read accesses, pick a random good node, connected or not
  storage: optimize storage layout of raw data for replication
  sqlite: remove useless AUTOINCREMENT for data.id (reuse of deleted ids is fine)
  storage: speed up reads by indexing 'obj' primarily by 'oid' (instead of 'tid')
  storage: pass schema of tables to migration methods
  storage: update backend version between each migration step

neo/client/Storage.py

@@ -136,9 +136,6 @@ class Storage(BaseStorage.BaseStorage,
     def supportsUndo(self):
         return True
 
-    def supportsTransactionalUndo(self):
-        return True
-
     def loadEx(self, oid, version):
         try:
             data, serial, _ = self.app.load(oid)

neo/client/pool.py

@@ -56,25 +56,28 @@ class ConnectionPool(object):
             logging.info('%r not ready', node)
         else:
             logging.info('Connected %r', node)
+            # Make sure this node will be considered for the next reads
+            # even if there was a previous recent failure.
+            self.node_failure_dict.pop(node.getUUID(), None)
             return conn
         self.node_failure_dict[node.getUUID()] = time.time() + MAX_FAILURE_AGE
 
     def getCellSortKey(self, cell, random=random.random):
-        # The use of 'random' suffles cells to randomise node to access.
-        uuid = cell.getUUID()
-        # First, prefer a connected node.
-        if uuid in self.connection_dict:
-            return random()
-        # Then one that didn't fail recently.
-        failure = self.node_failure_dict.get(uuid)
+        # Prefer a node that didn't fail recently.
+        failure = self.node_failure_dict.get(cell.getUUID())
         if failure:
             if time.time() < failure:
-                # At last, order by date of connection failure.
+                # Or order by date of connection failure.
                 return failure
             # Do not use 'del' statement: we didn't lock, so another
             # thread might have removed uuid from node_failure_dict.
-            self.node_failure_dict.pop(uuid, None)
-        return 1 + random()
+            self.node_failure_dict.pop(cell.getUUID(), None)
+        # A random one, connected or not, is a trivial and quite efficient way
+        # to distribute the load evenly. On write accesses, a client connects
+        # to all nodes of touched cells, but before that, or if a client is
+        # specialized to only do read-only accesses, it should not limit
+        # itself to only use the first connected nodes.
+        return random()
 
     def getConnForNode(self, node):
         """Return a locked connection object to a given node

neo/storage/database/importer.py

@@ -406,7 +406,7 @@ class ImporterDatabaseManager(DatabaseManager):
                         if compression:
                             data = compressed_data
                     checksum = util.makeChecksum(data)
-                    data_id = self.holdData(util.makeChecksum(data), data,
+                    data_id = self.holdData(util.makeChecksum(data), oid, data,
                                             compression)
                     data_id_list.append(data_id)
                 object_list.append((oid, data_id, data_tid))

neo/storage/database/manager.py

@@ -161,11 +161,14 @@ class DatabaseManager(object):
                 "The database can not be upgraded because you have unfinished"
                 " transactions. Use an older version of NEO to verify them.")
 
-    def _getVersion(self):
+    def migrate(self, *args, **kw):
         version = int(self.getConfiguration("version") or 0)
         if self.VERSION < version:
             raise DatabaseFailure("The database can not be downgraded.")
-        return version
+        while version < self.VERSION:
+            version += 1
+            getattr(self, '_migrate%s' % version)(*args, **kw)
+            self.setConfiguration("version", version)
 
     def doOperation(self, app):
         pass
@@ -485,7 +488,11 @@ class DatabaseManager(object):
         existing data is first thrown away.
         """
 
-    @requires(_changePartitionTable)
+    def _getDataLastId(self, partition):
+        """
+        """
+
+    @requires(_changePartitionTable, _getDataLastId)
     def changePartitionTable(self, ptid, cell_list, reset=False):
         readable_set = self._readable_set
         if reset:
@@ -500,6 +507,10 @@ class DatabaseManager(object):
                 raise NonReadableCell
             self._getPartition = _getPartition
             self._getReadablePartition = _getReadablePartition
+            d = self._data_last_ids = []
+            for p in xrange(np):
+                i = self._getDataLastId(p)
+                d.append(p << 48 if i is None else i + 1)
         me = self.getUUID()
         for offset, nid, state in cell_list:
             if nid == me:
@@ -567,7 +578,7 @@ class DatabaseManager(object):
         """
 
     @abstract
-    def storeData(self, checksum, data, compression):
+    def storeData(self, checksum, oid, data, compression):
         """To be overridden by the backend to store object raw data
 
         If same data was already stored, the storage only has to check there's

neo/storage/handlers/storage.py

@@ -113,7 +113,7 @@ class StorageOperationHandler(EventHandler):
                               checksum, data, data_serial):
         dm = self.app.dm
         if data or checksum != ZERO_HASH:
-            data_id = dm.storeData(checksum, data, compression)
+            data_id = dm.storeData(checksum, oid, data, compression)
         else:
             data_id = None
         # Directly store the transaction.

neo/storage/transactions.py

@@ -470,7 +470,7 @@ class TransactionManager(EventQueue):
         if data is None:
             data_id = None
         else:
-            data_id = self._app.dm.holdData(checksum, data, compression)
+            data_id = self._app.dm.holdData(checksum, oid, data, compression)
         transaction.store(oid, data_id, value_serial)
 
     def rebaseObject(self, ttid, oid):

neo/tests/storage/testStorageDBTests.py

@@ -104,6 +104,7 @@ class StorageDBTests(NeoUnitTestBase):
 
     def test_getPartitionTable(self):
         db = self.getDB()
+        db.setNumPartitions(3)
         uuid1, uuid2 = self.getStorageUUID(), self.getStorageUUID()
         cell1 = (0, uuid1, CellStates.OUT_OF_DATE)
         cell2 = (1, uuid1, CellStates.UP_TO_DATE)
@@ -124,7 +125,7 @@ class StorageDBTests(NeoUnitTestBase):
         self._last_ttid = ttid = add64(self._last_ttid, 1)
         transaction = oid_list, 'user', 'desc', 'ext', False, ttid
         H = "0" * 20
-        object_list = [(oid, self.db.holdData(H, '', 1), None)
+        object_list = [(oid, self.db.holdData(H, oid, '', 1), None)
                        for oid in oid_list]
         return (transaction, object_list)
 
@@ -203,6 +204,7 @@ class StorageDBTests(NeoUnitTestBase):
 
     def test_setPartitionTable(self):
         db = self.getDB()
+        db.setNumPartitions(3)
         ptid = 1
         uuid = self.getStorageUUID()
         cell1 = 0, uuid, CellStates.OUT_OF_DATE
@@ -452,8 +454,8 @@ class StorageDBTests(NeoUnitTestBase):
         tid4 = self.getNextTID()
         tid5 = self.getNextTID()
         oid1 = p64(1)
-        foo = db.holdData("3" * 20, 'foo', 0)
-        bar = db.holdData("4" * 20, 'bar', 0)
+        foo = db.holdData("3" * 20, oid1, 'foo', 0)
+        bar = db.holdData("4" * 20, oid1, 'bar', 0)
         db.releaseData((foo, bar))
         db.storeTransaction(
             tid1, (

neo/tests/storage/testStorageMySQL.py

@@ -19,6 +19,7 @@ from MySQLdb import NotSupportedError, OperationalError
 from MySQLdb.constants.ER import UNKNOWN_STORAGE_ENGINE
 from ..mock import Mock
 from neo.lib.exception import DatabaseFailure
+from neo.lib.protocol import ZERO_OID
 from neo.lib.util import p64
 from .. import DB_PREFIX, DB_SOCKET, DB_USER
 from .testStorageDBTests import StorageDBTests
@@ -114,7 +115,7 @@ class StorageMySQLdbTests(StorageDBTests):
         self.assertEqual(2, max(len(self.db.escape(chr(x)))
                                 for x in xrange(256)))
         self.assertEqual(2, len(self.db.escape('\0')))
-        self.db.storeData('\0' * 20, '\0' * (2**24-1), 0)
+        self.db.storeData('\0' * 20, ZERO_OID, '\0' * (2**24-1), 0)
         size, = query_list
         max_allowed = self.db.__class__._max_allowed_packet
         self.assertTrue(max_allowed - 1024 < size <= max_allowed, size)
@@ -123,7 +124,7 @@ class StorageMySQLdbTests(StorageDBTests):
             self.db._max_allowed_packet = max_allowed_packet
             del query_list[:]
             self.db.storeTransaction(p64(0),
-                ((p64(1<<i),0,None) for i in xrange(10)), None)
+                ((p64(1<<i),1234,None) for i in xrange(10)), None)
             self.assertEqual(max(query_list), max_allowed_packet)
             self.assertEqual(len(query_list), count)
 

neo/tests/storage/testTransactions.py

@@ -57,7 +57,7 @@ class TransactionManagerTests(NeoUnitTestBase):
         self.manager.storeObject(locking_serial, ram_serial, oid, 0, "3" * 20,
             'bar', None)
         holdData = self.app.dm.mockGetNamedCalls('holdData')
-        self.assertEqual(holdData.pop(0).params, ("3" * 20, 'bar', 0))
+        self.assertEqual(holdData.pop(0).params, ("3" * 20, oid, 'bar', 0))
         orig_object = self.manager.getObjectFromTransaction(locking_serial,
             oid)
         self.manager.updateObjectDataForPack(oid, orig_serial, None, checksum)

neo/tests/threaded/test.py

@@ -480,17 +480,18 @@ class Test(NEOThreadedTest):
     def test_notifyNodeInformation(self, cluster):
         # translated from MasterNotificationsHandlerTests
         # (neo.tests.client.testMasterHandler)
+        good = [1, 0].pop
         if 1:
             cluster.db # open DB
             s0, s1 = cluster.client.nm.getStorageList()
             conn = s0.getConnection()
             self.assertFalse(conn.isClosed())
             getCellSortKey = cluster.client.cp.getCellSortKey
-            self.assertEqual(getCellSortKey(s0, int), 0)
+            self.assertEqual(getCellSortKey(s0, good), 0)
             cluster.neoctl.dropNode(s0.getUUID())
             self.assertEqual([s1], cluster.client.nm.getStorageList())
             self.assertTrue(conn.isClosed())
-            self.assertEqual(getCellSortKey(s0, int), 1)
+            self.assertEqual(getCellSortKey(s0, good), 1)
             # XXX: the test originally checked that 'unregister' method
             #      was called (even if it's useless in this case),
             #      but we would need an API to do that easily.
@@ -1439,7 +1440,7 @@ class Test(NEOThreadedTest):
             bad = []
             ok = []
             def data_args(value):
-                return makeChecksum(value), value, 0
+                return makeChecksum(value), ZERO_OID, value, 0
             node_list = []
             for i, s in enumerate(cluster.storage_list):
                 node_list.append(s.uuid)