storage: fix severe performance issue by committing backend only at key moments

neo/storage/database/manager.py

@@ -15,7 +15,6 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from neo.lib import logging, util
-from neo.lib.exception import DatabaseFailure
 from neo.lib.protocol import ZERO_TID
 
 class CreationUndone(Exception):
@@ -28,7 +27,6 @@ class DatabaseManager(object):
         """
             Initialize the object.
         """
-        self._under_transaction = False
         self._wait = wait
         self._parse(database)
 
@@ -50,34 +48,9 @@ class DatabaseManager(object):
         """
         raise NotImplementedError
 
-    def __enter__(self):
-        """
-            Begin a transaction
-        """
-        if self._under_transaction:
-            raise DatabaseFailure('A transaction has already begun')
-        r = self.begin()
-        self._under_transaction = True
-        return r
-
-    def __exit__(self, exc_type, exc_value, tb):
-        if not self._under_transaction:
-            raise DatabaseFailure('The transaction has not begun')
-        self._under_transaction = False
-        if exc_type is None:
-            self.commit()
-        else:
-            self.rollback()
-
-    def begin(self):
-        pass
-
     def commit(self):
         pass
 
-    def rollback(self):
-        pass
-
     def _getPartition(self, oid_or_tid):
         return oid_or_tid % self.getNumPartitions()
 
@@ -91,11 +64,8 @@ class DatabaseManager(object):
         """
             Set a configuration value
         """
-        if self._under_transaction:
-            self._setConfiguration(key, value)
-        else:
-            with self:
-                self._setConfiguration(key, value)
+        self._setConfiguration(key, value)
+        self.commit()
 
     def _setConfiguration(self, key, value):
         raise NotImplementedError
@@ -344,8 +314,8 @@ class DatabaseManager(object):
             else:
                 del refcount[data_id]
         if prune:
-            with self:
-                self._pruneData(data_id_list)
+            self._pruneData(data_id_list)
+            self.commit()
 
     __getDataTID = set()
     def _getDataTID(self, oid, tid=None, before_tid=None):
@@ -465,11 +435,11 @@ class DatabaseManager(object):
 
     def truncate(self, tid):
         assert tid not in (None, ZERO_TID), tid
-        with self:
-            assert self.getBackupTID()
-            self.setBackupTID(tid)
-            for partition in xrange(self.getNumPartitions()):
-                self._deleteRange(partition, tid)
+        assert self.getBackupTID()
+        self.setBackupTID(tid)
+        for partition in xrange(self.getNumPartitions()):
+            self._deleteRange(partition, tid)
+        self.commit()
 
     def getTransaction(self, tid, all = False):
         """Return a tuple of the list of OIDs, user information,

neo/storage/handlers/storage.py

@@ -98,6 +98,7 @@ class StorageOperationHandler(EventHandler):
             for serial, oid_list in object_dict.iteritems():
                 for oid in oid_list:
                     deleteObject(oid, serial)
+        self.app.dm.commit()
         assert not pack_tid, "TODO"
         if next_tid:
             self.app.replicator.fetchObjects(next_tid, next_oid)

neo/tests/storage/testStorageDBTests.py

@@ -20,7 +20,6 @@ from mock import Mock
 from neo.lib.util import add64, dump, p64, u64
 from neo.lib.protocol import CellStates, ZERO_HASH, ZERO_OID, ZERO_TID, MAX_TID
 from .. import NeoUnitTestBase
-from neo.lib.exception import DatabaseFailure
 
 
 class StorageDBTests(NeoUnitTestBase):
@@ -93,29 +92,6 @@ class StorageDBTests(NeoUnitTestBase):
         db = self.getDB()
         self.checkConfigEntry(db.getPTID, db.setPTID, self.getPTID(1))
 
-    def test_transaction(self):
-        db = self.getDB()
-        x = []
-        class DB(db.__class__):
-            begin    = lambda self: x.append('begin')
-            commit   = lambda self: x.append('commit')
-            rollback = lambda self: x.append('rollback')
-        db.__class__ = DB
-        with db:
-            self.assertEqual(x.pop(), 'begin')
-        self.assertEqual(x.pop(), 'commit')
-        try:
-            with db:
-                self.assertEqual(x.pop(), 'begin')
-                with db:
-                    self.fail()
-            self.fail()
-        except DatabaseFailure:
-            pass
-        self.assertEqual(x.pop(), 'rollback')
-        self.assertRaises(DatabaseFailure, db.__exit__, None, None, None)
-        self.assertFalse(x)
-
     def test_getPartitionTable(self):
         db = self.getDB()
         ptid = self.getPTID(1)

neo/tests/threaded/__init__.py

@@ -300,11 +300,11 @@ class StorageApplication(ServerNode, neo.storage.app.Application):
             pass
 
     def switchTables(self):
-        with self.dm as q:
-            for table in ('trans', 'obj'):
-                q('ALTER TABLE %s RENAME TO tmp' % table)
-                q('ALTER TABLE t%s RENAME TO %s' % (table, table))
-                q('ALTER TABLE tmp RENAME TO t%s' % table)
+        q = self.dm.query
+        for table in 'trans', 'obj':
+            q('ALTER TABLE %s RENAME TO tmp' % table)
+            q('ALTER TABLE t%s RENAME TO %s' % (table, table))
+            q('ALTER TABLE tmp RENAME TO t%s' % table)
 
     def getDataLockInfo(self):
         dm = self.dm