storage: automatic upgrade of 'obj' table (change of indices)

neo/storage/database/mysqldb.py

@@ -50,7 +50,7 @@ def getPrintableQuery(query, max=70):
 class MySQLDatabaseManager(DatabaseManager):
     """This class manages a database on MySQL."""
 
-    VERSION = 1
+    VERSION = 2
     ENGINES = "InnoDB", "RocksDB", "TokuDB"
     _engine = ENGINES[0] # default engine
 
@@ -181,6 +181,17 @@ class MySQLDatabaseManager(DatabaseManager):
         self._checkNoUnfinishedTransactions()
         self.query("DROP TABLE IF EXISTS ttrans")
 
+    def _migrate2(self, schema_dict):
+        q = self.query
+        if self.nonempty('obj') is None:
+            if self.nonempty('new_obj') is None:
+                return
+        else:
+            q("DROP TABLE IF EXISTS new_obj")
+            q(schema_dict.pop('obj') % 'new_obj' + " SELECT * FROM obj")
+            q("DROP TABLE obj")
+        q("ALTER TABLE new_obj RENAME TO obj")
+
     def _setup(self, dedup=False):
         self._config.clear()
         q = self.query
@@ -226,8 +237,8 @@ class MySQLDatabaseManager(DatabaseManager):
                  tid BIGINT UNSIGNED NOT NULL,
                  data_id BIGINT UNSIGNED NULL,
                  value_tid BIGINT UNSIGNED NULL,
-                 PRIMARY KEY (`partition`, tid, oid),
-                 KEY (`partition`, oid, tid),
+                 PRIMARY KEY (`partition`, oid, tid),
+                 KEY tid (`partition`, tid, oid),
                  KEY (data_id)
              ) ENGINE=""" + p
 
@@ -341,15 +352,14 @@ class MySQLDatabaseManager(DatabaseManager):
         offset_list = self._getAssignedPartitionList()
         p64 = util.p64
         q = self.query
-        sql = ("SELECT MAX(tid) FROM %s FORCE INDEX (PRIMARY)"
-               " WHERE `partition`=%s")
+        sql = "SELECT MAX(tid) FROM %s WHERE `partition`=%s"
         trans, obj = ({partition: p64(tid)
             for partition in offset_list
             for tid, in q(sql % (t, partition))
             if tid is not None}
-            for t in ('trans', 'obj'))
+            for t in ('trans FORCE INDEX (PRIMARY)', 'obj FORCE INDEX (tid)'))
         oid = self._sqlmax(
-            "SELECT MAX(oid) FROM obj FORCE INDEX (`partition`)"
+            "SELECT MAX(oid) FROM obj FORCE INDEX (PRIMARY)"
             " WHERE `partition`=%s", offset_list)
         return trans, obj, None if oid is None else p64(oid)
 
@@ -369,7 +379,7 @@ class MySQLDatabaseManager(DatabaseManager):
 
     def getLastObjectTID(self, oid):
         oid = util.u64(oid)
-        r = self.query("SELECT tid FROM obj FORCE INDEX(`partition`)"
+        r = self.query("SELECT tid FROM obj FORCE INDEX(PRIMARY)"
                        " WHERE `partition`=%d AND oid=%d"
                        " ORDER BY tid DESC LIMIT 1"
                        % (self._getReadablePartition(oid), oid))
@@ -377,7 +387,7 @@ class MySQLDatabaseManager(DatabaseManager):
 
     def _getNextTID(self, *args): # partition, oid, tid
         r = self.query("SELECT tid FROM obj"
-                       " FORCE INDEX(`partition`)"
+                       " FORCE INDEX(PRIMARY)"
                        " WHERE `partition`=%d AND oid=%d AND tid>%d"
                        " ORDER BY tid LIMIT 1" % args)
         return r[0][0] if r else None
@@ -386,7 +396,7 @@ class MySQLDatabaseManager(DatabaseManager):
         q = self.query
         partition = self._getReadablePartition(oid)
         sql = ('SELECT tid, compression, data.hash, value, value_tid'
-               ' FROM obj FORCE INDEX(`partition`)'
+               ' FROM obj FORCE INDEX(PRIMARY)'
                ' LEFT JOIN data ON (obj.data_id = data.id)'
                ' WHERE `partition` = %d AND oid = %d') % (partition, oid)
         if before_tid is not None:
@@ -443,7 +453,7 @@ class MySQLDatabaseManager(DatabaseManager):
         for partition in offset_list:
             where = " WHERE `partition`=%d" % partition
             data_id_list = [x for x, in
-                q("SELECT DISTINCT data_id FROM obj FORCE INDEX(PRIMARY)"
+                q("SELECT DISTINCT data_id FROM obj FORCE INDEX(tid)"
                   + where)
                 if x]
             if not self._use_partition:
@@ -608,7 +618,7 @@ class MySQLDatabaseManager(DatabaseManager):
     del _structLL
 
     def _getDataTID(self, oid, tid=None, before_tid=None):
-        sql = ('SELECT tid, value_tid FROM obj FORCE INDEX(`partition`)'
+        sql = ('SELECT tid, value_tid FROM obj FORCE INDEX(PRIMARY)'
                ' WHERE `partition` = %d AND oid = %d'
               ) % (self._getReadablePartition(oid), oid)
         if tid is not None:
@@ -699,7 +709,7 @@ class MySQLDatabaseManager(DatabaseManager):
         p64 = util.p64
         r = self.query("SELECT tid, IF(compression < 128, LENGTH(value),"
             "  CAST(CONV(HEX(SUBSTR(value, 5, 4)), 16, 10) AS INT))"
-            " FROM obj FORCE INDEX(`partition`)"
+            " FROM obj FORCE INDEX(PRIMARY)"
             " LEFT JOIN data ON (obj.data_id = data.id)"
             " WHERE `partition` = %d AND oid = %d AND tid >= %d"
             " ORDER BY tid DESC LIMIT %d, %d" %
@@ -728,7 +738,7 @@ class MySQLDatabaseManager(DatabaseManager):
         u64 = util.u64
         p64 = util.p64
         min_tid = u64(min_tid)
-        r = self.query('SELECT tid, oid FROM obj FORCE INDEX(PRIMARY)'
+        r = self.query('SELECT tid, oid FROM obj FORCE INDEX(tid)'
                        ' WHERE `partition` = %d AND tid <= %d'
                        ' AND (tid = %d AND %d <= oid OR %d < tid)'
                        ' ORDER BY tid ASC, oid ASC LIMIT %d' % (
@@ -793,7 +803,7 @@ class MySQLDatabaseManager(DatabaseManager):
         q = self.query
         self._setPackTID(tid)
         for count, oid, max_serial in q("SELECT COUNT(*) - 1, oid, MAX(tid)"
-                                        " FROM obj FORCE INDEX(`partition`)"
+                                        " FROM obj FORCE INDEX(PRIMARY)"
                                         " WHERE tid <= %d GROUP BY oid"
                                         % tid):
             partition = getPartition(oid)
@@ -844,7 +854,7 @@ class MySQLDatabaseManager(DatabaseManager):
         # last grouped value, instead of the greatest one.
         r = self.query(
             """SELECT tid, oid
-               FROM obj FORCE INDEX(PRIMARY)
+               FROM obj FORCE INDEX(tid)
                WHERE `partition` = %(partition)s
                  AND tid <= %(max_tid)d
                  AND (tid > %(min_tid)d OR

neo/storage/database/sqlite.py

@@ -68,7 +68,7 @@ class SQLiteDatabaseManager(DatabaseManager):
              never be used for small requests.
     """
 
-    VERSION = 1
+    VERSION = 2
 
     def _parse(self, database):
         self.db = os.path.expanduser(database)
@@ -117,11 +117,25 @@ class SQLiteDatabaseManager(DatabaseManager):
         self._checkNoUnfinishedTransactions()
         self.query("DROP TABLE IF EXISTS ttrans")
 
+    def _migrate2(self, schema_dict, index_dict):
+        # BBB: As explained in _setup, no transactional DDL
+        #      so let's do the same dance as for MySQL.
+        q = self.query
+        if self.nonempty('obj') is None:
+            if self.nonempty('new_obj') is None:
+                return
+        else:
+            q("DROP TABLE IF EXISTS new_obj")
+            q(schema_dict.pop('obj') % 'new_obj')
+            q("INSERT INTO new_obj SELECT * FROM obj")
+            q("DROP TABLE obj")
+        q("ALTER TABLE new_obj RENAME TO obj")
+
     def _setup(self, dedup=False):
-        # SQLite does support transactional Data Definition Language statements
-        # but unfortunately, the built-in Python binding automatically commits
-        # between such statements. This anti-feature causes this method to be
-        # relatively slow; unit tests enables the UNSAFE boolean flag.
+        # BBB: SQLite has transactional DDL but before Python 3.6,
+        #      the binding automatically commits between such statements.
+        #      This anti-feature causes this method to be relatively slow.
+        #      Unit tests enables the UNSAFE boolean flag.
         self._config.clear()
         q = self.query
         schema_dict = OrderedDict()
@@ -162,10 +176,10 @@ class SQLiteDatabaseManager(DatabaseManager):
                  tid INTEGER NOT NULL,
                  data_id INTEGER,
                  value_tid INTEGER,
-                 PRIMARY KEY (partition, tid, oid))
+                 PRIMARY KEY (partition, oid, tid))
             """
         index_dict['obj'] = (
-            "CREATE INDEX %s ON %s(partition, oid, tid)",
+            "CREATE INDEX %s ON %s(partition, tid, oid)",
             "CREATE INDEX %s ON %s(data_id)")
 
         # The table "data" stores object data.