storage: pass schema of tables to migration methods

neo/storage/database/manager.py

@@ -161,13 +161,13 @@ class DatabaseManager(object):
                 "The database can not be upgraded because you have unfinished"
                 " transactions. Use an older version of NEO to verify them.")
 
-    def migrate(self):
+    def migrate(self, *args, **kw):
         version = int(self.getConfiguration("version") or 0)
         if self.VERSION < version:
             raise DatabaseFailure("The database can not be downgraded.")
         while version < self.VERSION:
             version += 1
-            getattr(self, '_migrate%s' % version)()
+            getattr(self, '_migrate%s' % version)(*args, **kw)
             self.setConfiguration("version", version)
 
     def doOperation(self, app):

neo/storage/database/mysqldb.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from binascii import a2b_hex
+from collections import OrderedDict
 import MySQLdb
 from MySQLdb import DataError, IntegrityError, \
     OperationalError, ProgrammingError
@@ -176,7 +177,7 @@ class MySQLDatabaseManager(DatabaseManager):
             if e.args[0] != NO_SUCH_TABLE:
                 raise
 
-    def _migrate1(self):
+    def _migrate1(self, _):
         self._checkNoUnfinishedTransactions()
         self.query("DROP TABLE IF EXISTS ttrans")
 
@@ -184,32 +185,29 @@ class MySQLDatabaseManager(DatabaseManager):
         self._config.clear()
         q = self.query
         p = engine = self._engine
+        schema_dict = OrderedDict()
 
-        if self.nonempty("config") is None:
         # The table "config" stores configuration
         # parameters which affect the persistent data.
-            q("""CREATE TABLE config (
+        schema_dict['config'] = """CREATE TABLE %s (
                   name VARBINARY(255) NOT NULL PRIMARY KEY,
                   value VARBINARY(255) NULL
-                 ) ENGINE=""" + engine)
-            self._setConfiguration("version", self.VERSION)
-        else:
-            self.migrate()
+              ) ENGINE=""" + engine
 
         # The table "pt" stores a partition table.
-        q("""CREATE TABLE IF NOT EXISTS pt (
+        schema_dict['pt'] = """CREATE TABLE %s (
                  rid INT UNSIGNED NOT NULL,
                  nid INT NOT NULL,
                  state TINYINT UNSIGNED NOT NULL,
                  PRIMARY KEY (rid, nid)
-             ) ENGINE=""" + engine)
+             ) ENGINE=""" + engine
 
         if self._use_partition:
             p += """ PARTITION BY LIST (`partition`) (
                 PARTITION dummy VALUES IN (NULL))"""
 
         # The table "trans" stores information on committed transactions.
-        q("""CREATE TABLE IF NOT EXISTS trans (
+        schema_dict['trans'] =  """CREATE TABLE %s (
                  `partition` SMALLINT UNSIGNED NOT NULL,
                  tid BIGINT UNSIGNED NOT NULL,
                  packed BOOLEAN NOT NULL,
@@ -219,10 +217,10 @@ class MySQLDatabaseManager(DatabaseManager):
                  ext BLOB NOT NULL,
                  ttid BIGINT UNSIGNED NOT NULL,
                  PRIMARY KEY (`partition`, tid)
-             ) ENGINE=""" + p)
+             ) ENGINE=""" + p
 
         # The table "obj" stores committed object metadata.
-        q("""CREATE TABLE IF NOT EXISTS obj (
+        schema_dict['obj'] = """CREATE TABLE %s (
                  `partition` SMALLINT UNSIGNED NOT NULL,
                  oid BIGINT UNSIGNED NOT NULL,
                  tid BIGINT UNSIGNED NOT NULL,
@@ -231,7 +229,7 @@ class MySQLDatabaseManager(DatabaseManager):
                  PRIMARY KEY (`partition`, tid, oid),
                  KEY (`partition`, oid, tid),
                  KEY (data_id)
-             ) ENGINE=""" + p)
+             ) ENGINE=""" + p
 
         if engine == "TokuDB":
             engine += " compression='tokudb_uncompressed'"
@@ -239,21 +237,21 @@ class MySQLDatabaseManager(DatabaseManager):
         # The table "data" stores object data.
         # We'd like to have partial index on 'hash' column (e.g. hash(4))
         # but 'UNIQUE' constraint would not work as expected.
-        q("""CREATE TABLE IF NOT EXISTS data (
+        schema_dict['data'] = """CREATE TABLE %%s (
                  id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
                  hash BINARY(20) NOT NULL,
                  compression TINYINT UNSIGNED NULL,
                  value MEDIUMBLOB NOT NULL%s
              ) ENGINE=%s""" % (""",
-                 UNIQUE (hash, compression)""" if dedup else "", engine))
+                 UNIQUE (hash, compression)""" if dedup else "", engine)
 
-        q("""CREATE TABLE IF NOT EXISTS bigdata (
+        schema_dict['bigdata'] = """CREATE TABLE %s (
                  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
                  value MEDIUMBLOB NOT NULL
-             ) ENGINE=""" + engine)
+             ) ENGINE=""" + engine
 
         # The table "ttrans" stores information on uncommitted transactions.
-        q("""CREATE TABLE IF NOT EXISTS ttrans (
+        schema_dict['ttrans'] = """CREATE TABLE %s (
                  `partition` SMALLINT UNSIGNED NOT NULL,
                  tid BIGINT UNSIGNED,
                  packed BOOLEAN NOT NULL,
@@ -262,17 +260,26 @@ class MySQLDatabaseManager(DatabaseManager):
                  description BLOB NOT NULL,
                  ext BLOB NOT NULL,
                  ttid BIGINT UNSIGNED NOT NULL
-             ) ENGINE=""" + engine)
+             ) ENGINE=""" + engine
 
         # The table "tobj" stores uncommitted object metadata.
-        q("""CREATE TABLE IF NOT EXISTS tobj (
+        schema_dict['tobj'] = """CREATE TABLE %s (
                  `partition` SMALLINT UNSIGNED NOT NULL,
                  oid BIGINT UNSIGNED NOT NULL,
                  tid BIGINT UNSIGNED NOT NULL,
                  data_id BIGINT UNSIGNED NULL,
                  value_tid BIGINT UNSIGNED NULL,
                  PRIMARY KEY (tid, oid)
-             ) ENGINE=""" + engine)
+             ) ENGINE=""" + engine
+
+        if self.nonempty('config') is None:
+            q(schema_dict.pop('config') % 'config')
+            self._setConfiguration('version', self.VERSION)
+        else:
+            self.migrate(schema_dict)
+
+        for table, schema in schema_dict.iteritems():
+            q(schema % ('IF NOT EXISTS ' + table))
 
         self._uncommitted_data.update(q("SELECT data_id, count(*)"
             " FROM tobj WHERE data_id IS NOT NULL GROUP BY data_id"))

neo/storage/database/sqlite.py

@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from collections import OrderedDict
 import os
 import sqlite3
 from hashlib import sha1
@@ -112,7 +113,7 @@ class SQLiteDatabaseManager(DatabaseManager):
             if not e.args[0].startswith("no such table:"):
                 raise
 
-    def _migrate1(self):
+    def _migrate1(self, *_):
         self._checkNoUnfinishedTransactions()
         self.query("DROP TABLE IF EXISTS ttrans")
 
@@ -123,27 +124,26 @@ class SQLiteDatabaseManager(DatabaseManager):
         # relatively slow; unit tests enables the UNSAFE boolean flag.
         self._config.clear()
         q = self.query
+        schema_dict = OrderedDict()
+        index_dict = {}
 
-        if self.nonempty("config") is None:
         # The table "config" stores configuration
         # parameters which affect the persistent data.
-            q("CREATE TABLE IF NOT EXISTS config ("
-              "  name TEXT NOT NULL PRIMARY KEY,"
-              "  value TEXT)")
-            self._setConfiguration("version", self.VERSION)
-        else:
-            self.migrate()
+        schema_dict['config'] = """CREATE TABLE %s (
+                 name TEXT NOT NULL PRIMARY KEY,
+                 value TEXT)
+            """
 
         # The table "pt" stores a partition table.
-        q("""CREATE TABLE IF NOT EXISTS pt (
+        schema_dict['pt'] = """CREATE TABLE %s (
                  rid INTEGER NOT NULL,
                  nid INTEGER NOT NULL,
                  state INTEGER NOT NULL,
                  PRIMARY KEY (rid, nid))
-          """)
+            """
 
         # The table "trans" stores information on committed transactions.
-        q("""CREATE TABLE IF NOT EXISTS trans (
+        schema_dict['trans'] = """CREATE TABLE %s (
                  partition INTEGER NOT NULL,
                  tid INTEGER NOT NULL,
                  packed BOOLEAN NOT NULL,
@@ -153,38 +153,34 @@ class SQLiteDatabaseManager(DatabaseManager):
                  ext BLOB NOT NULL,
                  ttid INTEGER NOT NULL,
                  PRIMARY KEY (partition, tid))
-          """)
+            """
 
         # The table "obj" stores committed object metadata.
-        q("""CREATE TABLE IF NOT EXISTS obj (
+        schema_dict['obj'] = """CREATE TABLE %s (
                  partition INTEGER NOT NULL,
                  oid INTEGER NOT NULL,
                  tid INTEGER NOT NULL,
                  data_id INTEGER,
                  value_tid INTEGER,
                  PRIMARY KEY (partition, tid, oid))
-          """)
-        q("""CREATE INDEX IF NOT EXISTS _obj_i1 ON
-                 obj(partition, oid, tid)
-          """)
-        q("""CREATE INDEX IF NOT EXISTS _obj_i2 ON
-                 obj(data_id)
-          """)
+            """
+        index_dict['obj'] = (
+            "CREATE INDEX %s ON %s(partition, oid, tid)",
+            "CREATE INDEX %s ON %s(data_id)")
 
         # The table "data" stores object data.
-        q("""CREATE TABLE IF NOT EXISTS data (
+        schema_dict['data'] = """CREATE TABLE %s (
                  id INTEGER PRIMARY KEY AUTOINCREMENT,
                  hash BLOB NOT NULL,
                  compression INTEGER NOT NULL,
                  value BLOB NOT NULL)
-          """)
+            """
         if dedup:
-            q("""CREATE UNIQUE INDEX IF NOT EXISTS _data_i1 ON
-                    data(hash, compression)
-              """)
+            index_dict['data'] = (
+                "CREATE UNIQUE INDEX %s ON %s(hash, compression)",)
 
         # The table "ttrans" stores information on uncommitted transactions.
-        q("""CREATE TABLE IF NOT EXISTS ttrans (
+        schema_dict['ttrans'] = """CREATE TABLE %s (
                  partition INTEGER NOT NULL,
                  tid INTEGER,
                  packed BOOLEAN NOT NULL,
@@ -193,17 +189,28 @@ class SQLiteDatabaseManager(DatabaseManager):
                  description BLOB NOT NULL,
                  ext BLOB NOT NULL,
                  ttid INTEGER NOT NULL)
-          """)
+            """
 
         # The table "tobj" stores uncommitted object metadata.
-        q("""CREATE TABLE IF NOT EXISTS tobj (
+        schema_dict['tobj'] = """CREATE TABLE %s (
                  partition INTEGER NOT NULL,
                  oid INTEGER NOT NULL,
                  tid INTEGER NOT NULL,
                  data_id INTEGER,
                  value_tid INTEGER,
                  PRIMARY KEY (tid, oid))
-          """)
+            """
+
+        if self.nonempty('config') is None:
+            q(schema_dict.pop('config') % 'config')
+            self._setConfiguration('version', self.VERSION)
+        else:
+            self.migrate(schema_dict, index_dict)
+
+        for table, schema in schema_dict.iteritems():
+            q(schema % ('IF NOT EXISTS ' + table))
+            for i, index in enumerate(index_dict.get(table, ()), 1):
+                q(index % ('IF NOT EXISTS _%s_i%s' % (table, i), table))
 
         self._uncommitted_data.update(q("SELECT data_id, count(*)"
             " FROM tobj WHERE data_id IS NOT NULL GROUP BY data_id"))