qa: new testStorageUpgrade

neo/storage/database/mysqldb.py

@@ -944,3 +944,25 @@ class MySQLDatabaseManager(DatabaseManager):
                     sha1(','.join(str(x[1]) for x in r)).digest(),
                     p64(r[-1][1]))
         return 0, ZERO_HASH, ZERO_TID, ZERO_HASH, ZERO_OID
+
+    def _cmdline(self):
+        for x in ('u', self.user), ('p', self.passwd), ('S', self.socket):
+            if x[1]:
+                yield '-%s%s' % x
+        yield self.db
+
+    def dump(self):
+        import subprocess
+        cmd = ['mysqldump', '--compact', '--hex-blob']
+        cmd += self._cmdline()
+        return subprocess.check_output(cmd)
+
+    def restore(self, sql):
+        import subprocess
+        cmd = ['mysql']
+        cmd += self._cmdline()
+        p = subprocess.Popen(cmd, stdin=subprocess.PIPE)
+        p.communicate(sql)
+        retcode = p.wait()
+        if retcode:
+            raise subprocess.CalledProcessError(retcode, cmd)

neo/storage/database/sqlite.py

@@ -694,3 +694,24 @@ class SQLiteDatabaseManager(DatabaseManager):
                     sha1(','.join(str(x[1]) for x in r)).digest(),
                     p64(r[-1][1]))
         return 0, ZERO_HASH, ZERO_TID, ZERO_HASH, ZERO_OID
+
+    def dump(self):
+        main = []
+        data = []
+        for line in self.conn.iterdump():
+            if line.startswith('INSERT '):
+                assert line.endswith(';'), line
+                data.append(line)
+                continue
+            if line.startswith('CREATE TABLE '):
+                # ALTER TABLE adds quotes.
+                create, table, name, tail = line.split(' ', 3)
+                line = ' '.join((create, table, name.strip('"'), tail))
+            main.append(line)
+        assert line == 'COMMIT;', line
+        data.sort()
+        main[-1:-1] = data
+        return '\n'.join(main) + '\n'
+
+    def restore(self, sql):
+        self.conn.executescript(sql)

neo/tests/__init__.py

@@ -170,6 +170,8 @@ def ImporterConfigParser(adapter, zodb, **kw):
 
 class NeoTestBase(unittest.TestCase):
 
+    maxDiff = None
+
     def setUp(self):
         logging.name = self.setupLog()
         unittest.TestCase.setUp(self)

neo/tests/threaded/__init__.py

@@ -652,8 +652,8 @@ class NEOCluster(object):
                        adapter=os.getenv('NEO_TESTS_ADAPTER', 'SQLite'),
                        storage_count=None, db_list=None, clear_databases=True,
                        db_user=DB_USER, db_password='', compress=True,
-                       importer=None, autostart=None, dedup=False):
-        self.name = 'neo_%s' % self._allocate('name',
+                       importer=None, autostart=None, dedup=False, name=None):
+        self.name = name or 'neo_%s' % self._allocate('name',
             lambda: random.randint(0, 100))
         self.compress = compress
         self.num_partitions = partitions

neo/tests/threaded/test.py

@@ -2361,6 +2361,44 @@ class Test(NEOThreadedTest):
                 self.assertEqual(expected,
                     (dm.getLastTID(u64(MAX_TID)), dm.getLastIDs()))
 
+    def testStorageUpgrade(self):
+        path = os.path.join(os.path.dirname(__file__),
+                            self._testMethodName + '-%s',
+                            's%s.sql')
+        dump_dict = {}
+        def switch(s):
+            dm = s.dm
+            dm.commit()
+            dump_dict[s.uuid] = dm.dump()
+            dm.erase()
+            with open(path % (s.getAdapter(), s.uuid)) as f:
+                dm.restore(f.read())
+        with NEOCluster(storage_count=3, partitions=3, replicas=1,
+                        name=self._testMethodName) as cluster:
+            s1, s2, s3 = cluster.storage_list
+            cluster.start(storage_list=(s1,))
+            for s in s2, s3:
+                s.start()
+                self.tic()
+                cluster.neoctl.enableStorageList([s.uuid])
+                cluster.neoctl.tweakPartitionTable()
+            self.tic()
+            nid_list = [s.uuid for s in cluster.storage_list]
+            switch(s3)
+            s3.stop()
+            storage = cluster.getZODBStorage()
+            txn = transaction.Transaction()
+            storage.tpc_begin(txn, p64(85**9)) # partition 1
+            storage.store(p64(0), None, 'foo', '', txn)
+            storage.tpc_vote(txn)
+            storage.tpc_finish(txn)
+            self.tic()
+            switch(s1)
+            switch(s2)
+            cluster.stop()
+            for i, s in zip(nid_list, cluster.storage_list):
+                self.assertMultiLineEqual(s.dm.dump(), dump_dict[i])
+
 
 if __name__ == "__main__":
     unittest.main()