Fix a few issues with ZODB5

In the Importer storage backend, the repickler code never really worked with
ZODB 5 (use of protocol > 1), and now the test does not pass anymore.

The other issues caused by ZODB commit 12ee41c47310156027a674932df34b60de86ba36
are fixed:

  TypeError: list indices must be integers, not binary

  ValueError: unsupported pickle protocol: 3

Although not necessary as long as we don't support Python 3,
this commit also replaces `str` by `bytes` in a few places.

importer.conf

@@ -14,6 +14,8 @@
 # directly to a NEO cluster with replicas or several storage nodes.
 # Importer backend can only be used with a single storage node.
 #
+# WARNING: Merging several DB only works if they were only used with ZODB < 5.
+#
 # Here is how to proceed once this file is ready:
 # 1. Restart ZODB clients to connect to new NEO cluster (not started yet).
 # 2. Start NEO cluster (use 'neoctl -a <admin> start' command if necessary).

neo/client/__init__.py

@@ -44,7 +44,7 @@ def patch():
         # <patch>
         serial = self._storage.tpc_finish(transaction, callback)
         if serial is not None:
-            assert isinstance(serial, str), repr(serial)
+            assert isinstance(serial, bytes), repr(serial)
             for oid_iterator in (self._modified, self._creating):
                 for oid in oid_iterator:
                     obj = self._cache.get(oid, None)

neo/client/app.py

@@ -273,7 +273,8 @@ class Application(ThreadedApplication):
     def _askStorageForRead(self, object_id, packet, askStorage=None):
         cp = self.cp
         pt = self.pt
-        if type(object_id) is str:
+        # BBB: On Py2, it can be a subclass of bytes (binary from zodbpickle).
+        if isinstance(object_id, bytes):
             object_id = pt.getPartition(object_id)
         if askStorage is None:
             askStorage = self._askStorage

neo/client/transactions.py

@@ -117,7 +117,7 @@ class Transaction(object):
         if uuid_list:
             return
         del self.data_dict[oid]
-        if type(data) is str:
+        if type(data) is bytes:
             size = len(data)
             self.data_size -= size
             size += self.cache_size

neo/lib/util.py

@@ -115,7 +115,7 @@ def add64(packed, offset):
 def dump(s):
     """Dump a binary string in hex."""
     if s is not None:
-        if isinstance(s, str):
+        if isinstance(s, bytes):
             return b2a_hex(s)
         return repr(s)
 

neo/storage/database/importer.py

@@ -146,7 +146,7 @@ class Repickler(pickle.Unpickler):
         args = self.stack[k+1:]
         self.stack[k:] = self._obj(klass, *args),
 
-    del dispatch[pickle.NEWOBJ] # ZODB has never used protocol 2
+    del dispatch[pickle.NEWOBJ] # ZODB < 5 has never used protocol 2
 
     @_noload
     def find_class(self, args):
@@ -221,7 +221,7 @@ class ZODB(object):
                 oid = u64(obj[0])
                 # If this oid pointed to a mount point, drop 2nd item because
                 # it's probably different than the real class of the new oid.
-            elif isinstance(obj, str):
+            elif isinstance(obj, bytes):
                 oid = u64(obj)
             else:
                 raise NotImplementedError(
@@ -232,7 +232,7 @@ class ZODB(object):
                 if not self.shift_oid:
                     return obj # common case for root db
             oid = p64(oid + self.shift_oid)
-            return oid if isinstance(obj, str) else (oid, obj[1])
+            return oid if isinstance(obj, bytes) else (oid, obj[1])
         self.repickle = Repickler(map_oid)
         return self.repickle(data)
 

neo/tests/__init__.py

@@ -29,7 +29,10 @@ import MySQLdb
 import transaction
 
 from cStringIO import StringIO
-from cPickle import Unpickler
+try:
+    from ZODB._compat import Unpickler
+except ImportError:
+    from cPickle import Unpickler
 from functools import wraps
 from inspect import isclass
 from .mock import Mock

neo/tests/threaded/testImporter.py

@@ -19,12 +19,14 @@ from cStringIO import StringIO
 from itertools import islice, izip_longest
 import os, shutil, unittest
 import neo, transaction, ZODB
+from neo.client.exception import NEOPrimaryMasterLost
 from neo.lib import logging
 from neo.lib.util import u64
 from neo.storage.database.importer import Repickler
 from ..fs2zodb import Inode
-from .. import getTempDirectory
+from .. import expectedFailure, getTempDirectory
 from . import NEOCluster, NEOThreadedTest
+from ZODB import serialize
 from ZODB.FileStorage import FileStorage
 
 
@@ -232,6 +234,10 @@ class ImporterTests(NEOThreadedTest):
                 for x, y, z in os.walk(src_root)))
             t.commit()
 
+    if getattr(serialize, '_protocol', 1) > 1:
+        # XXX: With ZODB5, we should at least keep a working test that does not
+        #      merge several DB.
+        test = expectedFailure(NEOPrimaryMasterLost)(test)
 
 if __name__ == "__main__":
     unittest.main()