Merge pull request #9 from NextThought/master

UnicodeDecodeError with bushy blob layout and non-ASCII OID

CHANGES.rst

@@ -2,6 +2,12 @@
  Change History
 ================
 
+Unreleased
+==========
+
+- Fixed: A ``UnicodeDecodeError`` could happen for non-ASCII OIDs
+  when using bushy blob layout.
+
 4.0.0b2 (2013-05-14)
 ====================
 

src/ZODB/blob.py

@@ -35,6 +35,7 @@ from ZODB._compat import BytesIO
 from ZODB._compat import Unpickler
 from ZODB._compat import decodebytes
 from ZODB._compat import ascii_bytes
+from ZODB._compat import INT_TYPES
 
 
 if sys.version_info[0] >= 3:
@@ -552,9 +553,14 @@ class BushyLayout(object):
         directories = []
         # Create the bushy directory structure with the least significant byte
         # first
-        for byte in oid.decode():
-            directories.append(
-                '0x%s' % binascii.hexlify(byte.encode()).decode())
+        for byte in ascii_bytes(oid):
+            if isinstance(byte,INT_TYPES): # Py3k iterates byte strings as ints
+                hex_segment_bytes = b'0x' + binascii.hexlify(bytes([byte]))
+                hex_segment_string = hex_segment_bytes.decode('ascii')
+            else:
+                hex_segment_string = '0x%s' % binascii.hexlify(byte)
+            directories.append(hex_segment_string)
+
         return os.path.sep.join(directories)
 
     def path_to_oid(self, path):

src/ZODB/tests/testblob.py

@@ -12,6 +12,7 @@
 #
 ##############################################################################
 from ZODB.blob import Blob
+from ZODB.blob import BushyLayout
 from ZODB.DB import DB
 from ZODB.FileStorage import FileStorage
 from ZODB.tests.testConfig import ConfigTestBase
@@ -138,6 +139,28 @@ class BlobCloneTests(ZODB.tests.util.TestCase):
         # tearDown
         database.close()
 
+
+class BushyLayoutTests(ZODB.tests.util.TestCase):
+
+    def testBushyLayoutOIDToPathUnicode(self):
+        "OID-to-path should produce valid results given non-ASCII byte strings"
+        non_ascii_oid = b'>\xf1<0\xe9Q\x99\xf0'
+        # The argument should already be bytes;
+        # os.path.sep is native string type under both 2 and 3
+        # binascii.hexlify takes bytes and produces bytes under both py2 and py3
+        # the result should be the native string type
+        oid_as_path = BushyLayout().oid_to_path(non_ascii_oid)
+        self.assertEqual(
+            oid_as_path,
+            '0x3e/0xf1/0x3c/0x30/0xe9/0x51/0x99/0xf0')
+
+        # the reverse holds true as well
+        path_as_oid = BushyLayout().path_to_oid(oid_as_path)
+        self.assertEqual(
+            path_as_oid,
+            non_ascii_oid )
+
+
 class BlobTestBase(ZODB.tests.StorageTestBase.StorageTestBase):
 
     def setUp(self):
@@ -779,6 +802,7 @@ def test_suite():
     suite = unittest.TestSuite()
     suite.addTest(unittest.makeSuite(ZODBBlobConfigTest))
     suite.addTest(unittest.makeSuite(BlobCloneTests))
+    suite.addTest(unittest.makeSuite(BushyLayoutTests))
     suite.addTest(doctest.DocFileSuite(
         "blob_basic.txt", "blob_consume.txt", "blob_tempdir.txt",
         "blobstorage_packing.txt",