Commit b72a79e6 authored by Kirill Smelkov's avatar Kirill Smelkov

fixup! ZBigFile: Add ZBlk format option 'h' (heuristic) (1)

Fix tests that started to fail with e.g,:

    bigarray/tests/test_arrayzodb.py::test_zbigarray FAILED

    ===================================================================== FAILURES =====================================================================
    __________________________________________________________________ test_zbigarray __________________________________________________________________

        @func
        def test_zbigarray():
            root = testdb.dbopen()
            defer(lambda: dbclose(root))

            root['zarray'] = ZBigArray((16*1024*1024,), uint8)
            transaction.commit()

            dbclose(root)

            root = testdb.dbopen()
            A = root['zarray']

            assert isinstance(A, ZBigArray)
            assert A.shape  == (16*1024*1024,)
            assert A.dtype  == dtype(uint8)

            assert all(A[:] == 0)

            a = A[:]
            a[1] = 1
            a[3] = 3
            a[5] = 5
            a[-1] = 99

            b = A[:]
            assert (b[0],b[1]) == (0,1)
            assert (b[2],b[3]) == (0,3)
            assert (b[4],b[5]) == (0,5)
            assert all(b[6:-1] == 0)
            assert b[-1] == 99

            # abort - should forget all changes
            transaction.abort()
            assert all(a[:] == 0)
            assert all(b[:] == 0)
            assert all(A[:] == 0)

            # now modify again and commit
            a[33] = 33
            a[-2] = 98
            assert all(b[:33] == 0)
            assert b[33] == 33
            assert all(b[33+1:-2] == 0)
            assert b[-2] == 98
            assert b[-1] == 0

    >       transaction.commit()

    bigarray/tests/test_arrayzodb.py:95:
    _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    ../venv/z-dev/lib/python2.7/site-packages/transaction/_manager.py:252: in commit
        return self.manager.commit()
    ../venv/z-dev/lib/python2.7/site-packages/transaction/_manager.py:131: in commit
        return self.get().commit()
    ../venv/z-dev/lib/python2.7/site-packages/transaction/_transaction.py:311: in commit
        reraise(t, v, tb)
    ../venv/z-dev/lib/python2.7/site-packages/transaction/_transaction.py:302: in commit
        self._commitResources()
    ../venv/z-dev/lib/python2.7/site-packages/transaction/_transaction.py:447: in _commitResources
        reraise(t, v, tb)
    ../venv/z-dev/lib/python2.7/site-packages/transaction/_transaction.py:421: in _commitResources
        rm.commit(self)
    bigfile/file_zodb.py:835: in commit
        self.zfileh.dirty_writeout(WRITEOUT_STORE)
    bigfile/_file_zodb.pyx:91: in wendelin.bigfile._file_zodb._ZBigFile.storeblk
        def storeblk(self, blk, buf):   return self.zself.storeblk(blk, buf)
    bigfile/file_zodb.py:551: in storeblk
        zblk_fmt = self._zblk_fmt_heuristic(zblk, blk, buf)
    _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

    self = <wendelin.bigfile.file_zodb.ZBigFile object at 0x7f297c99d8d0 oid 0x2 in <ZODB.Connection.Connection object at 0x7f297c99e4d0>>, zblk = None
    blk = 7L, buf = <read-only buffer ptr 0x(nil), size 0 at 0x7f297c99e770>

        def _zblk_fmt_heuristic(self, zblk, blk, buf):
            if _is_appending(zblk, buf):
                if not zblk and blk > 0:  # is new zblk?
                    # Set previous filled-up ZBlk to ZBlk0 for fast reads
                    previous_blk = blk - 1
                    previous_zblk = self.blktab.get(previous_blk)
    >               self._setzblk(previous_blk, previous_zblk, previous_zblk.loadblkdata(), "ZBlk0")
    E               AttributeError: 'NoneType' object has no attribute 'loadblkdata'

There is no guarantee that previous block is not hole.
parent 2bc0e789
...@@ -584,7 +584,8 @@ class ZBigFile(LivePersistent): ...@@ -584,7 +584,8 @@ class ZBigFile(LivePersistent):
# Set previous filled-up ZBlk to ZBlk0 for fast reads # Set previous filled-up ZBlk to ZBlk0 for fast reads
previous_blk = blk - 1 previous_blk = blk - 1
previous_zblk = self.blktab.get(previous_blk) previous_zblk = self.blktab.get(previous_blk)
self._setzblk(previous_blk, previous_zblk, previous_zblk.loadblkdata(), "ZBlk0") if previous_zblk is not None:
self._setzblk(previous_blk, previous_zblk, previous_zblk.loadblkdata(), "ZBlk0")
return "ZBlk1" return "ZBlk1"
else: # it's changing else: # it's changing
# kirr: "to support sporadic small changes over initial big fillup [...] # kirr: "to support sporadic small changes over initial big fillup [...]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment