Commit 70ea8573 authored by Kirill Smelkov's avatar Kirill Smelkov

bigfile/zodb: Prepare to have several ZBlk formats

- current ZBlk becomes format 0
- write format can be selected via WENDELIN_CORE_ZBLK_FMT env var
- upon writing a block we always make sure we write it in current write
  format - so if a block was previously written in one format, it could
  be changed on the next write.
- tox is prepared to test all write formats (so far only ZBlk0 there).

The reason is - in the next patch we'll introduce another format for
blocks which is optimized for small changes.
parent a7b7c294
...@@ -37,6 +37,7 @@ from BTrees.LOBTree import LOBTree ...@@ -37,6 +37,7 @@ from BTrees.LOBTree import LOBTree
from zope.interface import implementer from zope.interface import implementer
from ZODB.Connection import Connection from ZODB.Connection import Connection
from weakref import WeakSet from weakref import WeakSet
import os
# TODO document that first data access must be either after commit or Connection.add # TODO document that first data access must be either after commit or Connection.add
...@@ -98,8 +99,13 @@ class ZBlkBase(Persistent): ...@@ -98,8 +99,13 @@ class ZBlkBase(Persistent):
Persistent._p_invalidate(self) Persistent._p_invalidate(self)
# data of 1 file block as stored in ZODB
class ZBlk(ZBlkBase): # ZBlk storage formats
# NOTE once established formats do not change on disk
# ZBlk format 0: raw bytes
class ZBlk0(ZBlkBase):
# ._v_blkdata - bytes # ._v_blkdata - bytes
__slots__ = ('_v_blkdata',) __slots__ = ('_v_blkdata',)
...@@ -158,7 +164,7 @@ class ZBlk(ZBlkBase): ...@@ -158,7 +164,7 @@ class ZBlk(ZBlkBase):
# DB (through pickle) loads data to memory # DB (through pickle) loads data to memory
# DB -> ._v_blkdata (-> memory-page) # DB -> ._v_blkdata (-> memory-page)
def __setstate__(self, state): def __setstate__(self, state):
super(ZBlk, self).__init__() super(ZBlk0, self).__init__()
self._v_blkdata = state self._v_blkdata = state
# ZBlk as initially created (empty placeholder) # ZBlk as initially created (empty placeholder)
...@@ -166,6 +172,23 @@ class ZBlk(ZBlkBase): ...@@ -166,6 +172,23 @@ class ZBlk(ZBlkBase):
self.__setstate__(None) self.__setstate__(None)
# backward compatibility (early versions wrote ZBlk0 named as ZBlk)
ZBlk = ZBlk0
# format-name -> blk format type
ZBlk_fmt_registry = {
'ZBlk0': ZBlk0,
}
# format for updated blocks
ZBlk_fmt_write = os.environ.get('WENDELIN_CORE_ZBLK_FMT', 'ZBlk0')
if ZBlk_fmt_write not in ZBlk_fmt_registry:
raise RuntimeError('E: Unknown ZBlk format %r' % ZBlk_fmt_write)
# ----------------------------------------
# helper for ZBigFile - just redirect loadblk/storeblk back # helper for ZBigFile - just redirect loadblk/storeblk back
# (because it is not possible to inherit from both Persistent and BigFile at # (because it is not possible to inherit from both Persistent and BigFile at
# the same time - see below) # the same time - see below)
...@@ -218,7 +241,7 @@ class ZBigFile(LivePersistent): ...@@ -218,7 +241,7 @@ class ZBigFile(LivePersistent):
# file is split into blocks; each block is stored as separate object in the DB # file is split into blocks; each block is stored as separate object in the DB
# #
# .blksize # .blksize
# .blktab {} blk -> ZBlk(blkdata) # .blktab {} blk -> ZBlk*(blkdata)
# ._v_file _ZBigFile helper # ._v_file _ZBigFile helper
# ._v_filehset weakset( _ZBigFileH ) that we created # ._v_filehset weakset( _ZBigFileH ) that we created
...@@ -263,8 +286,11 @@ class ZBigFile(LivePersistent): ...@@ -263,8 +286,11 @@ class ZBigFile(LivePersistent):
# store data dirty page -> ZODB obj # store data dirty page -> ZODB obj
def storeblk(self, blk, buf): def storeblk(self, blk, buf):
zblk = self.blktab.get(blk) zblk = self.blktab.get(blk)
if zblk is None: zblk_type_write = ZBlk_fmt_registry[ZBlk_fmt_write]
zblk = self.blktab[blk] = ZBlk() # if zblk was absent or of different type - we (re-)create it anew
if zblk is None or \
type(zblk) is not zblk_type_write:
zblk = self.blktab[blk] = zblk_type_write()
zblk.setblkdata(buf) zblk.setblkdata(buf)
zblk._p_changed = True # if zblk was already in DB: _p_state -> CHANGED zblk._p_changed = True # if zblk was already in DB: _p_state -> CHANGED
......
# wendelin.core | tox setup # wendelin.core | tox setup
[tox] [tox]
envlist = py27-ZODB3-{fs,zeo,neo}-{numpy18,numpy19}, {py27,py34}-ZODB4-{fs,zeo}-{numpy18,numpy19} envlist = py27-ZODB3-{zblk0}-{fs,zeo,neo}-{numpy18,numpy19}, {py27,py34}-ZODB4-{zblk0}-{fs,zeo}-{numpy18,numpy19}
# (NOTE ZODB3 does not work on python3) # (NOTE ZODB3 does not work on python3)
# (NOTE NEO does not work on ZODB4) # (NOTE NEO does not work on ZODB4)
...@@ -31,6 +31,8 @@ setenv = ...@@ -31,6 +31,8 @@ setenv =
zeo: WENDELIN_CORE_TEST_DB=<zeo> zeo: WENDELIN_CORE_TEST_DB=<zeo>
neo: WENDELIN_CORE_TEST_DB=<neo> neo: WENDELIN_CORE_TEST_DB=<neo>
zblk0: WENDELIN_CORE_ZBLK_FMT=ZBlk0
commands= {envpython} setup.py test commands= {envpython} setup.py test
# XXX setenv = TMPDIR = ... ? (so that /tmp is not on tmpfs and we don't run out of memory on bench) # XXX setenv = TMPDIR = ... ? (so that /tmp is not on tmpfs and we don't run out of memory on bench)
# + {envpython} setup.py bench (?) # + {envpython} setup.py bench (?)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment