Commit a94ded9b authored by Levin Zimmermann's avatar Levin Zimmermann

ZBigFile: Allow explicitly overridding global ZBlk fmt

With this patch it's possible now to explicitly set the ZBlk format of a
single ZBigFile. Before this patch it was only possible to set the ZBlk
format globally, for all ZBigFile. This new feature is useful if we have a
project where we would like to have mixed ZBlk formats, e.g. some
ZBigFile with ZBlk0 and some ZBigFile with ZBlk1.
parent 885b3556
...@@ -509,17 +509,26 @@ class ZBigFile(LivePersistent): ...@@ -509,17 +509,26 @@ class ZBigFile(LivePersistent):
# NOTE Live: don't allow us to go to ghost state not to loose ._v_file # NOTE Live: don't allow us to go to ghost state not to loose ._v_file
# which DataManager _ZBigFileH refers to. # which DataManager _ZBigFileH refers to.
def __init__(self, blksize): def __init__(self, blksize, zblk_fmt=""):
LivePersistent.__init__(self) LivePersistent.__init__(self)
self.__setstate__((blksize, LOBTree())) # NOTE L enough for blk_t self.__setstate__((blksize, LOBTree(), zblk_fmt)) # NOTE L enough for blk_t
self.zblk_fmt = zblk_fmt # Evoke check if zblk_fmt is valid
# state is (.blksize, .blktab) # state is (.blksize, .blktab, .zblk_fmt)
def __getstate__(self): def __getstate__(self):
return (self.blksize, self.blktab) return (self.blksize, self.blktab, self.zblk_fmt)
def __setstate__(self, state): def __setstate__(self, state):
self.blksize, self.blktab = state state_length = len(state)
# NOTE set _zblk_fmt instead of zblk_fmt to avoid check => ↑ performance
if state_length == 2: # BBB
self.blksize, self.blktab = state
self._zblk_fmt = ""
elif state_length == 3:
self.blksize, self.blktab, self._zblk_fmt = state
else:
raise RuntimeError("E: Unexpected state length: %s" % state)
self._v_file = _ZBigFile._new(self, self.blksize) self._v_file = _ZBigFile._new(self, self.blksize)
self._v_filehset = WeakSet() self._v_filehset = WeakSet()
...@@ -546,7 +555,7 @@ class ZBigFile(LivePersistent): ...@@ -546,7 +555,7 @@ class ZBigFile(LivePersistent):
# store data dirty page -> ZODB obj # store data dirty page -> ZODB obj
def storeblk(self, blk, buf): def storeblk(self, blk, buf):
zblk = self.blktab.get(blk) zblk = self.blktab.get(blk)
zblk_type_write = ZBlk_fmt_registry[ZBlk_fmt_write] zblk_type_write = ZBlk_fmt_registry[self.zblk_fmt or ZBlk_fmt_write]
# if zblk was absent or of different type - we (re-)create it anew # if zblk was absent or of different type - we (re-)create it anew
if zblk is None or \ if zblk is None or \
type(zblk) is not zblk_type_write: type(zblk) is not zblk_type_write:
...@@ -605,6 +614,19 @@ class ZBigFile(LivePersistent): ...@@ -605,6 +614,19 @@ class ZBigFile(LivePersistent):
return fileh return fileh
# zblk_fmt is a property to implement check if user declared zblk_fmt
# is valid.
@property
def zblk_fmt(self):
return self._zblk_fmt
@zblk_fmt.setter
def zblk_fmt(self, zblk_fmt):
if zblk_fmt and zblk_fmt not in ZBlk_fmt_registry:
raise RuntimeError('E: Unknown ZBlk format %r' % zblk_fmt)
self._zblk_fmt = zblk_fmt
# _default_use_wcfs returns whether default virtmem setting is to use wcfs or not. # _default_use_wcfs returns whether default virtmem setting is to use wcfs or not.
@staticmethod @staticmethod
def _default_use_wcfs(): def _default_use_wcfs():
......
...@@ -690,3 +690,29 @@ def test_bigfile_zblk1_zdata_reuse(): ...@@ -690,3 +690,29 @@ def test_bigfile_zblk1_zdata_reuse():
assert len(zdata_v1) == len(zdata_v2) assert len(zdata_v1) == len(zdata_v2)
for i in range(len(zdata_v1)): for i in range(len(zdata_v1)):
assert zdata_v1[i] is zdata_v2[i] assert zdata_v1[i] is zdata_v2[i]
# test that explicitly setting the ZBlk format works
@func
def test_bigfile_set_zblk_fmt():
# ensure ZBlk_fmt_write is ZBlk0 during this test,
# so that we can be sure the global default is overridden
# by an explicitly set value during ZBigFile initialization
fmt_write_save = file_zodb.ZBlk_fmt_write
file_zodb.ZBlk_fmt_write = 'ZBlk0'
def _():
file_zodb.ZBlk_fmt_write = fmt_write_save
defer(_)
root = dbopen()
defer(lambda: dbclose(root))
root['zfile7'] = f = ZBigFile(blksize, zblk_fmt="ZBlk1")
transaction.commit()
fh = f.fileh_open()
vma = fh.mmap(0, blen)
Blk(vma, 0)[:] = 1
transaction.commit()
assert type(f.blktab[0]) is file_zodb.ZBlk1
...@@ -3,7 +3,10 @@ ...@@ -3,7 +3,10 @@
package pycompat package pycompat
import ( import (
"fmt"
"math/big" "math/big"
pickle "github.com/kisielk/og-rek"
) )
// Int64 tries to convert unpickled Python value to int64. // Int64 tries to convert unpickled Python value to int64.
...@@ -23,3 +26,21 @@ func Int64(xv interface{}) (v int64, ok bool) { ...@@ -23,3 +26,21 @@ func Int64(xv interface{}) (v int64, ok bool) {
return 0, false return 0, false
} }
// Xstrbytes verifies and extacts str|bytes from unpickled value.
func Xstrbytes(x interface{}) (string, error) {
var s string
switch x := x.(type) {
default:
return "", fmt.Errorf("expect str|bytes; got %T", x)
case string:
s = x
case pickle.Bytes:
s = string(x)
}
return s, nil
}
...@@ -354,32 +354,37 @@ func (zb *ZBlk1) LoadBlkData(ctx context.Context) (_ []byte, _ zodb.Tid, err err ...@@ -354,32 +354,37 @@ func (zb *ZBlk1) LoadBlkData(ctx context.Context) (_ []byte, _ zodb.Tid, err err
type ZBigFile struct { type ZBigFile struct {
zodb.Persistent zodb.Persistent
// state: (.blksize, .blktab) // state: (.blksize, .blktab, .zblk_fmt)
blksize int64 blksize int64
blktab *btree.LOBTree // {} blk -> ZBlk*(blkdata) blktab *btree.LOBTree // {} blk -> ZBlk*(blkdata)
zblk_fmt string
} }
type zBigFileState ZBigFile // hide state methods from public API type zBigFileState ZBigFile // hide state methods from public API
// DropState implements zodb.Ghostable. // DropState implements zodb.Ghostable.
func (bf *zBigFileState) DropState() { func (bf *zBigFileState) DropState() {
bf.blksize = 0 bf.blksize = 0
bf.blktab = nil bf.blktab = nil
bf.zblk_fmt = ""
} }
// PyGetState implements zodb.PyStateful. // PyGetState implements zodb.PyStateful.
func (bf *zBigFileState) PyGetState() interface{} { func (bf *zBigFileState) PyGetState() interface{} {
return pickle.Tuple{bf.blksize, bf.blktab} return pickle.Tuple{bf.blksize, bf.blktab, bf.zblk_fmt}
} }
// PySetState implements zodb.PyStateful. // PySetState implements zodb.PyStateful.
func (bf *zBigFileState) PySetState(pystate interface{}) (err error) { func (bf *zBigFileState) PySetState(pystate interface{}) (err error) {
t, ok := pystate.(pickle.Tuple) t, ok := pystate.(pickle.Tuple)
if !ok { if !ok {
return fmt.Errorf("expect [2](); got %s", xzodb.TypeOf(pystate)) return fmt.Errorf("expect [2|3](); got %s", xzodb.TypeOf(pystate))
} }
if len(t) != 2 { // BBB: we either accept data before adding zblk_fmt to state
return fmt.Errorf("expect [2](); got [%d]()", len(t)) // (lent==2) or data after adding zblk_fmt to state (lent==3).
lent := len(t)
if lent != 2 && lent != 3 {
return fmt.Errorf("expect [2|3](); got [%d]()", len(t))
} }
blksize, ok := pycompat.Int64(t[0]) blksize, ok := pycompat.Int64(t[0])
...@@ -397,6 +402,15 @@ func (bf *zBigFileState) PySetState(pystate interface{}) (err error) { ...@@ -397,6 +402,15 @@ func (bf *zBigFileState) PySetState(pystate interface{}) (err error) {
bf.blksize = blksize bf.blksize = blksize
bf.blktab = blktab bf.blktab = blktab
if lent != 2 {
zblk_fmt, err := pycompat.Xstrbytes(t[2])
if err != nil {
return fmt.Errorf("zblk_fmt: expect str; got %s", xzodb.TypeOf(t[2]))
}
bf.zblk_fmt = zblk_fmt
}
return nil return nil
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment