Commit 952959aa authored by Kirill Smelkov's avatar Kirill Smelkov

go/zodb/fs1: Test FileStorage on all py2/py3 ZODB kinds of data we care about

Previously we were testing FileStorage/go only with data generated by
python2 and pickle protocol=2. However even on py2 there are more pickle
protocols that are in use, and also there is python3.

-> Modernize py/gen-testdata to use run_with_all_zodb_pickle_kinds
   that was recently added as part of nexedi/zodbtools@f9d36ba7
   and generate test data with both python2 and python3. It is handy to
   use py2py3-venv(*) to prepare python environment to do that.

   Adjust tests on Go side to verify how FileStorage handles all generated zkinds.

py2_pickle1, py2_pickle2 and py2_pickle3 are handled well.
Tests for py3_pickle3 currently fail and so are marked with "xfail".

We will fix tests for py3_pickle3 in follow-up patches.

Old testdata are not yet removed because e.g. fs1tools and zodbdump
tests depend on them. We will remove old fs1 testdata after adjusting
tests in dependent packages step-by-step.

(*) see nexedi/zodbtools@fac2f190
parent e348423c
// Copyright (C) 2017-2021 Nexedi SA and Contributors.
// Copyright (C) 2017-2024 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -33,6 +33,17 @@ import (
"lab.nexedi.com/kirr/go123/exc"
)
// ztestdataReg maintains registry of all entries under testdata/ .
var ztestdataReg = xtesting.ZTestDataRegistry[_TestDataOK]{}
type ZTestData = xtesting.ZTestData[_TestDataOK]
// _TestDataOK describes expected data for one entry under testdata/ .
type _TestDataOK struct {
_1fs_indexTopPos int64 // topPos of 1fs.index
_1fs_indexEntryv []indexEntry // index content
_1fs_dbEntryv []dbEntry // database content
}
// one database transaction record
type dbEntry struct {
Header TxnHeader
......@@ -73,21 +84,33 @@ func xfsopenopt(t testing.TB, path string, opt *zodb.DriverOptions) (*FileStorag
}
func TestEmptyDB(t *testing.T) {
fs, _ := xfsopen(t, "testdata/empty.fs")
ztestdataReg.RunWithEach(t, _TestEmptyDB)
}
func _TestEmptyDB(t *testing.T, z *ZTestData) {
if z.Kind == "py3_pickle3" {
t.Skip("xfail")
}
fs, _ := xfsopen(t, z.Path("empty.fs"))
defer exc.XRun(fs.Close)
xtesting.DrvTestEmptyDB(t, fs)
}
func TestLoad(t *testing.T) {
fs, _ := xfsopen(t, "testdata/1.fs")
ztestdataReg.RunWithEach(t, _TestLoad)
}
func _TestLoad(t *testing.T, z *ZTestData) {
if z.Kind == "py3_pickle3" {
t.Skip("xfail")
}
fs, _ := xfsopen(t, z.Path("1.fs"))
defer exc.XRun(fs.Close)
// NOTE don't use xtesting.LoadDBHistory here - it is itself tested
// with the assumption that fs1.Load and fs1.Iterate work correctly.
// Use what testdata generator gave use with what to expect.
txnv := []xtesting.Txn{}
for _, dbe := range _1fs_dbEntryv {
for _, dbe := range z.Misc._1fs_dbEntryv {
txn := xtesting.Txn{Header: &zodb.TxnInfo{
Tid: dbe.Header.Tid,
......@@ -221,12 +244,20 @@ func testIterate(t *testing.T, fs *FileStorage, tidMin, tidMax zodb.Tid, expectv
// TODO -> xtesting
func TestIterate(t *testing.T) {
fs, _ := xfsopen(t, "testdata/1.fs")
ztestdataReg.RunWithEach(t, _TestIterate)
}
func _TestIterate(t *testing.T, z *ZTestData) {
if z.Kind == "py3_pickle3" {
t.Skip("xfail")
}
zz := z.Misc
fs, _ := xfsopen(t, z.Path("1.fs"))
defer exc.XRun(fs.Close)
// all []tids in test database
tidv := []zodb.Tid{}
for _, dbe := range _1fs_dbEntryv {
for _, dbe := range zz._1fs_dbEntryv {
tidv = append(tidv, dbe.Header.Tid)
}
......@@ -248,17 +279,23 @@ func TestIterate(t *testing.T) {
}
//fmt.Printf("%d%+d .. %d%+d\t -> %d steps\n", i, ii-1, j, jj-1, nsteps)
testIterate(t, fs, tmin, tmax, _1fs_dbEntryv[i + ii/2:][:nsteps])
testIterate(t, fs, tmin, tmax, zz._1fs_dbEntryv[i + ii/2:][:nsteps])
}}
}}
// also check 0..tidMax
testIterate(t, fs, 0, zodb.TidMax, _1fs_dbEntryv[:])
testIterate(t, fs, 0, zodb.TidMax, zz._1fs_dbEntryv[:])
}
// TODO -> xtesting
func BenchmarkIterate(b *testing.B) {
fs, _ := xfsopen(b, "testdata/1.fs")
ztestdataReg.BenchWithEach(b, _BenchmarkIterate)
}
func _BenchmarkIterate(b *testing.B, z *ZTestData) {
if z.Kind == "py3_pickle3" {
b.Skip("xfail")
}
fs, _ := xfsopen(b, z.Path("1.fs"))
defer exc.XRun(fs.Close)
ctx := context.Background()
......@@ -307,12 +344,19 @@ func TestWatch(t *testing.T) {
// TestOpenRecovery verifies how Open handles data file with not-finished voted
// transaction in the end.
func TestOpenRecovery(t *testing.T) {
ztestdataReg.RunWithEach(t, _TestOpenRecovery)
}
func _TestOpenRecovery(t *testing.T, z *ZTestData) {
if z.Kind == "py3_pickle3" {
t.Skip("xfail")
}
X := exc.Raiseif
main, err := ioutil.ReadFile("testdata/1.fs"); X(err)
index, err := ioutil.ReadFile("testdata/1.fs.index"); X(err)
headOk := _1fs_dbEntryv[len(_1fs_dbEntryv)-1].Header.Tid
topPos := int64(_1fs_indexTopPos)
voteTail, err := ioutil.ReadFile("testdata/1voted.tail"); X(err)
zz := z.Misc
main, err := ioutil.ReadFile(z.Path("1.fs")); X(err)
index, err := ioutil.ReadFile(z.Path("1.fs.index")); X(err)
headOk := zz._1fs_dbEntryv[len(zz._1fs_dbEntryv)-1].Header.Tid
topPos := int64(zz._1fs_indexTopPos)
voteTail, err := ioutil.ReadFile(z.Path("1voted.tail")); X(err)
workdir := xworkdir(t)
ctx := context.Background()
......@@ -381,7 +425,13 @@ func TestOpenRecovery(t *testing.T) {
// FileStorage/py.deleteObject allows to create whiteouts instead of raising
// POSKeyError.
func TestLoadWhiteout(t *testing.T) {
fs, _ := xfsopen(t, "testdata/whiteout.fs")
ztestdataReg.RunWithEach(t, _TestLoadWhiteout)
}
func _TestLoadWhiteout(t *testing.T, z *ZTestData) {
if z.Kind == "py3_pickle3" {
t.Skip("xfail")
}
fs, _ := xfsopen(t, z.Path("whiteout.fs"))
defer exc.XRun(fs.Close)
xid := zodb.Xid{At: zodb.Tid(0x17), Oid: zodb.Oid(1)}
......
// Copyright (C) 2017-2019 Nexedi SA and Contributors.
// Copyright (C) 2017-2024 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -19,7 +19,8 @@
package fs1
//go:generate ./py/gen-testdata
//go:generate python2 py/gen-testdata
//go:generate python3 py/gen-testdata
import (
"context"
......@@ -183,61 +184,80 @@ func TestIndexSaveLoad(t *testing.T) {
// {0xb000000000000000, 0x7fffffffffffffff}, // will cause 'entry position too large'
}
var _1fs_index = func() *Index {
func (z *_TestDataOK) _1fs_index() *Index {
idx := IndexNew()
idx.TopPos = _1fs_indexTopPos
setIndex(idx, _1fs_indexEntryv[:])
idx.TopPos = z._1fs_indexTopPos
setIndex(idx, z._1fs_indexEntryv)
return idx
}()
}
// test that we can correctly load index data as saved by zodb/py
func TestIndexLoadFromPy(t *testing.T) {
fsiPy, err := LoadIndexFile("testdata/1.fs.index")
ztestdataReg.RunWithEach(t, _TestIndexLoadFromPy)
}
func _TestIndexLoadFromPy(t *testing.T, z *ZTestData) {
if z.Kind == "py3_pickle3" {
t.Skip("xfail")
}
fsiPy, err := LoadIndexFile(z.Path("1.fs.index"))
if err != nil {
t.Fatal(err)
}
checkIndexEqual(t, "index load", fsiPy, _1fs_index)
checkIndexEqual(t, "index load", fsiPy, z.Misc._1fs_index())
}
// test zodb/py can read index data as saved by us
func TestIndexSaveToPy(t *testing.T) {
ztestdataReg.RunWithEach(t, _TestIndexSaveToPy)
}
func _TestIndexSaveToPy(t *testing.T, z *ZTestData) {
xtesting.WithEachPy(t, func(t *testing.T) {
if strings.HasSuffix(t.Name(), "/py3") {
t.Skip("xfail")
}
xtesting.NeedPy(t, "ZODB")
workdir := xworkdir(t)
_1fs_index := z.Misc._1fs_index()
err := _1fs_index.SaveFile(workdir + "/1.fs.index")
if err != nil {
t.Fatal(err)
}
// now ask python part to compare testdata and saved-by-us index
cmd := exec.Command("./py/indexcmp", "testdata/1.fs.index", workdir+"/1.fs.index")
cmd := exec.Command("./py/indexcmp", z.Path("1.fs.index"), workdir+"/1.fs.index")
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err = cmd.Run()
if err != nil {
t.Fatalf("zodb/py read/compare index: %v", err)
}
})
}
func TestIndexBuildVerify(t *testing.T) {
index, err := BuildIndexForFile(context.Background(), "testdata/1.fs", nil)
ztestdataReg.RunWithEach(t, _TestIndexBuildVerify)
}
func _TestIndexBuildVerify(t *testing.T, z *ZTestData) {
index, err := BuildIndexForFile(context.Background(), z.Path("1.fs"), nil)
if err != nil {
t.Fatalf("index build: %v", err)
}
if !index.Equal(_1fs_index) {
if !index.Equal(z.Misc._1fs_index()) {
t.Fatal("computed index differ from expected")
}
_, err = index.VerifyForFile(context.Background(), "testdata/1.fs", -1, nil)
_, err = index.VerifyForFile(context.Background(), z.Path("1.fs"), -1, nil)
if err != nil {
t.Fatalf("index verify: %v", err)
}
pos0, _ := index.Get(0)
index.Set(0, pos0+1)
_, err = index.VerifyForFile(context.Background(), "testdata/1.fs", -1, nil)
_, err = index.VerifyForFile(context.Background(), z.Path("1.fs"), -1, nil)
if err == nil {
t.Fatalf("index verify: expected error after tweak")
}
......@@ -245,9 +265,15 @@ func TestIndexBuildVerify(t *testing.T) {
func BenchmarkIndexLoad(b *testing.B) {
ztestdataReg.BenchWithEach(b, _BenchmarkIndexLoad)
}
func _BenchmarkIndexLoad(b *testing.B, z *ZTestData) {
if z.Kind == "py3_pickle3" {
b.Skip("xfail")
}
// FIXME small testdata/1.fs is not representative for benchmarks
for i := 0; i < b.N; i++ {
_, err := LoadIndexFile("testdata/1.fs.index")
_, err := LoadIndexFile(z.Path("1.fs.index"))
if err != nil {
b.Fatal(err)
}
......@@ -255,8 +281,14 @@ func BenchmarkIndexLoad(b *testing.B) {
}
func BenchmarkIndexSave(b *testing.B) {
ztestdataReg.BenchWithEach(b, _BenchmarkIndexSave)
}
func _BenchmarkIndexSave(b *testing.B, z *ZTestData) {
if z.Kind == "py3_pickle3" {
b.Skip("xfail")
}
// FIXME small testdata/1.fs is not representative for benchmarks
index, err := LoadIndexFile("testdata/1.fs.index")
index, err := LoadIndexFile(z.Path("1.fs.index"))
if err != nil {
b.Fatal(err)
}
......@@ -273,8 +305,14 @@ func BenchmarkIndexSave(b *testing.B) {
}
func BenchmarkIndexGet(b *testing.B) {
ztestdataReg.BenchWithEach(b, _BenchmarkIndexGet)
}
func _BenchmarkIndexGet(b *testing.B, z *ZTestData) {
if z.Kind == "py3_pickle3" {
b.Skip("xfail")
}
// FIXME small testdata/1.fs is not representative for benchmarks
fsi, err := LoadIndexFile("testdata/1.fs.index")
fsi, err := LoadIndexFile(z.Path("1.fs.index"))
if err != nil {
b.Fatal(err)
}
......
#!/usr/bin/env python2
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2017-2021 Nexedi SA and Contributors.
# Copyright (C) 2017-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -20,13 +20,16 @@
# See https://www.nexedi.com/licensing for rationale and options.
"""generate reference fs1 database and index for tests"""
from __future__ import print_function
from ZODB.FileStorage import FileStorage
from ZODB.FileStorage.FileStorage import FILESTORAGE_MAGIC, TxnHeader, DataHeader, TRANS_HDR_LEN
from ZODB import DB
from ZODB.Connection import TransactionMetaData
from zodbtools.test.gen_testdata import gen_testdb, precommit, run_with_zodb4py2_compat
from os import stat, remove
from shutil import copyfile
from zodbtools.test.gen_testdata import gen_testdb, precommit, run_with_all_zodb_pickle_kinds, current_zkind
from os import stat, remove, makedirs
from os.path import exists, relpath
from shutil import copyfile, rmtree
from golang.gcompat import qq
import struct
......@@ -41,29 +44,44 @@ def unpack64(packed):
def hex64(packed):
return '0x%016x' % unpack64(packed)
# bchar returns bytes corresponding to bytes_data[i].
# it works as bytes_data[i] on py2, but on py3, contrary to builtin behaviour, also returns bytes instead of int.
def bchar(bytes_data, i): # -> bytes
if i >= len(bytes_data):
raise IndexError('index out of range')
return bytes_data[i:i+1]
def main():
outfs = "testdata/1.fs"
def main2():
zkind = current_zkind()
prefix = "testdata/%s" % zkind
if exists(prefix):
rmtree(prefix)
makedirs(prefix)
outfs = "%s/1.fs" % prefix
gen_testdb(outfs)
# dump to go what to expect
stor = FileStorage(outfs, read_only=True)
with open("ztestdata_expect_test.go", "w") as f:
with open("ztestdata_expect_%s_test.go" % zkind, "w") as f:
def emit(v):
print >>f, v
emit("// Code generated by %s; DO NOT EDIT." % __file__)
print(v, file=f)
emit("// Code generated by %s; DO NOT EDIT." % relpath(__file__))
emit("package fs1\n")
emit("import \"lab.nexedi.com/kirr/neo/go/zodb\"\n")
emit("func init() {")
# index
emit("const _1fs_indexTopPos = %i" % stor._pos)
emit("var _1fs_indexEntryv = [...]indexEntry{")
emit("\tconst _1fs_indexTopPos = %i" % stor._pos)
emit("\tvar _1fs_indexEntryv = []indexEntry{")
for k, v in stor._index.iteritems():
emit("\t{%8i, %8i}," % (unpack64(k), v))
emit("}")
emit("\t\t{%8i, %8i}," % (unpack64(k), v))
emit("\t}")
# database records
emit("\nvar _1fs_dbEntryv = [...]dbEntry{")
emit("\n\tvar _1fs_dbEntryv = []dbEntry{")
txnLenPrev = 0
for txn in stor.iterator(): # txn is TransactionRecord
# txn.extension is already depickled dict - we want to put raw data from file
......@@ -76,26 +94,26 @@ def main():
# fs1/go keeps in RAM whole txn length, not len-8 as it is on disk
txnLen = th.tlen + 8
emit("\t{")
emit("\t\t{")
# -> TxnHeader
emit("\t\tTxnHeader{")
emit("\t\t\tPos:\t %i," % txn._tpos)
emit("\t\t\tLenPrev: %i," % txnLenPrev)
emit("\t\t\tLen:\t %i," % txnLen)
emit("\t\t\tTxnInfo:\tzodb.TxnInfo{")
emit("\t\t\t\tTid:\t%s," % hex64(txn.tid))
emit("\t\t\t\tStatus:\t'%s'," % txn.status)
emit("\t\t\t\tUser:\t\t[]byte(%s)," % qq(txn.user))
emit("\t\t\t\tDescription:\t[]byte(%s)," % qq(txn.description))
emit("\t\t\t\tExtension:\t[]byte(%s)," % qq(th.ext))
emit("\t\t\tTxnHeader{")
emit("\t\t\t\tPos:\t %i," % txn._tpos)
emit("\t\t\t\tLenPrev: %i," % txnLenPrev)
emit("\t\t\t\tLen:\t %i," % txnLen)
emit("\t\t\t\tTxnInfo:\tzodb.TxnInfo{")
emit("\t\t\t\t\tTid:\t%s," % hex64(txn.tid))
emit("\t\t\t\t\tStatus:\t'%s'," % txn.status)
emit("\t\t\t\t\tUser:\t\t[]byte(%s)," % qq(txn.user))
emit("\t\t\t\t\tDescription:\t[]byte(%s)," % qq(txn.description))
emit("\t\t\t\t\tExtension:\t[]byte(%s)," % qq(th.ext))
emit("\t\t\t\t},")
emit("\t\t\t},")
emit("\t\t},")
txnLenPrev = txnLen
# -> DataHeader + payload
emit("\n\t\t[]txnEntry{")
emit("\n\t\t\t[]txnEntry{")
for drec in txn: # drec is itemof(TransactionRecordIterator) = Record
# same as with txn - not everything is possible to get via
......@@ -105,17 +123,17 @@ def main():
assert dh.tid == drec.tid
assert dh.tloc == txn._tpos
emit("\t\t\t{")
emit("\t\t\t\tDataHeader{")
emit("\t\t\t\t\tPos:\t%i," % drec.pos)
emit("\t\t\t\t\tOid:\t%i," % unpack64(drec.oid))
emit("\t\t\t\t\tTid:\t%s," % hex64(drec.tid))
emit("\t\t\t\t{")
emit("\t\t\t\t\tDataHeader{")
emit("\t\t\t\t\t\tPos:\t%i," % drec.pos)
emit("\t\t\t\t\t\tOid:\t%i," % unpack64(drec.oid))
emit("\t\t\t\t\t\tTid:\t%s," % hex64(drec.tid))
emit("\t\t\t\t\tPrevRevPos:\t%i," % dh.prev)
emit("\t\t\t\t\tTxnPos:\t%i," % txn._tpos)
emit("\t\t\t\t\t\tPrevRevPos:\t%i," % dh.prev)
emit("\t\t\t\t\t\tTxnPos:\t%i," % txn._tpos)
assert drec.version == ''
emit("\t\t\t\t\tDataLen:\t%i," % dh.plen)
emit("\t\t\t\t},")
emit("\t\t\t\t\t\tDataLen:\t%i," % dh.plen)
emit("\t\t\t\t\t},")
plen = dh.plen
if plen == 0:
rawdata = p64(dh.back) # back-pointer or 0 (= delete)
......@@ -130,19 +148,22 @@ def main():
data = "/* same as ^^^ */ sameAsRaw"
datatid = "/* no copy */ 0"
emit("\t\t\t\t[]byte(%s)," % qq(rawdata))
emit("\t\t\t\t%s," % data)
emit("\t\t\t\t%s," % datatid)
emit("\t\t\t},")
emit("\t\t\t\t\t[]byte(%s)," % qq(rawdata))
emit("\t\t\t\t\t%s," % data)
emit("\t\t\t\t\t%s," % datatid)
emit("\t\t\t\t},")
emit("\t\t\t},")
emit("\t\t},")
emit("\t},")
emit("\t}")
emit("\n\tztestdataReg.Register(%s, %s, &_TestDataOK{_1fs_indexTopPos, _1fs_indexEntryv, _1fs_dbEntryv})" % (qq(zkind), qq(prefix)))
emit("}")
stor.close()
# prepare file with voted (not fully committed) tail
voted = "testdata/1voted.fs"
voted = "%s/1voted.fs" % prefix
copyfile(outfs, voted)
def _():
vstor = FileStorage(voted)
......@@ -154,19 +175,19 @@ def main():
txn = precommit(u"author", u"description", {'aaa': 'bbb'})
txn_stormeta = TransactionMetaData(txn.user, txn.description, txn.extension)
vstor.tpc_begin(txn_stormeta)
vstor.store(vroot._p_oid, vroot._p_serial, '000 data 000', '', txn_stormeta)
vstor.store(vroot._p_oid, vroot._p_serial, b'000 data 000', '', txn_stormeta)
vstor.tpc_vote(txn_stormeta)
# NO tpc_finish here so that status remain 'c' (voted) instead of ' ' (committed)
run_with_zodb4py2_compat(_)
_()
st = stat(outfs)
l = st.st_size
vf = open(voted, 'rb')
vf.seek(l)
voted_tail = vf.read()
assert voted_tail[-1+8+8+1] == 'c' # voted, not finished (' ')
assert bchar(voted_tail, -1+8+8+1) == b'c' # voted, not finished (' ')
with open("testdata/1voted.tail", "wb") as vt:
with open("%s/1voted.tail" % prefix, "wb") as vt:
vt.write(voted_tail)
remove(voted)
......@@ -176,7 +197,7 @@ def main():
# prepare file with whiteout (deletion of previously non-existing object)
whiteout = "testdata/whiteout.fs"
whiteout = "%s/whiteout.fs" % prefix
# as of 20210317 FileStorage.deleteObject verifies that object exists
# -> prepare magic/transaction/data records manually
with open(whiteout, "wb") as f:
......@@ -206,5 +227,14 @@ def main():
f.write(p64(tlen))
# prepare empty.fs
empty = "%s/empty.fs" % prefix
with open(empty, "wb") as f:
f.write(FILESTORAGE_MAGIC)
def main():
run_with_all_zodb_pickle_kinds(main2)
if __name__ == '__main__':
main()
#!/usr/bin/env python2
# Copyright (C) 2017 Nexedi SA and Contributors.
#!/usr/bin/env python
# Copyright (C) 2017-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -19,6 +19,8 @@
# See https://www.nexedi.com/licensing for rationale and options.
"""compare two ZODB FileStorage v1 index files"""
from __future__ import print_function
from ZODB.fsIndex import fsIndex
import sys
......@@ -31,9 +33,9 @@ def main():
topPos1, fsi1 = d1["pos"], d1["index"]
topPos2, fsi2 = d2["pos"], d2["index"]
#print topPos1, topPos2
#print fsi1.items()
#print fsi2.items()
#print(topPos1, topPos2)
#print(fsi1.items())
#print(fsi2.items())
equal = (topPos1 == topPos2 and fsi1.items() == fsi2.items())
sys.exit(int(not equal))
......
/*.lock
/*.tmp
/*.tr[0-9]
*.lock
*.tmp
*.tr[0-9]
*.old
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment