Skip to content

N
neo
Commit 80958131 authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

go/zodb/btree: go/zodb/fs1tools: Test it on all py2/py3 ZODB kinds of data we care about 

Previously, like with FileStorage, we were testing btree/go only with
data generated by python2 and pickle protocol=2. However even on py2
there are more pickle protocols that are in use, and also there is python3.

-> Modernize testdata/gen-testdata to use run_with_all_zodb_pickle_kinds
   that was recently added as part of nexedi/zodbtools@f9d36ba7
   and generate test data with both python2 and python3. It is handy to
   use py2py3-venv(*) to prepare python environment to do that.

   Adjust tests on Go side to verify how btree handles all generated zkinds.

All py2_pickle1, py2_pickle2, py2_pickle3 and py3_pickle3 are handled well out of the box.

(*) see nexedi/zodbtools@fac2f190
parent 3af5b2c1
Show whitespace changes
Inline Side-by-side
Showing with 200 additions and 68 deletions
go/zodb/btree/btree_test.go
View file @ 80958131
......@@ -19,7 +19,8 @@
package btree
//go:generate ./testdata/gen-testdata
//go:generate python2 testdata/gen-testdata
//go:generate python3 testdata/gen-testdata
import (
"context"
......@@ -28,6 +29,7 @@ import (
"lab.nexedi.com/kirr/go123/exc"
"lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/neo/go/internal/xtesting"
"lab.nexedi.com/kirr/neo/go/transaction"
"lab.nexedi.com/kirr/neo/go/zodb"
_ "lab.nexedi.com/kirr/neo/go/zodb/wks"
......@@ -35,6 +37,23 @@ import (
pickle "github.com/kisielk/og-rek"
)
// ztestdataReg maintains registry of all entries under testdata/ .
var ztestdataReg = xtesting.ZTestDataRegistry[_TestDataOK]{}
type ZTestData = xtesting.ZTestData[_TestDataOK]
// _TestDataOK describes expected data for one entry under testdata/ .
type _TestDataOK struct {
smallTestv []testEntry // which data is stored in small b0-b2 and B0-B2 nodes
B3_oid zodb.Oid // information about large B3 tree
B3_maxkey int64
Bv_oid zodb.Oid // information about Bv tree, used to verify visit callbacks
Bv_kmin int64
Bv_kmax int64
Bvdict map[int64][]tVisit
}
// kv is one (key, value) pair.
type kv struct {
key int64
......@@ -108,9 +127,14 @@ type tVisit struct {
}
func TestBTree(t *testing.T) {
ztestdataReg.RunWithEach(t, _TestBTree)
}
func _TestBTree(t *testing.T, z *ZTestData) {
zz := z.Misc
X := exc.Raiseif
ctx := context.Background()
stor, err := zodb.Open(ctx, "testdata/1.fs", &zodb.OpenOptions{ReadOnly: true})
stor, err := zodb.Open(ctx, z.Path("1.fs"), &zodb.OpenOptions{ReadOnly: true})
if err != nil {
t.Fatal(err)
}
......@@ -132,7 +156,7 @@ func TestBTree(t *testing.T) {
}
// go through small test Buckets/BTrees and verify that Get(key) is as expected.
for _, tt := range smallTestv {
for _, tt := range zz.smallTestv {
xobj, err := conn.Get(ctx, tt.oid)
if err != nil {
t.Fatal(err)
......@@ -207,16 +231,16 @@ func TestBTree(t *testing.T) {
// B3 is a large BTree with {i: i} data.
// verify Get(key), {Min,Max}Key and that different bucket links lead to the same in-RAM object.
xB3, err := conn.Get(ctx, B3_oid)
xB3, err := conn.Get(ctx, zz.B3_oid)
if err != nil {
t.Fatal(err)
}
B3, ok := xB3.(*LOBTree)
if !ok {
t.Fatalf("B3: %v; got %T; want LOBTree", B3_oid, xB3)
t.Fatalf("B3: %v; got %T; want LOBTree", zz.B3_oid, xB3)
}
for i := int64(0); i <= B3_maxkey; i++ {
for i := int64(0); i <= zz.B3_maxkey; i++ {
v, ok, err := B3.Get(ctx, i)
if err != nil {
t.Fatal(err)
......@@ -234,9 +258,9 @@ func TestBTree(t *testing.T) {
if err := xerr.Merge(emin, emax); err != nil {
t.Fatalf("B3: min/max key: %s", err)
}
if !(kmin == 0 && kmax == B3_maxkey && okmin && okmax) {
if !(kmin == 0 && kmax == zz.B3_maxkey && okmin && okmax) {
t.Fatalf("B3: min/max key wrong: got [%v, %v] (%v, %v); want [%v, %v] (%v, %v)",
kmin, kmax, okmin, okmax, 0, B3_maxkey, true, true)
kmin, kmax, okmin, okmax, 0, zz.B3_maxkey, true, true)
}
// verifyFirstBucket verifies that b.firstbucket is correct and returns it.
......@@ -274,13 +298,13 @@ func TestBTree(t *testing.T) {
verifyFirstBucket(B3)
// verify nodes/keycov visited through VGet/V{Min,Max}Key
xBv, err := conn.Get(ctx, Bv_oid); X(err)
xBv, err := conn.Get(ctx, zz.Bv_oid); X(err)
Bv, ok := xBv.(*LOBTree)
if !ok {
t.Fatalf("Bv: %v; got %T; want LOBTree", Bv_oid, xBv)
t.Fatalf("Bv: %v; got %T; want LOBTree", zz.Bv_oid, xBv)
}
for k, visitOK := range Bvdict {
for k, visitOK := range zz.Bvdict {
visit := []tVisit{}
_, _, err := Bv.VGet(ctx, k, func(node LONode, keycov LKeyRange) {
visit = append(visit, tVisit{node.POid(), keycov})
......@@ -290,8 +314,8 @@ func TestBTree(t *testing.T) {
}
}
visitMinOK := Bvdict[Bv_kmin]
visitMaxOK := Bvdict[Bv_kmax]
visitMinOK := zz.Bvdict[zz.Bv_kmin]
visitMaxOK := zz.Bvdict[zz.Bv_kmax]
visitMin := []tVisit{}
visitMax := []tVisit{}
_, _, err = Bv.VMinKey(ctx, func(node LONode, keycov LKeyRange) {
......@@ -312,5 +336,5 @@ func TestBTree(t *testing.T) {
// ---- misc ----
// ztestdata_* use bstr
// ztestdata_py2_* use bstr
type bstr = pickle.ByteString
go/zodb/btree/testdata/.gitignore
View file @ 80958131
/*.lock
/*.tmp
/*.tr[0-9]
*.lock
*.tmp
*.tr[0-9]
go/zodb/btree/testdata/gen-testdata
View file @ 80958131
#!/usr/bin/env python2
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
......@@ -20,25 +20,27 @@
# See https://www.nexedi.com/licensing for rationale and options.
"""generate test data for btree serialization tests"""
from __future__ import print_function
from ZODB.DB import DB
from BTrees.LOBTree import LOBucket, LOBTree
from BTrees.check import check as bcheck
from ZODB.utils import u64
from zodbtools.test.gen_testdata import run_with_zodb4py2_compat
import os, os.path, transaction
from zodbtools.test.gen_testdata import run_with_all_zodb_pickle_kinds, current_zkind
import os, os.path, shutil, transaction
from golang.gcompat import qq
def rm_f(path):
if os.path.exists(path):
os.remove(path)
def main2():
import zodbtools.test.gen_testdata # to make time predictable (XXX)
outfs = "testdata/1.fs"
rm_f(outfs)
rm_f(outfs + ".index")
zkind = current_zkind()
prefix = "testdata/%s" % zkind
if os.path.exists(prefix):
shutil.rmtree(prefix)
os.makedirs(prefix)
outfs = "%s/1.fs" % prefix
db = DB(outfs)
conn = db.open()
root = conn.root()
......@@ -79,18 +81,22 @@ def main2():
assert Bv[9] == "e"
with open("ztestdata_expect_test.go", "w") as f:
with open("ztestdata_expect_%s_test.go" % zkind, "w") as f:
def emit(v):
print >>f, v
emit("// Code generated by %s; DO NOT EDIT." % __file__)
print(v, file=f)
emit("// Code generated by %s; DO NOT EDIT." % os.path.relpath(__file__))
emit("package btree\n")
emit("func init() {")
emit("\ttestdataok := _TestDataOK{")
def emititems(b):
s = "testEntry{oid: %s, kind: %s, itemv: []kv{" \
% (u64(b._p_oid), "kind%s" % type(b).__name__[2:])
for k, v in b.items():
if isinstance(v, str):
v = qq(v)
if str is bytes:
v = "bstr(%s)" % v
elif isinstance(v, int):
v = "int64(%d)" % v
......@@ -99,20 +105,20 @@ def main2():
s += "{%s, %s}, " % (k, v)
s += "}},"
emit("\t"+s)
emit("\t\t\t"+s)
emit("\nvar smallTestv = [...]testEntry{")
emit("\t\tsmallTestv: []testEntry{")
for b in (b0, b1, b2, B0, B1, B2):
emititems(b)
emit("}")
emit("\t\t},")
emit("\nconst B3_oid = %s" % u64(B3._p_oid))
emit("const B3_maxkey = %d" % B3.maxKey())
emit("\n\t\tB3_oid: %s," % u64(B3._p_oid))
emit("\t\tB3_maxkey: %d," % B3.maxKey())
emit("\nconst Bv_oid = %s" % u64(Bv._p_oid))
emit("const Bv_kmin = %d" % Bv.minKey())
emit("const Bv_kmax = %d" % Bv.maxKey())
emit("var Bvdict = map[int64][]tVisit{")
emit("\n\t\tBv_oid: %s," % u64(Bv._p_oid))
emit("\t\tBv_kmin: %d," % Bv.minKey())
emit("\t\tBv_kmax: %d," % Bv.maxKey())
emit("\t\tBvdict: map[int64][]tVisit{")
noo = "_LKeyMin"
oo = "_LKeyMax"
def emitVisit(key, *visitv): # visitv = [](node, lo,hi)
......@@ -123,20 +129,24 @@ def main2():
else:
hi_ = hi-1
vstr.append("{%d, LKeyRange{%s, %s}}" % (u64(node._p_oid), lo, hi_))
emit("\t%d: []tVisit{%s}," % (key, ", ".join(vstr)))
emit("\t\t\t%d: []tVisit{%s}," % (key, ", ".join(vstr)))
emitVisit(1, (T4, noo,oo), (T2, noo,4), (v1, noo,2))
emitVisit(2, (T4, noo,oo), (T2, noo,4), (v2, 2,4))
emitVisit(5, (T4, noo,oo), (T, 4,oo), (T79, 4,oo), (v5, 4,7))
emitVisit(8, (T4, noo,oo), (T, 4,oo), (T79, 4,oo), (v8, 7,9))
emitVisit(9, (T4, noo,oo), (T, 4,oo), (T79, 4,oo), (v9, 9,oo))
emit("\t\t},")
emit("\t}")
emit("\n\tztestdataReg.Register(%s, %s, &testdataok)" % (qq(zkind), qq(prefix)))
emit("}")
conn.close()
db.close()
def main():
run_with_zodb4py2_compat(main2)
run_with_all_zodb_pickle_kinds(main2)
if __name__ == '__main__':
......
go/zodb/btree/ztestdata_expect_py2_pickle1_test.go 0 → 100644
View file @ 80958131
// Code generated by testdata/gen-testdata; DO NOT EDIT.
package btree
func init() {
testdataok := _TestDataOK{
smallTestv: []testEntry{
testEntry{oid: 7, kind: kindBucket, itemv: []kv{}},
testEntry{oid: 4, kind: kindBucket, itemv: []kv{{10, int64(17)}, }},
testEntry{oid: 1, kind: kindBucket, itemv: []kv{{15, int64(1)}, {23, bstr("hello")}, }},
testEntry{oid: 3, kind: kindBTree, itemv: []kv{}},
testEntry{oid: 8, kind: kindBTree, itemv: []kv{{5, int64(4)}, }},
testEntry{oid: 5, kind: kindBTree, itemv: []kv{{7, int64(3)}, {9, bstr("world")}, }},
},
B3_oid: 6,
B3_maxkey: 9999,
Bv_oid: 2,
Bv_kmin: 1,
Bv_kmax: 9,
Bvdict: map[int64][]tVisit{
1: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {342, LKeyRange{_LKeyMin, 3}}, {344, LKeyRange{_LKeyMin, 1}}},
2: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {342, LKeyRange{_LKeyMin, 3}}, {349, LKeyRange{2, 3}}},
5: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {346, LKeyRange{4, 6}}},
8: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {347, LKeyRange{7, 8}}},
9: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {348, LKeyRange{9, _LKeyMax}}},
},
}
ztestdataReg.Register("py2_pickle1", "testdata/py2_pickle1", &testdataok)
}
go/zodb/btree/ztestdata_expect_py2_pickle2_test.go 0 → 100644
View file @ 80958131
// Code generated by testdata/gen-testdata; DO NOT EDIT.
package btree
func init() {
testdataok := _TestDataOK{
smallTestv: []testEntry{
testEntry{oid: 7, kind: kindBucket, itemv: []kv{}},
testEntry{oid: 4, kind: kindBucket, itemv: []kv{{10, int64(17)}, }},
testEntry{oid: 1, kind: kindBucket, itemv: []kv{{15, int64(1)}, {23, bstr("hello")}, }},
testEntry{oid: 3, kind: kindBTree, itemv: []kv{}},
testEntry{oid: 8, kind: kindBTree, itemv: []kv{{5, int64(4)}, }},
testEntry{oid: 5, kind: kindBTree, itemv: []kv{{7, int64(3)}, {9, bstr("world")}, }},
},
B3_oid: 6,
B3_maxkey: 9999,
Bv_oid: 2,
Bv_kmin: 1,
Bv_kmax: 9,
Bvdict: map[int64][]tVisit{
1: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {342, LKeyRange{_LKeyMin, 3}}, {344, LKeyRange{_LKeyMin, 1}}},
2: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {342, LKeyRange{_LKeyMin, 3}}, {349, LKeyRange{2, 3}}},
5: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {346, LKeyRange{4, 6}}},
8: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {347, LKeyRange{7, 8}}},
9: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {348, LKeyRange{9, _LKeyMax}}},
},
}
ztestdataReg.Register("py2_pickle2", "testdata/py2_pickle2", &testdataok)
}
go/zodb/btree/ztestdata_expect_py2_pickle3_test.go 0 → 100644
View file @ 80958131
// Code generated by testdata/gen-testdata; DO NOT EDIT.
package btree
func init() {
testdataok := _TestDataOK{
smallTestv: []testEntry{
testEntry{oid: 7, kind: kindBucket, itemv: []kv{}},
testEntry{oid: 4, kind: kindBucket, itemv: []kv{{10, int64(17)}, }},
testEntry{oid: 1, kind: kindBucket, itemv: []kv{{15, int64(1)}, {23, bstr("hello")}, }},
testEntry{oid: 3, kind: kindBTree, itemv: []kv{}},
testEntry{oid: 8, kind: kindBTree, itemv: []kv{{5, int64(4)}, }},
testEntry{oid: 5, kind: kindBTree, itemv: []kv{{7, int64(3)}, {9, bstr("world")}, }},
},
B3_oid: 6,
B3_maxkey: 9999,
Bv_oid: 2,
Bv_kmin: 1,
Bv_kmax: 9,
Bvdict: map[int64][]tVisit{
1: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {342, LKeyRange{_LKeyMin, 3}}, {344, LKeyRange{_LKeyMin, 1}}},
2: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {342, LKeyRange{_LKeyMin, 3}}, {349, LKeyRange{2, 3}}},
5: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {346, LKeyRange{4, 6}}},
8: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {347, LKeyRange{7, 8}}},
9: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {348, LKeyRange{9, _LKeyMax}}},
},
}
ztestdataReg.Register("py2_pickle3", "testdata/py2_pickle3", &testdataok)
}
go/zodb/btree/ztestdata_expect_py3_pickle3_test.go 0 → 100644
View file @ 80958131
// Code generated by testdata/gen-testdata; DO NOT EDIT.
package btree
func init() {
testdataok := _TestDataOK{
smallTestv: []testEntry{
testEntry{oid: 1, kind: kindBucket, itemv: []kv{}},
testEntry{oid: 2, kind: kindBucket, itemv: []kv{{10, int64(17)}, }},
testEntry{oid: 3, kind: kindBucket, itemv: []kv{{15, int64(1)}, {23, "hello"}, }},
testEntry{oid: 4, kind: kindBTree, itemv: []kv{}},
testEntry{oid: 5, kind: kindBTree, itemv: []kv{{5, int64(4)}, }},
testEntry{oid: 6, kind: kindBTree, itemv: []kv{{7, int64(3)}, {9, "world"}, }},
},
B3_oid: 7,
B3_maxkey: 9999,
Bv_oid: 8,
Bv_kmin: 1,
Bv_kmax: 9,
Bvdict: map[int64][]tVisit{
1: []tVisit{{8, LKeyRange{_LKeyMin, _LKeyMax}}, {9, LKeyRange{_LKeyMin, 3}}, {11, LKeyRange{_LKeyMin, 1}}},
2: []tVisit{{8, LKeyRange{_LKeyMin, _LKeyMax}}, {9, LKeyRange{_LKeyMin, 3}}, {16, LKeyRange{2, 3}}},
5: []tVisit{{8, LKeyRange{_LKeyMin, _LKeyMax}}, {10, LKeyRange{4, _LKeyMax}}, {12, LKeyRange{4, _LKeyMax}}, {13, LKeyRange{4, 6}}},
8: []tVisit{{8, LKeyRange{_LKeyMin, _LKeyMax}}, {10, LKeyRange{4, _LKeyMax}}, {12, LKeyRange{4, _LKeyMax}}, {14, LKeyRange{7, 8}}},
9: []tVisit{{8, LKeyRange{_LKeyMin, _LKeyMax}}, {10, LKeyRange{4, _LKeyMax}}, {12, LKeyRange{4, _LKeyMax}}, {15, LKeyRange{9, _LKeyMax}}},
},
}
ztestdataReg.Register("py3_pickle3", "testdata/py3_pickle3", &testdataok)
}
go/zodb/btree/ztestdata_expect_test.go deleted 100644 → 0
View file @ 3af5b2c1
// Code generated by ./testdata/gen-testdata; DO NOT EDIT.
package btree
var smallTestv = [...]testEntry{
testEntry{oid: 7, kind: kindBucket, itemv: []kv{}},
testEntry{oid: 4, kind: kindBucket, itemv: []kv{{10, int64(17)}, }},
testEntry{oid: 1, kind: kindBucket, itemv: []kv{{15, int64(1)}, {23, bstr("hello")}, }},
testEntry{oid: 3, kind: kindBTree, itemv: []kv{}},
testEntry{oid: 8, kind: kindBTree, itemv: []kv{{5, int64(4)}, }},
testEntry{oid: 5, kind: kindBTree, itemv: []kv{{7, int64(3)}, {9, bstr("world")}, }},
}
const B3_oid = 6
const B3_maxkey = 9999
const Bv_oid = 2
const Bv_kmin = 1
const Bv_kmax = 9
var Bvdict = map[int64][]tVisit{
1: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {342, LKeyRange{_LKeyMin, 3}}, {344, LKeyRange{_LKeyMin, 1}}},
2: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {342, LKeyRange{_LKeyMin, 3}}, {349, LKeyRange{2, 3}}},
5: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {346, LKeyRange{4, 6}}},
8: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {347, LKeyRange{7, 8}}},
9: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {348, LKeyRange{9, _LKeyMax}}},
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7