go/zodb/btree: go/zodb/fs1tools: Test it on all py2/py3 ZODB kinds of data we care about

Previously, like with FileStorage, we were testing btree/go only with
data generated by python2 and pickle protocol=2. However even on py2
there are more pickle protocols that are in use, and also there is python3.

-> Modernize testdata/gen-testdata to use run_with_all_zodb_pickle_kinds
   that was recently added as part of nexedi/zodbtools@f9d36ba7
   and generate test data with both python2 and python3. It is handy to
   use py2py3-venv(*) to prepare python environment to do that.

   Adjust tests on Go side to verify how btree handles all generated zkinds.

All py2_pickle1, py2_pickle2, py2_pickle3 and py3_pickle3 are handled well out of the box.

(*) see nexedi/zodbtools@fac2f190

go/zodb/btree/btree_test.go

@@ -19,7 +19,8 @@
 
 package btree
 
-//go:generate ./testdata/gen-testdata
+//go:generate python2 testdata/gen-testdata
+//go:generate python3 testdata/gen-testdata
 
 import (
 	"context"
@@ -28,6 +29,7 @@ import (
 
 	"lab.nexedi.com/kirr/go123/exc"
 	"lab.nexedi.com/kirr/go123/xerr"
+	"lab.nexedi.com/kirr/neo/go/internal/xtesting"
 	"lab.nexedi.com/kirr/neo/go/transaction"
 	"lab.nexedi.com/kirr/neo/go/zodb"
 	_ "lab.nexedi.com/kirr/neo/go/zodb/wks"
@@ -35,6 +37,23 @@ import (
 	pickle "github.com/kisielk/og-rek"
 )
 
+// ztestdataReg maintains registry of all entries under testdata/ .
+var  ztestdataReg = xtesting.ZTestDataRegistry[_TestDataOK]{}
+type ZTestData    = xtesting.ZTestData[_TestDataOK]
+
+// _TestDataOK describes expected data for one entry under testdata/ .
+type _TestDataOK struct {
+	smallTestv []testEntry // which data is stored in small b0-b2 and B0-B2 nodes
+
+	B3_oid    zodb.Oid // information about large B3 tree
+	B3_maxkey int64
+
+	Bv_oid  zodb.Oid // information about Bv tree, used to verify visit callbacks
+	Bv_kmin int64
+	Bv_kmax int64
+	Bvdict  map[int64][]tVisit
+}
+
 // kv is one (key, value) pair.
 type kv struct {
 	key   int64
@@ -108,9 +127,14 @@ type tVisit struct {
 }
 
 func TestBTree(t *testing.T) {
+	ztestdataReg.RunWithEach(t, _TestBTree)
+}
+func _TestBTree(t *testing.T, z *ZTestData) {
+	zz := z.Misc
+
 	X := exc.Raiseif
 	ctx := context.Background()
-	stor, err := zodb.Open(ctx, "testdata/1.fs", &zodb.OpenOptions{ReadOnly: true})
+	stor, err := zodb.Open(ctx, z.Path("1.fs"), &zodb.OpenOptions{ReadOnly: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -132,7 +156,7 @@ func TestBTree(t *testing.T) {
 	}
 
 	// go through small test Buckets/BTrees and verify that Get(key) is as expected.
-	for _, tt := range smallTestv {
+	for _, tt := range zz.smallTestv {
 		xobj, err := conn.Get(ctx, tt.oid)
 		if err != nil {
 			t.Fatal(err)
@@ -207,16 +231,16 @@ func TestBTree(t *testing.T) {
 
 	// B3 is a large BTree with {i: i} data.
 	// verify Get(key), {Min,Max}Key and that different bucket links lead to the same in-RAM object.
-	xB3, err := conn.Get(ctx, B3_oid)
+	xB3, err := conn.Get(ctx, zz.B3_oid)
 	if err != nil {
 		t.Fatal(err)
 	}
 	B3, ok := xB3.(*LOBTree)
 	if !ok {
-		t.Fatalf("B3: %v; got %T;  want LOBTree", B3_oid, xB3)
+		t.Fatalf("B3: %v; got %T;  want LOBTree", zz.B3_oid, xB3)
 	}
 
-	for i := int64(0); i <= B3_maxkey; i++ {
+	for i := int64(0); i <= zz.B3_maxkey; i++ {
 		v, ok, err := B3.Get(ctx, i)
 		if err != nil {
 			t.Fatal(err)
@@ -234,9 +258,9 @@ func TestBTree(t *testing.T) {
 	if err := xerr.Merge(emin, emax); err != nil {
 		t.Fatalf("B3: min/max key: %s", err)
 	}
-	if !(kmin == 0 && kmax == B3_maxkey && okmin && okmax) {
+	if !(kmin == 0 && kmax == zz.B3_maxkey && okmin && okmax) {
 		t.Fatalf("B3: min/max key wrong: got [%v, %v] (%v, %v);  want [%v, %v] (%v, %v)",
-			kmin, kmax, okmin, okmax, 0, B3_maxkey, true, true)
+			kmin, kmax, okmin, okmax, 0, zz.B3_maxkey, true, true)
 	}
 
 	// verifyFirstBucket verifies that b.firstbucket is correct and returns it.
@@ -274,13 +298,13 @@ func TestBTree(t *testing.T) {
 	verifyFirstBucket(B3)
 
 	// verify nodes/keycov visited through VGet/V{Min,Max}Key
-	xBv, err := conn.Get(ctx, Bv_oid);  X(err)
+	xBv, err := conn.Get(ctx, zz.Bv_oid);  X(err)
 	Bv, ok := xBv.(*LOBTree)
 	if !ok {
-		t.Fatalf("Bv: %v; got %T;  want LOBTree", Bv_oid, xBv)
+		t.Fatalf("Bv: %v; got %T;  want LOBTree", zz.Bv_oid, xBv)
 	}
 
-	for k, visitOK := range Bvdict {
+	for k, visitOK := range zz.Bvdict {
 		visit := []tVisit{}
 		_, _, err := Bv.VGet(ctx, k, func(node LONode, keycov LKeyRange) {
 			visit = append(visit, tVisit{node.POid(), keycov})
@@ -290,8 +314,8 @@ func TestBTree(t *testing.T) {
 		}
 	}
 
-	visitMinOK := Bvdict[Bv_kmin]
-	visitMaxOK := Bvdict[Bv_kmax]
+	visitMinOK := zz.Bvdict[zz.Bv_kmin]
+	visitMaxOK := zz.Bvdict[zz.Bv_kmax]
 	visitMin := []tVisit{}
 	visitMax := []tVisit{}
 	_, _, err = Bv.VMinKey(ctx, func(node LONode, keycov LKeyRange) {
@@ -312,5 +336,5 @@ func TestBTree(t *testing.T) {
 
 // ---- misc ----
 
-// ztestdata_* use bstr
+// ztestdata_py2_* use bstr
 type bstr = pickle.ByteString

go/zodb/btree/testdata/.gitignore

-/*.lock
-/*.tmp
-/*.tr[0-9]
+*.lock
+*.tmp
+*.tr[0-9]

go/zodb/btree/testdata/gen-testdata

-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright (C) 2018-2024  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
@@ -20,25 +20,27 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 """generate test data for btree serialization tests"""
 
+from __future__ import print_function
+
 from ZODB.DB import DB
 from BTrees.LOBTree import LOBucket, LOBTree
 from BTrees.check import check as bcheck
 from ZODB.utils import u64
-from zodbtools.test.gen_testdata import run_with_zodb4py2_compat
-import os, os.path, transaction
+from zodbtools.test.gen_testdata import run_with_all_zodb_pickle_kinds, current_zkind
+import os, os.path, shutil, transaction
 
 from golang.gcompat import qq
 
-def rm_f(path):
-    if os.path.exists(path):
-        os.remove(path)
-
 def main2():
     import zodbtools.test.gen_testdata  # to make time predictable (XXX)
 
-    outfs = "testdata/1.fs"
-    rm_f(outfs)
-    rm_f(outfs + ".index")
+    zkind = current_zkind()
+    prefix = "testdata/%s" % zkind
+    if os.path.exists(prefix):
+        shutil.rmtree(prefix)
+    os.makedirs(prefix)
+
+    outfs = "%s/1.fs" % prefix
     db = DB(outfs)
     conn = db.open()
     root = conn.root()
@@ -79,18 +81,22 @@ def main2():
     assert Bv[9] == "e"
 
 
-    with open("ztestdata_expect_test.go", "w") as f:
+    with open("ztestdata_expect_%s_test.go" % zkind, "w") as f:
         def emit(v):
-            print >>f, v
-        emit("// Code generated by %s; DO NOT EDIT." % __file__)
+            print(v, file=f)
+        emit("// Code generated by %s; DO NOT EDIT." % os.path.relpath(__file__))
         emit("package btree\n")
 
+        emit("func init() {")
+        emit("\ttestdataok := _TestDataOK{")
+
         def emititems(b):
             s = "testEntry{oid: %s, kind: %s, itemv: []kv{" \
                     % (u64(b._p_oid), "kind%s" % type(b).__name__[2:])
             for k, v in b.items():
                 if isinstance(v, str):
                     v = qq(v)
+                    if str is bytes:
                         v = "bstr(%s)" % v
                 elif isinstance(v, int):
                     v = "int64(%d)" % v
@@ -99,20 +105,20 @@ def main2():
 
                 s += "{%s, %s}, " % (k, v)
             s += "}},"
-            emit("\t"+s)
+            emit("\t\t\t"+s)
 
-        emit("\nvar smallTestv = [...]testEntry{")
+        emit("\t\tsmallTestv: []testEntry{")
         for b in (b0, b1, b2, B0, B1, B2):
             emititems(b)
-        emit("}")
+        emit("\t\t},")
 
-        emit("\nconst B3_oid = %s" % u64(B3._p_oid))
-        emit("const B3_maxkey = %d" % B3.maxKey())
+        emit("\n\t\tB3_oid: %s,"  % u64(B3._p_oid))
+        emit("\t\tB3_maxkey: %d," % B3.maxKey())
 
-        emit("\nconst Bv_oid = %s" % u64(Bv._p_oid))
-        emit("const Bv_kmin = %d"  % Bv.minKey())
-        emit("const Bv_kmax = %d"  % Bv.maxKey())
-        emit("var Bvdict = map[int64][]tVisit{")
+        emit("\n\t\tBv_oid: %s," % u64(Bv._p_oid))
+        emit("\t\tBv_kmin: %d,"  % Bv.minKey())
+        emit("\t\tBv_kmax: %d,"  % Bv.maxKey())
+        emit("\t\tBvdict: map[int64][]tVisit{")
         noo = "_LKeyMin"
         oo  = "_LKeyMax"
         def emitVisit(key, *visitv): # visitv = [](node, lo,hi)
@@ -123,20 +129,24 @@ def main2():
                 else:
                     hi_ = hi-1
                 vstr.append("{%d, LKeyRange{%s, %s}}" % (u64(node._p_oid), lo, hi_))
-            emit("\t%d: []tVisit{%s}," % (key, ", ".join(vstr)))
+            emit("\t\t\t%d: []tVisit{%s}," % (key, ", ".join(vstr)))
 
         emitVisit(1, (T4, noo,oo), (T2, noo,4), (v1, noo,2))
         emitVisit(2, (T4, noo,oo), (T2, noo,4), (v2, 2,4))
         emitVisit(5, (T4, noo,oo), (T,  4,oo),  (T79, 4,oo), (v5, 4,7))
         emitVisit(8, (T4, noo,oo), (T,  4,oo),  (T79, 4,oo), (v8, 7,9))
         emitVisit(9, (T4, noo,oo), (T,  4,oo),  (T79, 4,oo), (v9, 9,oo))
+        emit("\t\t},")
+        emit("\t}")
+
+        emit("\n\tztestdataReg.Register(%s, %s, &testdataok)" % (qq(zkind), qq(prefix)))
         emit("}")
 
     conn.close()
     db.close()
 
 def main():
-    run_with_zodb4py2_compat(main2)
+    run_with_all_zodb_pickle_kinds(main2)
 
 
 if __name__ == '__main__':

go/zodb/btree/ztestdata_expect_py2_pickle1_test.go

+// Code generated by testdata/gen-testdata; DO NOT EDIT.
+package btree
+
+func init() {
+	testdataok := _TestDataOK{
+		smallTestv: []testEntry{
+			testEntry{oid: 7, kind: kindBucket, itemv: []kv{}},
+			testEntry{oid: 4, kind: kindBucket, itemv: []kv{{10, int64(17)}, }},
+			testEntry{oid: 1, kind: kindBucket, itemv: []kv{{15, int64(1)}, {23, bstr("hello")}, }},
+			testEntry{oid: 3, kind: kindBTree, itemv: []kv{}},
+			testEntry{oid: 8, kind: kindBTree, itemv: []kv{{5, int64(4)}, }},
+			testEntry{oid: 5, kind: kindBTree, itemv: []kv{{7, int64(3)}, {9, bstr("world")}, }},
+		},
+
+		B3_oid: 6,
+		B3_maxkey: 9999,
+
+		Bv_oid: 2,
+		Bv_kmin: 1,
+		Bv_kmax: 9,
+		Bvdict: map[int64][]tVisit{
+			1: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {342, LKeyRange{_LKeyMin, 3}}, {344, LKeyRange{_LKeyMin, 1}}},
+			2: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {342, LKeyRange{_LKeyMin, 3}}, {349, LKeyRange{2, 3}}},
+			5: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {346, LKeyRange{4, 6}}},
+			8: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {347, LKeyRange{7, 8}}},
+			9: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {348, LKeyRange{9, _LKeyMax}}},
+		},
+	}
+
+	ztestdataReg.Register("py2_pickle1", "testdata/py2_pickle1", &testdataok)
+}

go/zodb/btree/ztestdata_expect_py2_pickle2_test.go

+// Code generated by testdata/gen-testdata; DO NOT EDIT.
+package btree
+
+func init() {
+	testdataok := _TestDataOK{
+		smallTestv: []testEntry{
+			testEntry{oid: 7, kind: kindBucket, itemv: []kv{}},
+			testEntry{oid: 4, kind: kindBucket, itemv: []kv{{10, int64(17)}, }},
+			testEntry{oid: 1, kind: kindBucket, itemv: []kv{{15, int64(1)}, {23, bstr("hello")}, }},
+			testEntry{oid: 3, kind: kindBTree, itemv: []kv{}},
+			testEntry{oid: 8, kind: kindBTree, itemv: []kv{{5, int64(4)}, }},
+			testEntry{oid: 5, kind: kindBTree, itemv: []kv{{7, int64(3)}, {9, bstr("world")}, }},
+		},
+
+		B3_oid: 6,
+		B3_maxkey: 9999,
+
+		Bv_oid: 2,
+		Bv_kmin: 1,
+		Bv_kmax: 9,
+		Bvdict: map[int64][]tVisit{
+			1: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {342, LKeyRange{_LKeyMin, 3}}, {344, LKeyRange{_LKeyMin, 1}}},
+			2: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {342, LKeyRange{_LKeyMin, 3}}, {349, LKeyRange{2, 3}}},
+			5: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {346, LKeyRange{4, 6}}},
+			8: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {347, LKeyRange{7, 8}}},
+			9: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {348, LKeyRange{9, _LKeyMax}}},
+		},
+	}
+
+	ztestdataReg.Register("py2_pickle2", "testdata/py2_pickle2", &testdataok)
+}

go/zodb/btree/ztestdata_expect_py2_pickle3_test.go

+// Code generated by testdata/gen-testdata; DO NOT EDIT.
+package btree
+
+func init() {
+	testdataok := _TestDataOK{
+		smallTestv: []testEntry{
+			testEntry{oid: 7, kind: kindBucket, itemv: []kv{}},
+			testEntry{oid: 4, kind: kindBucket, itemv: []kv{{10, int64(17)}, }},
+			testEntry{oid: 1, kind: kindBucket, itemv: []kv{{15, int64(1)}, {23, bstr("hello")}, }},
+			testEntry{oid: 3, kind: kindBTree, itemv: []kv{}},
+			testEntry{oid: 8, kind: kindBTree, itemv: []kv{{5, int64(4)}, }},
+			testEntry{oid: 5, kind: kindBTree, itemv: []kv{{7, int64(3)}, {9, bstr("world")}, }},
+		},
+
+		B3_oid: 6,
+		B3_maxkey: 9999,
+
+		Bv_oid: 2,
+		Bv_kmin: 1,
+		Bv_kmax: 9,
+		Bvdict: map[int64][]tVisit{
+			1: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {342, LKeyRange{_LKeyMin, 3}}, {344, LKeyRange{_LKeyMin, 1}}},
+			2: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {342, LKeyRange{_LKeyMin, 3}}, {349, LKeyRange{2, 3}}},
+			5: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {346, LKeyRange{4, 6}}},
+			8: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {347, LKeyRange{7, 8}}},
+			9: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {348, LKeyRange{9, _LKeyMax}}},
+		},
+	}
+
+	ztestdataReg.Register("py2_pickle3", "testdata/py2_pickle3", &testdataok)
+}

go/zodb/btree/ztestdata_expect_py3_pickle3_test.go

+// Code generated by testdata/gen-testdata; DO NOT EDIT.
+package btree
+
+func init() {
+	testdataok := _TestDataOK{
+		smallTestv: []testEntry{
+			testEntry{oid: 1, kind: kindBucket, itemv: []kv{}},
+			testEntry{oid: 2, kind: kindBucket, itemv: []kv{{10, int64(17)}, }},
+			testEntry{oid: 3, kind: kindBucket, itemv: []kv{{15, int64(1)}, {23, "hello"}, }},
+			testEntry{oid: 4, kind: kindBTree, itemv: []kv{}},
+			testEntry{oid: 5, kind: kindBTree, itemv: []kv{{5, int64(4)}, }},
+			testEntry{oid: 6, kind: kindBTree, itemv: []kv{{7, int64(3)}, {9, "world"}, }},
+		},
+
+		B3_oid: 7,
+		B3_maxkey: 9999,
+
+		Bv_oid: 8,
+		Bv_kmin: 1,
+		Bv_kmax: 9,
+		Bvdict: map[int64][]tVisit{
+			1: []tVisit{{8, LKeyRange{_LKeyMin, _LKeyMax}}, {9, LKeyRange{_LKeyMin, 3}}, {11, LKeyRange{_LKeyMin, 1}}},
+			2: []tVisit{{8, LKeyRange{_LKeyMin, _LKeyMax}}, {9, LKeyRange{_LKeyMin, 3}}, {16, LKeyRange{2, 3}}},
+			5: []tVisit{{8, LKeyRange{_LKeyMin, _LKeyMax}}, {10, LKeyRange{4, _LKeyMax}}, {12, LKeyRange{4, _LKeyMax}}, {13, LKeyRange{4, 6}}},
+			8: []tVisit{{8, LKeyRange{_LKeyMin, _LKeyMax}}, {10, LKeyRange{4, _LKeyMax}}, {12, LKeyRange{4, _LKeyMax}}, {14, LKeyRange{7, 8}}},
+			9: []tVisit{{8, LKeyRange{_LKeyMin, _LKeyMax}}, {10, LKeyRange{4, _LKeyMax}}, {12, LKeyRange{4, _LKeyMax}}, {15, LKeyRange{9, _LKeyMax}}},
+		},
+	}
+
+	ztestdataReg.Register("py3_pickle3", "testdata/py3_pickle3", &testdataok)
+}

go/zodb/btree/ztestdata_expect_test.go

-// Code generated by ./testdata/gen-testdata; DO NOT EDIT.
-package btree
-
-
-var smallTestv = [...]testEntry{
-	testEntry{oid: 7, kind: kindBucket, itemv: []kv{}},
-	testEntry{oid: 4, kind: kindBucket, itemv: []kv{{10, int64(17)}, }},
-	testEntry{oid: 1, kind: kindBucket, itemv: []kv{{15, int64(1)}, {23, bstr("hello")}, }},
-	testEntry{oid: 3, kind: kindBTree, itemv: []kv{}},
-	testEntry{oid: 8, kind: kindBTree, itemv: []kv{{5, int64(4)}, }},
-	testEntry{oid: 5, kind: kindBTree, itemv: []kv{{7, int64(3)}, {9, bstr("world")}, }},
-}
-
-const B3_oid = 6
-const B3_maxkey = 9999
-
-const Bv_oid = 2
-const Bv_kmin = 1
-const Bv_kmax = 9
-var Bvdict = map[int64][]tVisit{
-	1: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {342, LKeyRange{_LKeyMin, 3}}, {344, LKeyRange{_LKeyMin, 1}}},
-	2: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {342, LKeyRange{_LKeyMin, 3}}, {349, LKeyRange{2, 3}}},
-	5: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {346, LKeyRange{4, 6}}},
-	8: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {347, LKeyRange{7, 8}}},
-	9: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {348, LKeyRange{9, _LKeyMax}}},
-}