go/zodb: Test PyData and Map/List on all py2/py3 kinds of data we care about

Previously we were testing PyData and Map/Lust only with data generated by python2
and pickle protocol=2. However even on py2 there are more pickle
protocols that are in use, and also there is python3.

-> Modernize py/pydata-gen-testdata to use run_with_all_zodb_pickle_kinds
   that was recently added as part of nexedi/zodbtools@f9d36ba7
   and generate test data with both python2 and python3. It is handy to
   use py2py3-venv(*) to prepare python environment to do that.

   Adjust tests on Go side to verify how PyData and Map/List handle all generated
   zkinds.

py2_pickle1, py2_pickle2 and py2_pickle3 are handled well.
Tests for py3_pickle3 currently fail and so are marked with "xfail".

We will fix tests for py3_pickle3 in the next patches.

(*) see nexedi/zodbtools@fac2f190

go/zodb/persistent_x_test.go

@@ -25,16 +25,29 @@ import (
 	"lab.nexedi.com/kirr/neo/go/zodb"
 	pickle "github.com/kisielk/og-rek"
 
+	"lab.nexedi.com/kirr/neo/go/internal/xtesting"
 	assert "github.com/stretchr/testify/require"
 )
 
 
+// ztestdataReg maintains registry of all entries under testdata/ .
+var  ztestdataReg = xtesting.ZTestDataRegistry[struct{}]{}
+type ZTestData    = xtesting.ZTestData[struct{}]
+
+
 // Verify that Map and List can load from data saved by py.
 // TODO go saves Map/List -> py loads and checks.
-func TestPersistentMapListLoad(t0 *testing.T) {
+func TestPersistentMapListLoad(t *testing.T) {
+	ztestdataReg.RunWithEach(t, _TestPersistentMapListLoad)
+}
+func _TestPersistentMapListLoad(t0 *testing.T, z *ZTestData) {
+	if z.Kind == "py3_pickle3" {
+		t0.Skip("xfail")
+	}
+
 	assert := assert.New(t0)
 
-	tdb := zodb.OpenTestDB(t0, "testdata/data.fs")
+	tdb := zodb.OpenTestDB(t0, z.Path("data.fs"))
 	defer tdb.Close()
 
 	t := tdb.Open(&zodb.ConnOptions{})

go/zodb/py/pydata-gen-testdata

-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright (C) 2017-2024  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
@@ -20,23 +20,35 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 """generate reference database and pickle objects encoding for tests"""
 
+from __future__ import print_function
+
 from ZODB import DB, serialize
 from ZODB.utils import z64
 from persistent.mapping import PersistentMapping
 from persistent.list import PersistentList
 import transaction
-from zodbtools.test.gen_testdata import run_with_zodb4py2_compat
+from zodbtools.test.gen_testdata import run_with_all_zodb_pickle_kinds, current_zkind
 from golang.gcompat import qq
-from os import remove
-from os.path import exists
+from os import makedirs
+from os.path import exists, relpath
+from shutil import rmtree
+
+from six import PY3
+if PY3:
+    from importlib import reload
+else:
+    reload  # available as builtin
 
 
 # gen_ztestdata generates test databases that ZODB/go will try to load.
 def gen_ztestdata():
-    outfs = "testdata/data.fs"
-    if exists(outfs):
-        remove(outfs)
-        remove(outfs+".index")
+    zkind = current_zkind()
+    prefix = "testdata/%s" % zkind
+    if exists(prefix):
+        rmtree(prefix)
+    makedirs(prefix)
+
+    outfs = "%s/data.fs" % prefix
     db = DB(outfs)
     conn = db.open()
     root = conn.root()
@@ -50,17 +62,29 @@ def gen_ztestdata():
 
     transaction.commit()
 
+    with open("ztestdata_%s_x_test.go" % zkind, "w") as f:
+        def emit(v):
+            print(v, file=f)
+        emit("// Code generated by %s; DO NOT EDIT." % relpath(__file__))
+        emit("package zodb_test")
+        emit("func init() {")
+        emit("\tztestdataReg.Register(%s, %s, nil)" % (qq(zkind), qq(prefix)))
+        emit("}")
+
 
 # gen_test_pydata generates testdata for PyData serialization tests.
 def gen_test_pydata():
     # import ZODB.tests at runtime after ZODB.X._protocol is patched
+    # reload the module each time because SerializerTestCase uses make_pickle to initialize class-level attributes
     from ZODB.tests import testSerialize
+    reload(testSerialize)
 
     # dump to go what to expect
-    with open("ztestdata_pydata_test.go", "w") as f:
+    zkind = current_zkind()
+    with open("ztestdata_pydata_%s_test.go" % zkind, "w") as f:
         def emit(v):
-            print >>f, v
-        emit("// Code generated by %s; DO NOT EDIT." % __file__)
+            print(v, file=f)
+        emit("// Code generated by %s; DO NOT EDIT." % relpath(__file__))
         emit("package zodb")
 
         # [] of pickle
@@ -74,18 +98,20 @@ def gen_test_pydata():
 
         r = serialize.ObjectReader(factory=testSerialize._factory)
 
-        emit("\nvar _PyData_ClassName_Testv = [...]_PyDataClassName_TestEntry{")
+        emit("func init() {")
+        emit("\t_PyDataClassName_TestDataRegistry[%s] = []_PyDataClassName_TestEntry{" % qq(zkind))
         for test in testv:
-            emit("\t{")
-            emit("\t\t%s," % qq(test))
-            emit("\t\t%s," % qq(r.getClassName(test)))
-            emit("\t},")
-        emit('\t{"aaa", "?.?"},') # invalid
+            emit("\t\t{")
+            emit("\t\t\t%s," % qq(test))
+            emit("\t\t\t%s," % qq(r.getClassName(test)))
+            emit("\t\t},")
+        emit('\t\t{"aaa", "?.?"},') # invalid
+        emit("\t}")
         emit("}")
 
 def main():
-    run_with_zodb4py2_compat(gen_ztestdata)
-    run_with_zodb4py2_compat(gen_test_pydata)
+    run_with_all_zodb_pickle_kinds(gen_ztestdata)
+    run_with_all_zodb_pickle_kinds(gen_test_pydata)
 
 if __name__ == '__main__':
     main()

go/zodb/pydata_test.go

-// Copyright (C) 2016-2017  Nexedi SA and Contributors.
+// Copyright (C) 2016-2024  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -19,10 +19,13 @@
 
 package zodb
 
-//go:generate ./py/pydata-gen-testdata
+//go:generate python2 py/pydata-gen-testdata
+//go:generate python3 py/pydata-gen-testdata
 
 import (
 	"testing"
+
+	"lab.nexedi.com/kirr/neo/go/internal/xmaps"
 )
 
 type _PyDataClassName_TestEntry struct {
@@ -30,7 +33,20 @@ type _PyDataClassName_TestEntry struct {
 	className string
 }
 
+var _PyDataClassName_TestDataRegistry = map[/*zkind*/string][]_PyDataClassName_TestEntry{}
+
 func TestPyClassName(t *testing.T) {
+	for _, zkind := range xmaps.SortedKeys(_PyDataClassName_TestDataRegistry) {
+		t.Run(zkind, func(t *testing.T) {
+			if zkind == "py3_pickle3" {
+				t.Skip("xfail")
+			}
+			_TestPyClassName(t, _PyDataClassName_TestDataRegistry[zkind])
+		})
+	}
+}
+
+func _TestPyClassName(t *testing.T, _PyData_ClassName_Testv []_PyDataClassName_TestEntry) {
 	for _, tt := range _PyData_ClassName_Testv {
 		className := PyData(tt.pydata).ClassName()
 		if className != tt.className {

go/zodb/testdata/py2_pickle1/data.fs.index

+K.N.
\ No newline at end of file

go/zodb/testdata/py2_pickle3/data.fs.index

+K.N.
\ No newline at end of file

go/zodb/testdata/py3_pickle3/data.fs.index

+K.N.
\ No newline at end of file

go/zodb/ztestdata_py2_pickle1_x_test.go

+// Code generated by py/pydata-gen-testdata; DO NOT EDIT.
+package zodb_test
+func init() {
+	ztestdataReg.Register("py2_pickle1", "testdata/py2_pickle1", nil)
+}

go/zodb/ztestdata_py2_pickle2_x_test.go

+// Code generated by py/pydata-gen-testdata; DO NOT EDIT.
+package zodb_test
+func init() {
+	ztestdataReg.Register("py2_pickle2", "testdata/py2_pickle2", nil)
+}

go/zodb/ztestdata_py2_pickle3_x_test.go

+// Code generated by py/pydata-gen-testdata; DO NOT EDIT.
+package zodb_test
+func init() {
+	ztestdataReg.Register("py2_pickle3", "testdata/py2_pickle3", nil)
+}

go/zodb/ztestdata_py3_pickle3_x_test.go

+// Code generated by py/pydata-gen-testdata; DO NOT EDIT.
+package zodb_test
+func init() {
+	ztestdataReg.Register("py3_pickle3", "testdata/py3_pickle3", nil)
+}

go/zodb/ztestdata_pydata_py2_pickle1_test.go

+// Code generated by py/pydata-gen-testdata; DO NOT EDIT.
+package zodb
+func init() {
+	_PyDataClassName_TestDataRegistry["py2_pickle1"] = []_PyDataClassName_TestEntry{
+		{
+			"((U\x18ZODB.tests.testSerializeq\x01U\x13ClassWithoutNewargsq\x02tNtq\x03.",
+			"ZODB.tests.testSerialize.ClassWithoutNewargs",
+		},
+		{
+			"((U\x18ZODB.tests.testSerializeq\x01U\x10ClassWithNewargsq\x02t(K\x01tq\x03tq\x04.",
+			"ZODB.tests.testSerialize.ClassWithNewargs",
+		},
+		{
+			"cZODB.tests.testSerialize\nClassWithoutNewargs\nq\x01.",
+			"ZODB.tests.testSerialize.ClassWithoutNewargs",
+		},
+		{
+			"(cZODB.tests.testSerialize\nClassWithNewargs\nq\x01(K\x01tq\x02tq\x03.",
+			"ZODB.tests.testSerialize.ClassWithNewargs",
+		},
+		{"aaa", "?.?"},
+	}
+}

go/zodb/ztestdata_pydata_py2_pickle2_test.go

+// Code generated by py/pydata-gen-testdata; DO NOT EDIT.
+package zodb
+func init() {
+	_PyDataClassName_TestDataRegistry["py2_pickle2"] = []_PyDataClassName_TestEntry{
+		{
+			"\x80\x02U\x18ZODB.tests.testSerializeq\x01U\x13ClassWithoutNewargsq\x02\x86N\x86q\x03.",
+			"ZODB.tests.testSerialize.ClassWithoutNewargs",
+		},
+		{
+			"\x80\x02U\x18ZODB.tests.testSerializeq\x01U\x10ClassWithNewargsq\x02\x86K\x01\x85q\x03\x86q\x04.",
+			"ZODB.tests.testSerialize.ClassWithNewargs",
+		},
+		{
+			"\x80\x02cZODB.tests.testSerialize\nClassWithoutNewargs\nq\x01.",
+			"ZODB.tests.testSerialize.ClassWithoutNewargs",
+		},
+		{
+			"\x80\x02cZODB.tests.testSerialize\nClassWithNewargs\nq\x01K\x01\x85q\x02\x86q\x03.",
+			"ZODB.tests.testSerialize.ClassWithNewargs",
+		},
+		{"aaa", "?.?"},
+	}
+}

go/zodb/ztestdata_pydata_py2_pickle3_test.go

+// Code generated by py/pydata-gen-testdata; DO NOT EDIT.
+package zodb
+func init() {
+	_PyDataClassName_TestDataRegistry["py2_pickle3"] = []_PyDataClassName_TestEntry{
+		{
+			"\x80\x03U\x18ZODB.tests.testSerializeq\x01U\x13ClassWithoutNewargsq\x02\x86N\x86q\x03.",
+			"ZODB.tests.testSerialize.ClassWithoutNewargs",
+		},
+		{
+			"\x80\x03U\x18ZODB.tests.testSerializeq\x01U\x10ClassWithNewargsq\x02\x86K\x01\x85q\x03\x86q\x04.",
+			"ZODB.tests.testSerialize.ClassWithNewargs",
+		},
+		{
+			"\x80\x03cZODB.tests.testSerialize\nClassWithoutNewargs\nq\x01.",
+			"ZODB.tests.testSerialize.ClassWithoutNewargs",
+		},
+		{
+			"\x80\x03cZODB.tests.testSerialize\nClassWithNewargs\nq\x01K\x01\x85q\x02\x86q\x03.",
+			"ZODB.tests.testSerialize.ClassWithNewargs",
+		},
+		{"aaa", "?.?"},
+	}
+}

go/zodb/ztestdata_pydata_py3_pickle3_test.go

+// Code generated by py/pydata-gen-testdata; DO NOT EDIT.
+package zodb
+func init() {
+	_PyDataClassName_TestDataRegistry["py3_pickle3"] = []_PyDataClassName_TestEntry{
+		{
+			"\x80\x03X\x18\x00\x00\x00ZODB.tests.testSerializeq\x00X\x13\x00\x00\x00ClassWithoutNewargsq\x01\x86q\x02N\x86q\x03.",
+			"ZODB.tests.testSerialize.ClassWithoutNewargs",
+		},
+		{
+			"\x80\x03X\x18\x00\x00\x00ZODB.tests.testSerializeq\x00X\x10\x00\x00\x00ClassWithNewargsq\x01\x86q\x02K\x01\x85q\x03\x86q\x04.",
+			"ZODB.tests.testSerialize.ClassWithNewargs",
+		},
+		{
+			"\x80\x03cZODB.tests.testSerialize\nClassWithoutNewargs\nq\x00.",
+			"ZODB.tests.testSerialize.ClassWithoutNewargs",
+		},
+		{
+			"\x80\x03cZODB.tests.testSerialize\nClassWithNewargs\nq\x00K\x01\x85q\x01\x86q\x02.",
+			"ZODB.tests.testSerialize.ClassWithNewargs",
+		},
+		{"aaa", "?.?"},
+	}
+}

go/zodb/ztestdata_pydata_test.go

-// Code generated by ./py/pydata-gen-testdata; DO NOT EDIT.
-package zodb
-
-var _PyData_ClassName_Testv = [...]_PyDataClassName_TestEntry{
-	{
-		"\x80\x02U\x18ZODB.tests.testSerializeq\x01U\x13ClassWithoutNewargsq\x02\x86N\x86q\x03.",
-		"ZODB.tests.testSerialize.ClassWithoutNewargs",
-	},
-	{
-		"\x80\x02U\x18ZODB.tests.testSerializeq\x01U\x10ClassWithNewargsq\x02\x86K\x01\x85q\x03\x86q\x04.",
-		"ZODB.tests.testSerialize.ClassWithNewargs",
-	},
-	{
-		"\x80\x02cZODB.tests.testSerialize\nClassWithoutNewargs\nq\x01.",
-		"ZODB.tests.testSerialize.ClassWithoutNewargs",
-	},
-	{
-		"\x80\x02cZODB.tests.testSerialize\nClassWithNewargs\nq\x01K\x01\x85q\x02\x86q\x03.",
-		"ZODB.tests.testSerialize.ClassWithNewargs",
-	},
-	{"aaa", "?.?"},
-}