.

wcfs/testdata/zblk_test_gen.py

@@ -23,14 +23,14 @@
 from ZODB.DB import DB
 from ZODB.utils import u64
 from wendelin.bigfile.file_zodb import ZBlk0, ZBlk1, ZBigFile
+from BTrees.IOBTree import IOBTree, IOBucket
 from numpy import arange
 import os, os.path, transaction
 
 import zodbtools.test.gen_testdata  # to make time predictable
 from zodbtools.test.gen_testdata import run_with_zodb3py2_compat
 
-blksize   = 2*1024*1024   # XXX hardcoded
-blksize32 = blksize // 4
+K = 1024
 
 def rm_f(path):
     if os.path.exists(path):
@@ -47,20 +47,28 @@ def main2():
     conn = db.open()
     root = conn.root()
 
-    # 0, 1, 2, ... as u32
-    #
-    # XXX also include hole inside and at tail.
-    data = arange(0, blksize32, dtype='>u4').tobytes()
 
     root['zblk0'] = z0 = ZBlk0()
     root['zblk1'] = z1 = ZBlk1()    # also covers ZData
-    root['zbigf'] = zf = ZBigFile(blksize)
-
-    z0.setblkdata(data)
-    z1.setblkdata(data)
+    root['zbigf'] = zf = ZBigFile(2*1024*K)
+
+    # zblk0 with small data
+    z0.setblkdata(brange32(16*K))
+    # zblk1 with head + large hole + tail.
+    # head and tail are 128K because if smaller, z1.chunktab would serialize without buckets.
+    z1ht = 128*K
+    z1.setblkdata(brange32(z1ht) + bzeros(zf.blksize - 2*z1ht) + breverse(brange32(z1ht)))
     zf.blktab[0] = z0
     zf.blktab[1] = z1
 
+    # make sure there is at least one bucket in Z1's chunktab
+    #
+    # we need at least one bucket to verify how zblk.go handles chunktab bucket loading;
+    # more than 1 bucket just wastes space, however with only 1 bucket a tree
+    # is serialized to have leaf nodes diectly in the tree state.
+    # -> verify for ==2 buckets.
+    assertIOBTreeHas2Buckets(z1.chunktab)
+
     transaction.commit()
 
     with open("ztestdata_zblk_test.go", "w") as f:
@@ -70,17 +78,50 @@ def main2():
         emit("package main\n")
         emit('import "lab.nexedi.com/kirr/neo/go/zodb"\n')
 
-        emit("const blksize = %s" % blksize)
-        emit("const z0_oid = %s" % u64(z0._p_oid))
-        emit("const z1_oid = %s" % u64(z1._p_oid))
-        emit("const zf_oid = %s" % u64(zf._p_oid))
-        emit("const z0_rev = zodb.Tid(0x%x)"  % u64(z0._p_serial))
-        emit("const z1_rev = zodb.Tid(0x%x)"  % u64(z1._p_serial))
+        emit("const zf_blksize = %d" % zf.blksize)
+        emit("const z0_oid     = zodb.Oid(%d)" % u64(z0._p_oid))
+        emit("const z1_oid     = zodb.Oid(%d)" % u64(z1._p_oid))
+        emit("const zf_oid     = zodb.Oid(%d)" % u64(zf._p_oid))
+        emit("const z0_rev     = zodb.Tid(0x%x)"  % u64(z0._p_serial))
+        emit("const z1_rev     = zodb.Tid(0x%x)"  % u64(z1._p_serial))
+        emit("const z0_len     = %d" % len(z0.loadblkdata()))
+        emit("const z1_htlen   = %d" % z1ht)
 
 
     conn.close()
     db.close()
 
 
+# brange32 returns bytes with big-endian uint32 seqence filling them.
+# returned bytes has len == size.
+def brange32(size):
+    # 0, 1, 2, ... as u32
+    return arange(0, size//4, dtype='>u4').tobytes()
+
+# bzeros returns bytes of requested size with 0 filling them.
+def bzeros(size):
+    return b'\0'*size
+
+# breverse returns bytes in the reverse order.
+def breverse(b):
+    assert isinstance(b, bytes)
+    _ = bytearray(b)
+    _.reverse()
+    return bytes(_)
+
+# assertIOBTreeHas2Buckets asserts that IOBTree has 2 buckets.
+def assertIOBTreeHas2Buckets(t):
+    assert isinstance(t, IOBTree)
+
+    # https://github.com/zopefoundation/BTrees/blob/4.5.0-1-gc8bf24e/BTrees/BTreeTemplate.c#L1087-L1109
+    _ = t.__getstate__()
+    assert len(_) == 2, (len(_), _)
+    assert len(_[0]) == 3, _[0]                   # (bucket0, key1, bucket1)
+    assert isinstance(_[0][0], IOBucket), _[0][0] # bucket0
+    assert isinstance(_[0][1], int),      _[0][2] # key1
+    assert isinstance(_[0][2], IOBucket), _[0][2] # bucket1
+    assert isinstance(_[1],    IOBucket), _[1]    # .firstbucket
+
+
 if __name__ == '__main__':
     main()

wcfs/zblk_test.go

@@ -21,11 +21,8 @@ package main
 
 //go:generate ./testdata/zblk_test_gen.py
 
-// XXX do not commit 4MB (ZBlk*) to testdata - generate them at test runtime.
-// XXX save into git only small-sized ZBlk0 and ZBlk1 so that we can be sure
-//     their format does not change.
-
 import (
+	"bytes"
 	"context"
 	"encoding/binary"
 	"testing"
@@ -38,6 +35,8 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+const K = 1024
+
 // TestZBlk verifies that ZBlk* and ZBigFile saved by Python can be read correctly by Go.
 func TestZBlk(t *testing.T) {
 	X := exc.Raiseif
@@ -81,25 +80,24 @@ func TestZBlk(t *testing.T) {
 		}
 	}
 
-	// data that must be stored in ZBlk*
-	data := make([]byte, blksize)
-	for i := uint32(0); i < blksize / 4; i++ {
-		binary.BigEndian.PutUint32(data[i*4:], i)
-	}
 
 	z0Data, z0Rev, err := z0.loadBlkData(ctx);	X(err)
-	assert.Equal(z0Data, data,   "ZBlk0 data wrong")
-	assert.Equal(z0Rev,  z0_rev, "ZBlk0 rev wrong")
+	assert.Equal(z0Data, brange32(z0_len),  "ZBlk0 data wrong")
+	assert.Equal(z0Rev,  z0_rev,	        "ZBlk0 rev wrong")
 
 	z1Data, z1Rev, err := z1.loadBlkData(ctx);	X(err)
-	assert.Equal(z1Data, data,   "ZBlk1 data wrong")
-	assert.Equal(z1Rev,  z1_rev, "ZBlk1 rev wrong")
+	z1DataOK := make([]byte, zf_blksize)                                  // zeros
+	copy(z1DataOK[0:],                      brange32(z1_htlen))           // head
+	copy(z1DataOK[len(z1DataOK)-z1_htlen:], breverse(brange32(z1_htlen))) // tail
+	z1DataOK = bytes.TrimRight(z1DataOK, "\x00") // strip trailing 0 as they are not persisted
+	assert.Equal(z1Data, z1DataOK,   "ZBlk1 data wrong")
+	assert.Equal(z1Rev,  z1_rev,     "ZBlk1 rev wrong")
 
 
 
 	xactivate(zf)
-	if zf.blksize != blksize {
-		t.Fatalf("zf: blksize=%d;  want %d", zf.blksize, blksize)
+	if zf.blksize != zf_blksize {
+		t.Fatalf("zf: blksize=%d;  want %d", zf.blksize, zf_blksize)
 	}
 
 	z0_, ok, err := zf.blktab.Get(ctx, 0);	X(err)
@@ -117,3 +115,23 @@ func TestZBlk(t *testing.T) {
 	// XXX check zf.LoadBlk()
 	// XXX PyGetState vs PySetState
 }
+
+
+// brange32 returns bytes with big-endian uint32 seqence filling them.
+// returned bytes has len == size.
+func brange32(size int) []byte {
+	data := make([]byte, size)
+	for i := 0; i < size / 4; i++ {
+		binary.BigEndian.PutUint32(data[i*4:], uint32(i))
+	}
+	return data
+}
+
+// breverse returns bytes in the reverse order.
+func breverse(b []byte) []byte {
+	r := make([]byte, len(b))
+	for i := range(b) {
+		r[i] = b[len(b)-i-1]
+	}
+	return r
+}

wcfs/ztestdata_zblk_test.go

@@ -3,9 +3,11 @@ package main
 
 import "lab.nexedi.com/kirr/neo/go/zodb"
 
-const blksize = 2097152
-const z0_oid = 2
-const z1_oid = 3
-const zf_oid = 1
-const z0_rev = zodb.Tid(0x285cbac3851eb99)
-const z1_rev = zodb.Tid(0x285cbac3851eb99)
+const zf_blksize = 2097152
+const z0_oid     = zodb.Oid(2)
+const z1_oid     = zodb.Oid(3)
+const zf_oid     = zodb.Oid(1)
+const z0_rev     = zodb.Tid(0x285cbac3851eb99)
+const z1_rev     = zodb.Tid(0x285cbac3851eb99)
+const z0_len     = 16384
+const z1_htlen   = 131072