go/zodb/btree: New package to work with ZODB BTrees (draft)

Provide minimal support for BTrees.LOBTree Get for now.

go/zodb/btree/btree_test.go

+// Copyright (C) 2018  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package btree
+
+//go:generate ./testdata/gen-testdata
+
+import (
+	"context"
+	"testing"
+
+	"lab.nexedi.com/kirr/go123/exc"
+	"lab.nexedi.com/kirr/neo/go/transaction"
+	"lab.nexedi.com/kirr/neo/go/zodb"
+	_ "lab.nexedi.com/kirr/neo/go/zodb/wks"
+)
+
+// kv is one (key, value) pair.
+type kv struct {
+	key   KEY
+	value interface{}
+}
+
+type bkind int
+const (
+	kindBucket bkind = iota
+	kindBTree
+)
+
+// testEntry is information about a Bucket or a BTree.
+type testEntry struct {
+	oid   zodb.Oid
+	kind  bkind
+	itemv []kv
+}
+
+// bmapping represents Get of Bucket or BTree.
+type bmapping interface {
+	Get(context.Context, KEY) (interface{}, bool, error)
+}
+
+func TestBTree(t *testing.T) {
+	X := exc.Raiseif
+	ctx := context.Background()
+	stor, err := zodb.OpenStorage(ctx, "testdata/1.fs", &zodb.OpenOptions{ReadOnly: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	db := zodb.NewDB(stor)
+
+	txn, ctx := transaction.New(ctx)
+	defer txn.Abort()
+
+	conn, err := db.Open(ctx, &zodb.ConnOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// XXX close db/stor
+
+	// go through small test Buckets/BTrees and verify that Get(key) is as expected.
+	for _, tt := range smallTestv {
+		xobj, err := conn.Get(ctx, tt.oid)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		obj, ok := xobj.(bmapping)
+		if !ok {
+			t.Fatalf("%s: got %T;  want Bucket|BTree", tt.oid, xobj)
+		}
+
+		want := ""
+		switch tt.kind {
+		case kindBucket:
+			if _, ok = obj.(*Bucket); !ok {
+				want = "Bucket"
+			}
+		case kindBTree:
+			if _, ok = obj.(*BTree); !ok {
+				want = "BTree"
+			}
+		default:
+			panic(0)
+		}
+
+		if want != "" {
+			t.Fatalf("%s: got %T;  want %s", tt.oid, obj, want)
+		}
+
+		for _, kv := range tt.itemv {
+			value, ok, err := obj.Get(ctx, kv.key)
+			if err != nil {
+				t.Error(err)
+				continue
+			}
+
+			if !ok {
+				t.Errorf("%s: get %v -> ø;  want %v", tt.oid, kv.key, kv.value)
+				continue
+			}
+
+			if value != kv.value {
+				t.Errorf("%s: get %v -> %v;  want %v", tt.oid, kv.key, value, kv.value)
+			}
+
+			// XXX .next == nil
+			// XXX check keys, values directly (i.e. there is nothing else)
+		}
+	}
+
+
+	// B3 is a large BTree with {i: i} data.
+	// verify Get(key) and that different bucket links lead to the same in-RAM object.
+	xB3, err := conn.Get(ctx, B3_oid)
+	if err != nil {
+		t.Fatal(err)
+	}
+	B3, ok := xB3.(*BTree)
+	if !ok {
+		t.Fatalf("B3: %v; got %T;  want BTree", B3_oid, xB3)
+	}
+
+	for i := KEY(0); i <= B3_maxkey; i++ {
+		v, ok, err := B3.Get(ctx, i)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !ok {
+			t.Fatalf("B3: get %v -> ø;  want %v", i, i)
+		}
+		if int64(i) != v {
+			t.Fatalf("B3: get %v -> %v;  want %v", i, v, i)
+		}
+	}
+
+	// verifyFirstBucket verifies that b.firstbucket is correct and returns it.
+	var verifyFirstBucket func(b *BTree) *Bucket
+	verifyFirstBucket = func(b *BTree) *Bucket {
+		err := b.PActivate(ctx);	X(err)
+		defer b.PDeactivate()
+
+		var firstbucket *Bucket
+
+		switch child := b.data[0].child.(type) {
+		default:
+			t.Fatalf("btree(%s): child[0] is %T", b.POid(), b.data[0].child)
+
+		case *BTree:
+			firstbucket = verifyFirstBucket(child)
+
+		case *Bucket:
+			firstbucket = child
+		}
+
+		if firstbucket != b.firstbucket {
+			t.Fatalf("btree(%s): firstbucket -> %p (oid: %s); actual first bucket = %p (oid: %s)",
+				b.POid(), b.firstbucket, b.firstbucket.POid(), firstbucket, firstbucket.POid())
+		}
+
+		return firstbucket
+	}
+
+	verifyFirstBucket(B3)
+}

go/zodb/btree/testdata/.gitignore

+/*.lock
+/*.tmp
+/*.tr[0-9]

go/zodb/btree/testdata/gen-testdata

+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+# Copyright (C) 2018  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+"""generate test data for btree serialization tests"""
+
+from ZODB.DB import DB
+from BTrees.LOBTree import LOBucket, LOBTree
+from ZODB.utils import u64
+import os, os.path, transaction
+
+from golang.gcompat import qq
+
+def rm_f(path):
+    if os.path.exists(path):
+        os.remove(path)
+
+def main():
+    import zodbtools.test.gen_testdata  # to make time predictable (XXX)
+
+    outfs = "testdata/1.fs"
+    rm_f(outfs)
+    rm_f(outfs + ".index")
+    db = DB(outfs)
+    conn = db.open()
+    root = conn.root()
+
+    root['b0'] = b0 = LOBucket()                         # empty bucket
+    root['b1'] = b1 = LOBucket([(10, 17)])               # 1k -> 1v
+    root['b2'] = b2 = LOBucket([(15, 1), (23, "hello")]) # 2k -> 2v
+
+    root['B0'] = B0 = LOBTree()                          # empty btree
+    root['B1'] = B1 = LOBTree({5: 4})                    # btree with 1 bucket (1kv)
+    root['B2'] = B2 = LOBTree({7: 3, 9: "world"})        # btree with 1 bucket (2kv)
+
+    root['B3'] = B3 = LOBTree(dict([(_, _) for _ in range(10000)]))
+
+    transaction.commit()
+
+    with open("ztestdata_expect_test.go", "w") as f:
+        def emit(v):
+            print >>f, v
+        emit("// Code generated by %s; DO NOT EDIT." % __file__)
+        emit("package btree\n")
+        #emit("import \"lab.nexedi.com/kirr/neo/go/zodb\"\n")
+
+        def emititems(b):
+            s = "testEntry{oid: %s, kind: %s, itemv: []kv{" \
+                    % (u64(b._p_oid), "kind%s" % type(b).__name__[2:])
+            for k, v in b.items():
+                if isinstance(v, str):
+                    v = qq(v)
+                elif isinstance(v, int):
+                    v = "int64(%d)" % v
+                else:
+                    raise RuntimeError("unsupported value type: %r" % v)
+
+                s += "{%s, %s}, " % (k, v)
+            s += "}},"
+            emit("\t"+s)
+
+        emit("\nvar smallTestv = [...]testEntry{")
+        for b in (b0, b1, b2, B0, B1, B2):
+            emititems(b)
+        emit("}")
+
+        emit("\nconst B3_oid = %s" % u64(B3._p_oid))
+        emit("const B3_maxkey = %d" % B3.maxKey())
+
+
+    conn.close()
+    db.close()
+
+
+if __name__ == '__main__':
+    main()

go/zodb/btree/ztestdata_expect_test.go

+// Code generated by ./testdata/gen-testdata; DO NOT EDIT.
+package btree
+
+
+var smallTestv = [...]testEntry{
+	testEntry{oid: 6, kind: kindBucket, itemv: []kv{}},
+	testEntry{oid: 3, kind: kindBucket, itemv: []kv{{10, int64(17)}, }},
+	testEntry{oid: 1, kind: kindBucket, itemv: []kv{{15, int64(1)}, {23, "hello"}, }},
+	testEntry{oid: 2, kind: kindBTree, itemv: []kv{}},
+	testEntry{oid: 7, kind: kindBTree, itemv: []kv{{5, int64(4)}, }},
+	testEntry{oid: 4, kind: kindBTree, itemv: []kv{{7, int64(3)}, {9, "world"}, }},
+}
+
+const B3_oid = 5
+const B3_maxkey = 9999

go/zodb/zodb.go

@@ -144,6 +144,10 @@
 // level types that are registered with state type providing PyStateful (see
 // RegisterClass) are automatically (de)serialized as Python pickles(*).
 //
+// An example of application-level type with ZODB/py compatibility can be seen in
+// package lab.nexedi.com/kirr/neo/go/zodb/btree which provides BTree handling
+// for ZODB/go.
+//
 // --------
 //
 // (*) for pickle support package github.com/kisielk/og-rek is used.