go/internal/xtesting: LoadDBHistory

This is utility to load whole content of a ZODB database.

It will be useful to load data to expect from e.g. dataok.fs and feed it
as txnvOk to DrvTestLoad when verifying another storage driver.

go/internal/xtesting/xtesting.go

@@ -24,6 +24,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"net/url"
 	"os"
 	"os/exec"
@@ -147,6 +148,66 @@ type Txn struct {
 	Data   []*zodb.DataInfo
 }
 
+// LoadDBHistory loads whole content of a ZODB database.
+//
+// it returns full history of all transactions with committed data.
+func LoadDBHistory(zurl string) (_ []Txn, err error) {
+	xerr.Contextf(&err, "loadDBHistory %s", zurl)
+	ctx := context.Background()
+	zstor, err := zodb.Open(ctx, zurl, &zodb.OpenOptions{ReadOnly: true})
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		err2 := zstor.Close()
+		if err == nil {
+			err = err2
+		}
+	}()
+
+	txnv := []Txn{}
+	it := zstor.Iterate(ctx, 0, zodb.TidMax)
+	for {
+		txni, dataIt, err := it.NextTxn(ctx)
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, err
+		}
+		txn := Txn{Header: &zodb.TxnInfo{
+			// clone (txni stays valid only until next iteration)
+			Tid:         txni.Tid,
+			Status:      txni.Status,
+			User:        bcopy(txni.User),
+			Description: bcopy(txni.Description),
+			Extension:   bcopy(txni.Extension),
+		}}
+
+		for {
+			datai, err := dataIt.NextData(ctx)
+			if err != nil {
+				if err == io.EOF {
+					break
+				}
+				return nil, err
+			}
+
+			txn.Data = append(txn.Data, &zodb.DataInfo{
+				// clone (datai stays valid only until next iteration)
+				Oid:         datai.Oid,
+				Tid:         datai.Tid,
+				Data:        bcopy(datai.Data),
+				DataTidHint: datai.DataTidHint,
+			})
+		}
+
+		txnv = append(txnv, txn)
+	}
+
+	return txnv, nil
+}
+
 // checkLoad verifies that zdrv.Load(xid) returns expected result.
 func checkLoad(t *testing.T, zdrv zodb.IStorageDriver, xid zodb.Xid, expect objState) {
 	t.Helper()
@@ -346,3 +407,13 @@ func FatalIf(t *testing.T) func(error) {
 func b(data string) []byte {
 	return []byte(data)
 }
+
+// bcopy makes a clone of byte slice.
+func bcopy(data []byte) []byte {
+	if data == nil {
+		return nil
+	}
+	d2 := make([]byte, len(data))
+	copy(d2, data)
+	return d2
+}

go/internal/xtesting/xtesting_test.go

+// Copyright (C) 2020  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package xtesting
+
+import (
+	"context"
+	"testing"
+
+	"lab.nexedi.com/kirr/neo/go/zodb"
+	"lab.nexedi.com/kirr/neo/go/zodb/storage/fs1"
+)
+
+func TestLoadDBHistory(t *testing.T) {
+	X := FatalIf(t)
+	data := "../../zodb/storage/fs1/testdata/1.fs"
+
+	txnv, err := LoadDBHistory(data); X(err)
+
+	// verify the history by running DrvTestLoad against original data.
+	// FileStorage verifies its Load and Iterate without relying on LoadDBHistory.
+	f, _, err := fs1.Open(context.Background(), data, &zodb.DriverOptions{ReadOnly: true}); X(err)
+	defer func() {
+		err := f.Close(); X(err)
+	}()
+
+	DrvTestLoad(t, f, txnv)
+}

go/zodb/storage/fs1/filestorage_test.go

@@ -76,6 +76,9 @@ func TestLoad(t *testing.T) {
 	fs, _ := xfsopen(t, "testdata/1.fs")
 	defer exc.XRun(fs.Close)
 
+	// NOTE don't use xtesting.LoadDBHistory here - it is itself tested
+	// with the assumption that fs1.Load and fs1.Iterate work correctly.
+	// Use what testdata generator gave use with what to expect.
 	txnv := []xtesting.Txn{}
 	for _, dbe := range _1fs_dbEntryv {
 		txn := xtesting.Txn{Header: &zodb.TxnInfo{