.

go/neo/server/cluster_test.go

@@ -486,7 +486,7 @@ func TestMasterStorage(t *testing.T) {
 	pg.Done()
 
 	for {
-		_, dataIter, err := ziter.NextTxn()
+		_, dataIter, err := ziter.NextTxn(bg)
 		if err == io.EOF {
 			break
 		}
@@ -495,7 +495,7 @@ func TestMasterStorage(t *testing.T) {
 		}
 
 		for {
-			datai, err := dataIter.NextData()
+			datai, err := dataIter.NextData(bg)
 			if err == io.EOF {
 				break
 			}

go/zodb/storage/fs1/filestorage.go

@@ -205,19 +205,17 @@ func (fs *FileStorage) Close() error {
 
 
 func (fs *FileStorage) LastTid(_ context.Context) (zodb.Tid, error) {
-	// XXX check we have transactions at all
-	// XXX what to return if not?
+	// XXX check we have transactions at all - what to return if not?
 	// XXX must be under lock
-	return fs.txnhMax.Tid, nil	// XXX error always nil ?
+	return fs.txnhMax.Tid, nil
 }
 
 func (fs *FileStorage) LastOid(_ context.Context) (zodb.Oid, error) {
-	// XXX check we have objects at all?
-	// XXX what to return if not?
+	// XXX check we have objects at all - what to return if not?
 	// XXX must be under lock
 	// XXX what if an oid was deleted?
 	lastOid, _ := fs.index.Last() // returns zero-value, if empty
-	return lastOid, nil	// XXX error always nil?
+	return lastOid, nil
 }
 
 // ErrXidLoad is returned when there is an error while loading xid
@@ -334,7 +332,7 @@ const (
 )
 
 // NextTxn iterates to next/previous transaction record according to iteration direction
-func (zi *zIter) NextTxn() (*zodb.TxnInfo, zodb.IDataIterator, error) {
+func (zi *zIter) NextTxn(_ context.Context) (*zodb.TxnInfo, zodb.IDataIterator, error) {
 	switch {
 	case zi.zFlags & zIterEOF != 0:
 		//println("already eof")
@@ -366,7 +364,7 @@ func (zi *zIter) NextTxn() (*zodb.TxnInfo, zodb.IDataIterator, error) {
 }
 
 // NextData iterates to next data record and loads data content
-func (zi *zIter) NextData() (*zodb.DataInfo, error) {
+func (zi *zIter) NextData(_ context.Context) (*zodb.DataInfo, error) {
 	err := zi.iter.NextData()
 	if err != nil {
 		return nil, err	// XXX recheck
@@ -395,7 +393,8 @@ func (zi *zIter) NextData() (*zodb.DataInfo, error) {
 
 
 
-// iterStartError is the iterator created when there are preparatory errors
+// iterStartError is the iterator created when there are preparatory errors.
+//
 // this way we offload clients, besides handling NextTxn errors, from also
 // handling error cases from Iterate.
 //
@@ -404,7 +403,7 @@ type iterStartError struct {
 	err error
 }
 
-func (e *iterStartError) NextTxn() (*zodb.TxnInfo, zodb.IDataIterator, error) {
+func (e *iterStartError) NextTxn(_ context.Context) (*zodb.TxnInfo, zodb.IDataIterator, error) {
 	return nil, nil, e.err
 }
 

go/zodb/storage/fs1/filestorage_test.go

@@ -145,10 +145,11 @@ func TestLoad(t *testing.T) {
 
 // iterate tidMin..tidMax and expect db entries in expectv
 func testIterate(t *testing.T, fs *FileStorage, tidMin, tidMax zodb.Tid, expectv []dbEntry) {
+	ctx := context.Background()
 	iter := fs.Iterate(tidMin, tidMax)
 	fsi, ok := iter.(*zIter)
 	if !ok {
-		_, _, err := iter.NextTxn()
+		_, _, err := iter.NextTxn(ctx)
 		t.Errorf("iterating %v..%v: iter type is %T  ; want zIter\nNextTxn gives: _, _, %v", tidMin, tidMax, iter, err)
 		return
 	}
@@ -161,7 +162,7 @@ func testIterate(t *testing.T, fs *FileStorage, tidMin, tidMax zodb.Tid, expectv
 			t.Errorf("%v: %v", subj, msg)
 		}
 
-		txni, dataIter, err := iter.NextTxn()
+		txni, dataIter, err := iter.NextTxn(ctx)
 		if err != nil {
 			if err == io.EOF {
 				if k != len(expectv) {
@@ -203,7 +204,7 @@ func testIterate(t *testing.T, fs *FileStorage, tidMin, tidMax zodb.Tid, expectv
 				txnErrorf("%v: %v", dsubj, msg)
 			}
 
-			datai, err := dataIter.NextData()
+			datai, err := dataIter.NextData(ctx)
 			if err != nil {
 				if err == io.EOF {
 					if kdata != len(dbe.Entryv) {
@@ -293,12 +294,14 @@ func BenchmarkIterate(b *testing.B) {
 	fs := xfsopen(b, "testdata/1.fs")	// TODO open ro
 	defer exc.XRun(fs.Close)
 
+	ctx := context.Background()
+
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		iter := fs.Iterate(zodb.Tid(0), zodb.TidMax)
 
 		for {
-			txni, dataIter, err := iter.NextTxn()
+			txni, dataIter, err := iter.NextTxn(ctx)
 			if err != nil {
 				if err == io.EOF {
 					break
@@ -310,7 +313,7 @@ func BenchmarkIterate(b *testing.B) {
 			_ = txni.Tid
 
 			for {
-				datai, err := dataIter.NextData()
+				datai, err := dataIter.NextData(ctx)
 				if err != nil {
 					if err == io.EOF {
 						break

go/zodb/storage/fs1/fs1tools/dump_test.go

@@ -31,8 +31,7 @@ import (
 
 	"lab.nexedi.com/kirr/neo/go/zodb/storage/fs1"
 
-	"github.com/sergi/go-diff/diffmatchpatch"
-	//diffpkg "github.com/kylelemons/godebug/diff"
+	"github.com/kylelemons/godebug/diff"
 )
 
 // XXX -> xtesting ?
@@ -44,14 +43,6 @@ func loadFile(t *testing.T, path string) string {
 	return string(data)
 }
 
-// XXX -> xtesting ?
-// XXX dup in zodbdump_test.go
-func diff(a, b string) string {
-	dmp := diffmatchpatch.New()
-	diffv := dmp.DiffMain(a, b, /*checklines=*/false)
-	return dmp.DiffPrettyText(diffv)
-}
-
 func testDump(t *testing.T, dir fs1.IterDir, d Dumper) {
 	buf := bytes.Buffer{}
 
@@ -63,7 +54,7 @@ func testDump(t *testing.T, dir fs1.IterDir, d Dumper) {
 	dumpOk := loadFile(t, fmt.Sprintf("testdata/1.%s.ok", d.DumperName()))
 
 	if dumpOk != buf.String() {
-		t.Errorf("%s: dump different:\n%v", d.DumperName(), diff(dumpOk, buf.String()))
+		t.Errorf("%s: dump different:\n%v", d.DumperName(), diff.Diff(dumpOk, buf.String()))
 	}
 }
 

go/zodb/zodb.go

@@ -75,7 +75,7 @@ type DataInfo struct {
 	Tid	Tid
 	Data	[]byte	// nil means: deleted	XXX -> *Buf ?
 
-	// original tid data was committed (e.g. in case of undo)
+	// original tid data was committed at (e.g. in case of undo)
 	//
 	// FIXME we don't really need this and this unnecessarily constraints interfaces.
 	DataTid	Tid
@@ -133,8 +133,6 @@ func (e *ErrXidMissing) Error() string {
 
 // IStorage is the interface provided by ZODB storages
 type IStorage interface {
-	// TODO add invalidation channel
-
 	// StorageName returns storage name
 	StorageName() string
 
@@ -148,7 +146,7 @@ type IStorage interface {
 
 	// LastOid returns highest object id of objects committed to storage.
 	//
-	// if there is no data committed yet, LastOid returns Oid zero value	XXX -> better -1
+	// if there is no data committed yet, LastOid returns Oid zero value	XXX -> better -1 ?
 	// XXX ZODB/py does not define this in IStorage.
 	LastOid(ctx context.Context) (Oid, error)
 
@@ -166,6 +164,8 @@ type IStorage interface {
 
 	// Prefetch(ctx, xid Xid)	(no error)
 
+	// TODO add invalidation channel (notify about changes made to DB not by us)
+
 	// Store(oid Oid, serial Tid, data []byte, txn ITransaction) error
 	// XXX Restore ?
 	// CheckCurrentSerialInTransaction(oid Oid, serial Tid, txn ITransaction)   // XXX naming
@@ -191,7 +191,7 @@ type ITxnIterator interface {
 	// 2. iterator over transaction's data records. 
 	// transaction metadata stays valid until next call to NextTxn().
 	// end of iteration is indicated with io.EOF
-	NextTxn() (*TxnInfo, IDataIterator, error)	// XXX ctx
+	NextTxn(ctx context.Context) (*TxnInfo, IDataIterator, error)
 }
 
 // IDataIterator is the interface to iterate data records.
@@ -199,7 +199,7 @@ type IDataIterator interface {
 	// NextData yields information about next storage data record.
 	// returned data stays valid until next call to NextData().
 	// end of iteration is indicated with io.EOF
-	NextData() (*DataInfo, error)	// XXX ctx
+	NextData(ctx context.Context) (*DataInfo, error)
 }
 
 // ---- misc ----

go/zodb/zodbtools/dump.go

@@ -144,7 +144,7 @@ out:
 }
 
 // DumpTxn dumps one transaction record
-func (d *dumper) DumpTxn(txni *zodb.TxnInfo, dataIter zodb.IDataIterator) error {
+func (d *dumper) DumpTxn(ctx context.Context, txni *zodb.TxnInfo, dataIter zodb.IDataIterator) error {
 	var datai *zodb.DataInfo
 
 	// LF in-between txn records
@@ -162,7 +162,7 @@ func (d *dumper) DumpTxn(txni *zodb.TxnInfo, dataIter zodb.IDataIterator) error
 
 	// data records
 	for {
-		datai, err = dataIter.NextData()
+		datai, err = dataIter.NextData(ctx)
 		if err != nil {
 			if err == io.EOF {
 				err = nil	// XXX -> okEOF ?
@@ -189,7 +189,7 @@ out:
 }
 
 // Dump dumps transaction records in between tidMin..tidMax
-func (d *dumper) Dump(stor zodb.IStorage, tidMin, tidMax zodb.Tid) error {
+func (d *dumper) Dump(ctx context.Context, stor zodb.IStorage, tidMin, tidMax zodb.Tid) error {
 	var txni     *zodb.TxnInfo
 	var dataIter zodb.IDataIterator
 	var err      error
@@ -198,7 +198,7 @@ func (d *dumper) Dump(stor zodb.IStorage, tidMin, tidMax zodb.Tid) error {
 
 	// transactions
 	for {
-		txni, dataIter, err = iter.NextTxn()
+		txni, dataIter, err = iter.NextTxn(ctx)
 		if err != nil {
 			if err == io.EOF {
 				err = nil	// XXX -> okEOF ?
@@ -207,7 +207,7 @@ func (d *dumper) Dump(stor zodb.IStorage, tidMin, tidMax zodb.Tid) error {
 			break
 		}
 
-		err = d.DumpTxn(txni, dataIter)
+		err = d.DumpTxn(ctx, txni, dataIter)
 		if err != nil {
 			break
 		}
@@ -221,10 +221,11 @@ func (d *dumper) Dump(stor zodb.IStorage, tidMin, tidMax zodb.Tid) error {
 }
 
 // Dump dumps contents of a storage in between tidMin..tidMax range to a writer.
+//
 // see top-level documentation for the dump format.
-func Dump(w io.Writer, stor zodb.IStorage, tidMin, tidMax zodb.Tid, hashOnly bool) error {
+func Dump(ctx context.Context, w io.Writer, stor zodb.IStorage, tidMin, tidMax zodb.Tid, hashOnly bool) error {
 	d := dumper{W: w, HashOnly: hashOnly}
-	return d.Dump(stor, tidMin, tidMax)
+	return d.Dump(ctx, stor, tidMin, tidMax)
 }
 
 // ----------------------------------------
@@ -270,13 +271,15 @@ func dumpMain(argv []string) {
 		prog.Fatal(err)
 	}
 
-	stor, err := zodb.OpenStorage(context.Background(), storUrl, &zodb.OpenOptions{ReadOnly: true})
+	ctx := context.Background()
+
+	stor, err := zodb.OpenStorage(ctx, storUrl, &zodb.OpenOptions{ReadOnly: true})
 	if err != nil {
 		prog.Fatal(err)
 	}
 	// TODO defer stor.Close()
 
-	err = Dump(os.Stdout, stor, tidMin, tidMax, hashOnly)
+	err = Dump(ctx, os.Stdout, stor, tidMin, tidMax, hashOnly)
 	if err != nil {
 		prog.Fatal(err)
 	}

go/zodb/zodbtools/dump_test.go

@@ -28,22 +28,13 @@ import (
 	"regexp"
 	"testing"
 
-	"lab.nexedi.com/kirr/neo/go/zodb/storage/fs1"
 	"lab.nexedi.com/kirr/neo/go/zodb"
+	_ "lab.nexedi.com/kirr/neo/go/zodb/wks"
 
-	"github.com/sergi/go-diff/diffmatchpatch"
+	"github.com/kylelemons/godebug/diff"
 	"lab.nexedi.com/kirr/go123/exc"
 )
 
-// diff computes difference for two strings a and b
-// XXX -> xtesting ?
-// XXX dup in fstail_test.go
-func diff(a, b string) string {
-	dmp := diffmatchpatch.New()
-	diffv := dmp.DiffMain(a, b, /*checklines=*/false)
-	return dmp.DiffPrettyText(diffv)
-}
-
 // loadZdumpPy loads a zdump file and normalizes escaped strings to the way go
 // would escape them.
 func loadZdumpPy(t *testing.T, path string) string {
@@ -73,23 +64,22 @@ func loadZdumpPy(t *testing.T, path string) string {
 	return string(dump)
 }
 
-func withTestdata1Fs(t testing.TB, f func(fs *fs1.FileStorage)) {
-	// XXX -> zodb.OpenURL
-	fs, err := fs1.Open(context.Background(), "../../zodb/storage/fs1/testdata/1.fs")	// XXX read-only, path?
+func withTestdata1Fs(t testing.TB, f func(zstor zodb.IStorage)) {
+	zstor, err := zodb.OpenStorage(context.Background(), "../../zodb/storage/fs1/testdata/1.fs", &zodb.OpenOptions{ReadOnly: true})
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	defer exc.XRun(fs.Close)
+	defer exc.XRun(zstor.Close)
 
-	f(fs)
+	f(zstor)
 }
 
 func TestZodbDump(t *testing.T) {
-	withTestdata1Fs(t, func(fs *fs1.FileStorage) {
+	withTestdata1Fs(t, func(zstor zodb.IStorage) {
 		buf := bytes.Buffer{}
 
-		err := Dump(&buf, fs, 0, zodb.TidMax, false)
+		err := Dump(context.Background(), &buf, zstor, 0, zodb.TidMax, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -97,7 +87,7 @@ func TestZodbDump(t *testing.T) {
 		dumpOk := loadZdumpPy(t, "testdata/1.zdump.pyok")
 
 		if dumpOk != buf.String() {
-			t.Errorf("dump different:\n%v", diff(dumpOk, buf.String()))
+			t.Errorf("dump different:\n%v", diff.Diff(dumpOk, buf.String()))
 		}
 	})
 }
@@ -105,11 +95,11 @@ func TestZodbDump(t *testing.T) {
 
 func BenchmarkZodbDump(b *testing.B) {
 	// FIXME small testdata/1.fs is not representative for benchmarking
-	withTestdata1Fs(b, func(fs *fs1.FileStorage) {
+	withTestdata1Fs(b, func(zstor zodb.IStorage) {
 		b.ResetTimer()
 
 		for i := 0; i < b.N; i++ {
-			err := Dump(ioutil.Discard, fs, 0, zodb.TidMax, false)
+			err := Dump(context.Background(), ioutil.Discard, zstor, 0, zodb.TidMax, false)
 			if err != nil {
 				b.Fatal(err)
 			}

go/zodb/zodbtools/help.go

@@ -51,19 +51,20 @@ http://docs.pylonsproject.org/projects/zodburi/
 `
 
 const helpXid =
-`TODO describe
+`An object address for loading from ZODB should be specified as follows:
 
-	=tid:oid
-	<tid:oid
-`
+	- "=" or "<" character depending on whether it is exact or "tid before" query
+	- tid
+	- ":"
+	- oid
+
+for example
 
-// TODO dump format
-const helpZDump =
-`TODO describe zodb dump format
+	=0285cbac258bf266:0000000000000001	- oid 1 at exactly 0285cbac258bf266 transaction
+	<0285cbac258bf266:0000000000000001	- oid 1 at first newest transaction changing it with tid < 0285cbac258bf266
 `
 
 var helpTopics = prog.HelpRegistry{
-	{"zurl",  "specifying database URL", helpZURL},
-	{"xid",   "specifying object address", helpXid},
-	{"zdump", "description of ZODB dump format", helpZDump},
+	{"zurl",  "specifying database URL",	helpZURL},
+	{"xid",   "specifying object address",	helpXid},
 }