.

t/neo/storage/fs1/fsb/fsbtree.go

 // DO NOT EDIT - AUTOGENERATED (by gen-fsbtree from github.com/cznic/b bcff30a)
-// KEY=zodb.Oid  VALUE=zodb.Tid
+// KEY=zodb.Oid  VALUE=int64
 // ---- 8< ----
 
 // Copyright 2014 The b Authors. All rights reserved.
@@ -72,7 +72,7 @@ type (
 
 	de struct { // d element
 		k zodb.Oid
-		v zodb.Tid
+		v int64
 	}
 
 	// Enumerator captures the state of enumerating a tree. It is returned
@@ -338,7 +338,7 @@ func (t *Tree) Delete(k zodb.Oid) (ok bool) {
 	}
 }
 
-func (t *Tree) extract(q *d, i int) { // (r zodb.Tid) {
+func (t *Tree) extract(q *d, i int) { // (r int64) {
 	t.ver++
 	//r = q.d[i].v // prepared for Extract
 	q.c--
@@ -388,7 +388,7 @@ func (t *Tree) find(q interface{}, k zodb.Oid) (i int, ok bool) {
 
 // First returns the first item of the tree in the key collating order, or
 // (zero-value, zero-value) if the tree is empty.
-func (t *Tree) First() (k zodb.Oid, v zodb.Tid) {
+func (t *Tree) First() (k zodb.Oid, v int64) {
 	if q := t.first; q != nil {
 		q := &q.d[0]
 		k, v = q.k, q.v
@@ -398,7 +398,7 @@ func (t *Tree) First() (k zodb.Oid, v zodb.Tid) {
 
 // Get returns the value associated with k and true if it exists. Otherwise Get
 // returns (zero-value, false).
-func (t *Tree) Get(k zodb.Oid) (v zodb.Tid, ok bool) {
+func (t *Tree) Get(k zodb.Oid) (v int64, ok bool) {
 	q := t.r
 	if q == nil {
 		return
@@ -424,7 +424,7 @@ func (t *Tree) Get(k zodb.Oid) (v zodb.Tid, ok bool) {
 	}
 }
 
-func (t *Tree) insert(q *d, i int, k zodb.Oid, v zodb.Tid) *d {
+func (t *Tree) insert(q *d, i int, k zodb.Oid, v int64) *d {
 	t.ver++
 	c := q.c
 	if i < c {
@@ -439,7 +439,7 @@ func (t *Tree) insert(q *d, i int, k zodb.Oid, v zodb.Tid) *d {
 
 // Last returns the last item of the tree in the key collating order, or
 // (zero-value, zero-value) if the tree is empty.
-func (t *Tree) Last() (k zodb.Oid, v zodb.Tid) {
+func (t *Tree) Last() (k zodb.Oid, v int64) {
 	if q := t.last; q != nil {
 		q := &q.d[q.c-1]
 		k, v = q.k, q.v
@@ -452,7 +452,7 @@ func (t *Tree) Len() int {
 	return t.c
 }
 
-func (t *Tree) overflow(p *x, q *d, pi, i int, k zodb.Oid, v zodb.Tid) {
+func (t *Tree) overflow(p *x, q *d, pi, i int, k zodb.Oid, v int64) {
 	t.ver++
 	l, r := p.siblings(pi)
 
@@ -533,7 +533,7 @@ func (t *Tree) SeekLast() (e *Enumerator, err error) {
 }
 
 // Set sets the value associated with k.
-func (t *Tree) Set(k zodb.Oid, v zodb.Tid) {
+func (t *Tree) Set(k zodb.Oid, v int64) {
 	//dbg("--- PRE Set(%v, %v)\n%s", k, v, t.dump())
 	//defer func() {
 	//	dbg("--- POST\n%s\n====\n", t.dump())
@@ -598,11 +598,11 @@ func (t *Tree) Set(k zodb.Oid, v zodb.Tid) {
 // 	tree.Put(k, func(zodb.Oid, bool){ return v, true })
 //
 // modulo the differing return values.
-func (t *Tree) Put(k zodb.Oid, upd func(oldV zodb.Tid, exists bool) (newV zodb.Tid, write bool)) (oldV zodb.Tid, written bool) {
+func (t *Tree) Put(k zodb.Oid, upd func(oldV int64, exists bool) (newV int64, write bool)) (oldV int64, written bool) {
 	pi := -1
 	var p *x
 	q := t.r
-	var newV zodb.Tid
+	var newV int64
 	if q == nil {
 		// new KV pair in empty tree
 		newV, written = upd(newV, false)
@@ -664,7 +664,7 @@ func (t *Tree) Put(k zodb.Oid, upd func(oldV zodb.Tid, exists bool) (newV zodb.T
 	}
 }
 
-func (t *Tree) split(p *x, q *d, pi, i int, k zodb.Oid, v zodb.Tid) {
+func (t *Tree) split(p *x, q *d, pi, i int, k zodb.Oid, v int64) {
 	t.ver++
 	r := btDPool.Get().(*d)
 	if q.n != nil {
@@ -826,7 +826,7 @@ func (e *Enumerator) Close() {
 // Next returns the currently enumerated item, if it exists and moves to the
 // next item in the key collation order. If there is no item to return, err ==
 // io.EOF is returned.
-func (e *Enumerator) Next() (k zodb.Oid, v zodb.Tid, err error) {
+func (e *Enumerator) Next() (k zodb.Oid, v int64, err error) {
 	if err = e.err; err != nil {
 		return
 	}
@@ -880,7 +880,7 @@ func (e *Enumerator) next() error {
 // Prev returns the currently enumerated item, if it exists and moves to the
 // previous item in the key collation order. If there is no item to return, err
 // == io.EOF is returned.
-func (e *Enumerator) Prev() (k zodb.Oid, v zodb.Tid, err error) {
+func (e *Enumerator) Prev() (k zodb.Oid, v int64, err error) {
 	if err = e.err; err != nil {
 		return
 	}

t/neo/storage/fs1/fsb/gen-fsbtree

 #!/bin/sh -e
-# generate b.Tree with compile-time KEY=zodb.Oid, VALUE=zodb.Tid
+# generate b.Tree with compile-time KEY=zodb.Oid, VALUE=int64
 
 KEY=zodb.Oid
-VALUE=zodb.Tid
+VALUE=int64
 
 b=github.com/cznic/b
 Bdir=`go list -f '{{.Dir}}' $b`

t/neo/storage/fs1/index.go

@@ -36,25 +36,15 @@ type fsIndex struct {
 }
 
 
-// TODO Encode		(__getstate__)
-// TODO Decode		(__setstate__)
-// TODO ? save(pos, fname)
-// TODO ? klass.load(fname)
-
-/*
-// __getitem__
-func (fsi *fsIndex) Get(oid zodb.Oid) zodb.Tid {	// XXX ,ok ?
-	tid, ok := fsi.oid2tid.Get(oid)
-	return tid, ok
-}
-*/
+// TODO Encode		(__getstate__)	?
+// TODO Decode		(__setstate__)	?
 
-// __setitem__ -> Set
+// NOTE Get/Set/... are taken as-is from fsb.Tree
 
 // on-disk index format
 // (changed in 2010 in https://github.com/zopefoundation/ZODB/commit/1bb14faf)
 //
-// pos     (?) (int or long - INT_TYPES)
+// topPos     position pointing just past the last committed transaction
 // (oid[:6], v.toString)   # ._data[oid]-> v (fsBucket().toString())
 // (oid[:6], v.toString)
 // ...
@@ -62,51 +52,61 @@ func (fsi *fsIndex) Get(oid zodb.Oid) zodb.Tid {	// XXX ,ok ?
 //
 //
 // fsBucket.toString():
-// oid[6:8]oid[6:8]oid[6:8]...tid[0:6]tid[0:6]tid[0:6]...
+// oid[6:8]oid[6:8]oid[6:8]...pos[0:6]pos[0:6]pos[0:6]...
 
 
 const oidPrefixMask zodb.Oid = ((1<<64-1) ^ (1<<16 -1))	// 0xffffffffffff0000
 
+// IndexIOError is the error type returned by index save and load routines
+type IndexIOError struct {
+	Op  string	// operation performed - "save" or "load"
+	Err error	// error that occured during the operation
+}
+
+func (e *IndexIOError) Error() string {
+	s := "index " + e.Op + ": " + e.Err.Error()
+	return s
+}
+
 // Save saves the index to a writer
-// XXX proper error context
-func (fsi *fsIndex) Save(fsPos int64, w io.Writer) error {
+func (fsi *fsIndex) Save(topPos int64, w io.Writer) error {
 	p := pickle.NewEncoder(w)
 
-	err := p.Encode(fsPos)
+	err := p.Encode(topPos)
 	if err != nil {
-		return err
+		goto out
 	}
 
 	var oidb [8]byte
-	var tidb [8]byte
+	var posb [8]byte
 	var oidPrefixCur zodb.Oid	// current oid[0:6] with [6:8] = 00
 	oidBuf := []byte{}		// current oid[6:8]oid[6:8]...
-	tidBuf := []byte{}		// current tid[0:6]tid[0:6]...
+	posBuf := []byte{}		// current pos[0:6]pos[0:6]...
 	var t [2]interface{}		// tuple for (oid, fsBucket.toString())
 
 
 	e, err := fsi.SeekFirst()
-	if err == io.EOF {
-		goto skip	// empty btree
+	if err == io.EOF {	// always only io.EOF indicating an empty btree
+		goto skip
 	}
 
 	for {
-		oid, tid, errStop := e.Next()
+		oid, pos, errStop := e.Next()
 		oidPrefix := oid & oidPrefixMask
 
 		if oidPrefix != oidPrefixCur {
-			// emit pickle for current oid06
+			// emit (oid[:6], oid[6:8]oid[6:8]...pos[0:6]pos[0:6]...)
 			binary.BigEndian.PutUint64(oidb[:], uint64(oid))
 			t[0] = oidb[0:6]
-			t[1] = bytes.Join([][]byte{oidBuf, tidBuf}, nil)
+			t[1] = bytes.Join([][]byte{oidBuf, posBuf}, nil)
 			err = p.Encode(t)
 			if err != nil {
-				return err
+				goto out
 			}
 
 			oidPrefixCur = oidPrefix
 			oidBuf = oidBuf[:0]
-			tidBuf = tidBuf[:0]
+			posBuf = posBuf[:0]
 		}
 
 		if errStop != nil {
@@ -114,21 +114,34 @@ func (fsi *fsIndex) Save(fsPos int64, w io.Writer) error {
 		}
 
 		binary.BigEndian.PutUint64(oidb[:], uint64(oid))
-		binary.BigEndian.PutUint64(tidb[:], uint64(tid))
+		binary.BigEndian.PutUint64(posb[:], uint64(pos))
 
+		// XXX check pos does not overflow 6 bytes
 		oidBuf = append(oidBuf, oidb[6:8]...)
-		tidBuf = append(tidBuf, tidb[0:6]...)
+		posBuf = append(posBuf, posb[0:6]...)
 	}
 
 	e.Close()
 
 skip:
 	err = p.Encode(pickle.None{})
+
+out:
+	if err == nil {
 		return err
+	}
+
+	if _, ok := err.(pikle.TypeError); ok {
+		panic(err)	// all our types are expected to be supported by pickle
+	}
+
+	// otherwise it is an error returned by writer, which should already
+	// have filename & op as context.
+	return &IndexIOError{"save", err}
 }
 
 // LoadIndex loads index from a reader
-func LoadIndex(r io.Reader) (fsPos int64, fsi *fsIndex, err error) {
+func LoadIndex(r io.Reader) (topPos int64, fsi *fsIndex, err error) {
 	p := pickle.NewDecoder(r)
 
 	// if we can know file position we can show it in error context
@@ -140,18 +153,18 @@ func LoadIndex(r io.Reader) (fsPos int64, fsi *fsIndex, err error) {
 		}
 	}
 
-	xpos, err := p.Decode()
+	xtopPos, err := p.Decode()
 	if err != nil {
 		// TODO err
 	}
-	fsPos, ok := xpos.(int64)
+	topPos, ok := xtopPos.(int64)
 	if !ok {
 		// TODO err
 	}
 
 	fsi = &fsIndex{}	// TODO cmpFunc ...
 	var oidb [8]byte
-	var tidb [8]byte
+	var posb [8]byte
 
 loop:
 	for {
@@ -194,22 +207,61 @@ loop:
 
 		n := len(kvBuf) / 8
 		oidBuf := kvBuf[:n*2]
-		tidBuf := kvBuf[n*2:]
+		posBuf := kvBuf[n*2:]
 
 		for i:=0; i<n; i++ {
 			oid := zodb.Oid(binary.BigEndian.Uint16(oidBuf[i*2:]))
 			oid |= oidPrefix
-			copy(tidb[2:], tidBuf[i*6:])
-			tid := zodb.Tid(binary.BigEndian.Uint64(tidb[:]))
+			copy(posb[2:], posBuf[i*6:])
+			tid := zodb.Tid(binary.BigEndian.Uint64(posb[:]))
 
 			fsi.Set(oid, tid)
 		}
 	}
 
-	return fsPos, fsi, nil
+	return topPos, fsi, nil
+
+out:
+	if err == nil {
+		return topPos, fsi, err
+	}
+
+	rname := IOName(r)
 
-xerror:
 	// same for file name
 	rname, _ := r.(interface{ Name() string })
 
 }
+
+
+// IOName returns a "filename" associated with io.Reader, io.Writer, net.Conn, ...
+// if name cannot be deterined - "" is returned.
+func IOName(f interface {}) string {
+	switch f := f.(type) {
+	case *os.File:
+		// XXX better via interface { Name() string } ?
+		//     but Name() is too broad compared to FileName()
+		return f.Name()
+
+	case net.Conn:
+		// XXX not including LocalAddr is ok?
+		return f.RemoteAddr.String()
+
+	case *io.LimitedReader:
+		return IOName(f.R)
+
+
+	case *io.PipeReader:
+		fallthrough
+	case *io.PipeWriter:
+		return "pipe"
+
+	// XXX SectionReader MultiReader TeeReader
+
+	// bufio.Reader bufio.Writer bufio.Scanner
+
+
+	default:
+		return ""
+	}
+}