.

t/neo/storage/filestorage.go

-// XXX license
-
-// filestorage support  XXX text
-package storage
-
-import (
-	"os"
-	"../zodb"
-)
-
-type FileStorage struct {
-    f *os.File	// XXX naming -> file ?
-}
-
-// IStorage
-var _ zodb.IStorage = (*FileStorage)(nil)
-
-type TxnRecHead struct {
-    Tid             zodb.Tid
-    RecLenm8        uint64
-    Status          zodb.TxnStatus
-    //UserLen         uint16
-    //DescriptionLen  uint16
-    //ExtensionLen    uint16
-    User            []byte  // TODO Encode ^^^
-    Description     []byte
-    Extension       []byte
-
-    Datav   []DataRec
-}
-
-type DataRec struct {
-    Oid             zodb.Oid
-    Tid             zodb.Tid
-    PrevDataRecPos  uint64  // previous-record file-position
-    TxnPos          uint64  // beginning of transaction record file position
-    // 2-bytes with zero values. (Was version length.)
-    //DataLen     uint64
-    Data            []byte
-    DataRecPos      uint64  // if Data == nil -> byte position of data record containing data
-}
-
-
-func (rh *TxnRecHead) MarshalFS() []byte {
-    panic("TODO")
-}
-
-func (rh *TxnRecHead) UnmarshalFS(data []byte) {
-    //TODO
-}
-
-
-
-func NewFileStorage(path string) (*FileStorage, error) {
-    f, err := os.Open(path)	// note opens in O_RDONLY
-    if err != nil {
-        return nil, err
-    }
-    // TODO read file header
-    //Read(f, 4) != "FS21" -> invalid header
-    return &FileStorage{f: f}, nil
-}
-
-func (f *FileStorage) Close() error {
-    err := f.f.Close()
-    if err != nil {
-        return err
-    }
-    f.f = nil
-    return nil
-}
-
-func (f *FileStorage) Iterate(start, stop zodb.Tid) zodb.IStorageIterator {
-    if start != zodb.Tid0 || stop != zodb.TidMax {
-        panic("TODO start/stop support")
-    }
-
-    // TODO
-    return nil
-}

t/neo/storage/fs1/filestorage.go

+// Copyright (C) 2017  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 2, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+//
+// XXX based on code from ZODB ?
+
+// FileStorage v1.  XXX text
+package fs1
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+
+	"../../zodb"
+)
+
+type FileStorage struct {
+	f		*os.File	// XXX naming -> file ?
+	index	*fsIndex
+}
+
+// IStorage
+var _ zodb.IStorage = (*FileStorage)(nil)
+
+type TxnRecHead struct {
+	Tid             zodb.Tid
+	RecLenm8        uint64
+	Status          zodb.TxnStatus
+	//UserLen         uint16
+	//DescriptionLen  uint16
+	//ExtensionLen    uint16
+	User            []byte  // TODO Encode ^^^
+	Description     []byte
+	Extension       []byte
+
+	//Datav   []DataRec
+}
+
+// DataHeader represents header of a data record
+type DataHeader struct {
+	Oid             zodb.Oid
+	Tid             zodb.Tid
+	PrevDataRecPos  uint64  // previous-record file-position
+	TxnPos          uint64  // position of transaction record this data record belongs to
+	_		uint16	// 2-bytes with zero values. (Was version length.)
+	DataLen		uint64	// length of following data. if 0 -> following = 8 bytes backpointer
+				// if backpointer == 0 -> oid deleted
+	//Data            []byte
+	//DataRecPos      uint64  // if Data == nil -> byte position of data record containing data
+}
+
+const DataHeaderSize	= 42
+
+// ErrDataRead is returned on data record read / decode errors
+type ErrDataRead struct {
+	Pos	int64
+	Err	error
+}
+
+func (e *ErrDataRead) Error() string {
+	return fmt.Sprintf("data read: record @%v: %v", e.Pos, e.Err)
+}
+
+// XXX -> zodb?
+var ErrVersionNonZero = errors.New("non-zero version")
+
+
+
+// XXX io.ReaderAt -> *os.File  (if iface conv costly)
+func (dh *DataHeader) decode(r io.ReaderAt, pos int64, tmpBuf *[DataHeaderSize]byte) error {
+	n, err := r.ReadAt(tmpBuf[:], pos)
+	if n == DataHeaderSize {
+		err = nil	// we don't care if it was EOF after full header read	XXX ok?
+	}
+
+	if err != nil {
+		return &ErrDataRead{pos, err}
+	}
+
+	dh.Oid.Decode(tmpBuf[0:])
+	dh.Tid.Decode(tmpBuf[8:])
+	dh.PrevDataRecPos = binary.BigEndian.Uint64(tmpBuf[16:])
+	dh.TxnPos = binary.BigEndian.Uint64(tmpBuf[24:])
+	verlen := binary.BigEndian.Uint16(tmpBuf[32:])
+	dh.DataLen = binary.BigEndian.Uint64(tmpBuf[34:])
+
+	if verlen != 0 {
+		return &ErrDataRead{pos, ErrVersionNonZero}
+	}
+
+	return nil
+}
+
+func (dh *DataHeader) Decode(r io.ReaderAt, pos int64) error {
+	var tmpBuf [DataHeaderSize]byte
+	return dh.decode(r, pos, &tmpBuf)
+}
+
+
+
+func NewFileStorage(path string) (*FileStorage, error) {
+	f, err := os.Open(path)	// XXX opens in O_RDONLY
+	if err != nil {
+		return nil, err	// XXX err more context ?
+	}
+	// TODO read file header
+	//Read(f, 4) != "FS21" -> invalid header
+	return &FileStorage{f: f}, nil
+
+	// TODO read/recreate index
+}
+
+func (fs *FileStorage) LoadBefore(oid zodb.Oid, beforeTid zodb.Tid) (data []byte, tid zodb.Tid, err error) {
+	// lookup in index position of oid data record with latest transaction who changed this oid
+	dataPos, ok := fs.index.Get(oid)
+	if !ok {
+		return nil, zodb.Tid(0), zodb.ErrOidMissing{Oid: oid}
+	}
+
+	// search backwards for when we first have data record with tid < beforeTid
+}
+
+func (fs *FileStorage) Close() error {
+	// TODO dump index
+	err := fs.f.Close()
+	if err != nil {
+		return err
+	}
+	fs.f = nil
+	return nil
+}
+
+func (fs *FileStorage) StorageName() string {
+	return "FileStorage v1"
+}
+
+func (fs *FileStorage) Iterate(start, stop zodb.Tid) zodb.IStorageIterator {
+	if start != zodb.Tid0 || stop != zodb.TidMax {
+		panic("TODO start/stop support")
+	}
+
+	// TODO
+	return nil
+}

t/neo/storage/fs1/index.go

@@ -12,7 +12,7 @@
 //
 // XXX based on code from ZODB ?
 
-// FileStorage. Index
+// FileStorage v1. Index
 package fs1
 
 import (

t/neo/storage/fs1/py/gen-testdata

 #!/usr/bin/env python2
+# TODO author/copyright
 """generate reference database and index for tests"""
 
 from ZODB.FileStorage import FileStorage

t/neo/storage/fs1/py/indexcmp

 #!/usr/bin/env python2
+# TODO author/copyright
 """compare two index files"""
 
 from ZODB.fsIndex import fsIndex

t/neo/zodb/zodb.go

 // TODO copyright / license
 
-// Package zodb defines types and interfaces used in ZODB databases
+// Package zodb defines types, interfaces and errors used in ZODB databases
 
 // XXX partly based on ZODB/py
 
 package zodb
 
+import (
+	"fmt"
+)
+
 // ZODB types
 type Tid uint64  // transaction identifier
 type Oid uint64  // object identifier
@@ -13,8 +17,8 @@ type Oid uint64  // object identifier
 /*
 // XXX "extended" oid - oid + serial, completely specifying object revision
 type Xid struct {
-    Tid
-    Oid
+	Tid
+	Oid
 }
 */
 
@@ -27,6 +31,26 @@ const (
 	Oid0	Oid = 0
 )
 
+func (tid Tid) String() string {
+	// XXX also print "tid:" prefix ?
+	return fmt.Sprintf("%016x", uint64(tid))
+}
+
+func (oid Oid) String() string {
+	// XXX also print "oid:" prefix ?
+	return fmt.Sprintf("%016x", uint64(oid))
+}
+
+// ----------------------------------------
+
+type ErrOidMissing struct {
+	Oid	Oid
+}
+
+func (e ErrOidMissing) Error() string {
+	return "%v: no such oid"
+}
+
 // ----------------------------------------
 
 // TxnStatus represents status of a transaction
@@ -44,61 +68,62 @@ const (
 // XXX -> storage.ITransactionInformation
 //type IStorageTransactionInformation interface {
 type StorageTransactionInformation struct {
-    Tid         Tid
-    Status      TxnStatus
-    User        []byte
-    Description []byte
-    Extension   []byte
-
-    // TODO iter -> IStorageRecordInformation
-    Iter	IStorageRecordIterator
+	Tid         Tid
+	Status      TxnStatus
+	User        []byte
+	Description []byte
+	Extension   []byte
+
+	// TODO iter -> IStorageRecordInformation
+	Iter	IStorageRecordIterator
 }
 
 // Information about single storage record
 type StorageRecordInformation struct {
-    Oid         Oid
-    Tid         Tid
-    Data        []byte
-    // XXX .version ?
-    // XXX .data_txn    (The previous transaction id)
+	Oid         Oid
+	Tid         Tid
+	Data        []byte
+	// XXX .version ?
+	// XXX .data_txn    (The previous transaction id)
 }
 
 
 
 type IStorage interface {
-    Close() error
+	Close() error
 
-    // Name returns storage name
-    Name() string
+	// StorageName returns storage name
+	StorageName() string
 
-    // History(oid, size=1)
+	// History(oid, size=1)
 
-    // LastTid returns the id of the last committed transaction.
-    // if not transactions have been committed yet, LastTid returns Tid zero value
-    // XXX ^^^ ok ?
-    LastTid() Tid	// XXX -> Tid, ok ?
+	// LastTid returns the id of the last committed transaction.
+	// if not transactions have been committed yet, LastTid returns Tid zero value
+	// XXX ^^^ ok ?
+	LastTid() Tid	// XXX -> Tid, ok ?
 
-    LoadBefore(oid Oid, beforeTid Tid) (data []byte, tid Tid, err error)
-    LoadSerial(oid Oid, serial Tid)    (data []byte, err error)
-    // PrefetchBefore(oidv []Oid, beforeTid Tid) error (?)
+	// TODO data []byte -> something allocated from slab ?
+	LoadBefore(oid Oid, beforeTid Tid) (data []byte, tid Tid, err error)
+	LoadSerial(oid Oid, serial Tid)    (data []byte, err error)
+	// PrefetchBefore(oidv []Oid, beforeTid Tid) error (?)
 
-    // Store(oid Oid, serial Tid, data []byte, txn ITransaction) error
-    // XXX Restore ?
-    // CheckCurrentSerialInTransaction(oid Oid, serial Tid, txn ITransaction)   // XXX naming
+	// Store(oid Oid, serial Tid, data []byte, txn ITransaction) error
+	// XXX Restore ?
+	// CheckCurrentSerialInTransaction(oid Oid, serial Tid, txn ITransaction)   // XXX naming
 
-    // tpc_begin(txn)
-    // tpc_vote(txn)
-    // tpc_finish(txn, callback)    XXX clarify about callback
-    // tpc_abort(txn)
+	// tpc_begin(txn)
+	// tpc_vote(txn)
+	// tpc_finish(txn, callback)    XXX clarify about callback
+	// tpc_abort(txn)
 
-    Iterate(start, stop Tid) IStorageIterator   // XXX -> Iter() ?
+	Iterate(start, stop Tid) IStorageIterator   // XXX -> Iter() ?
 }
 
 type IStorageIterator interface {
-    Next() (*StorageTransactionInformation, error)  // XXX -> NextTxn() ?
+	Next() (*StorageTransactionInformation, error)  // XXX -> NextTxn() ?
 }
 
 type IStorageRecordIterator interface {         // XXX naming -> IRecordIterator
-    Next() (*StorageRecordInformation, error)    // XXX vs ptr & nil ?
-                                                // XXX -> NextTxnObject() ?
+	Next() (*StorageRecordInformation, error)    // XXX vs ptr & nil ?
+	                                            // XXX -> NextTxnObject() ?
 }