Commit f53dc308 authored by Kirill Smelkov's avatar Kirill Smelkov

.

parent d1054012
...@@ -60,6 +60,10 @@ func NewSeqReaderAtSize(r io.ReaderAt, size int) *SeqReaderAt { ...@@ -60,6 +60,10 @@ func NewSeqReaderAtSize(r io.ReaderAt, size int) *SeqReaderAt {
return sb return sb
} }
// XXX not yet sure it is good idea
func (sb *SeqReaderAt) IOUnwrap() interface{} {
return sb.r
}
// // XXX temp // // XXX temp
// func init() { // func init() {
......
...@@ -48,6 +48,11 @@ func (l *LimitedWriter) Write(p []byte) (n int, err error) { ...@@ -48,6 +48,11 @@ func (l *LimitedWriter) Write(p []byte) (n int, err error) {
func LimitWriter(w io.Writer, n int64) io.Writer { return &LimitedWriter{w, n} } func LimitWriter(w io.Writer, n int64) io.Writer { return &LimitedWriter{w, n} }
// XXX not sure it is a good ide
// XXX another option is IOName() - but that requires leaf packages to import xio
type Wrapper interface {
IOUnwrap() interface{}
}
// Name returns a "filename" associated with io.Reader, io.Writer, net.Conn, ... // Name returns a "filename" associated with io.Reader, io.Writer, net.Conn, ...
func Name(f interface {}) string { func Name(f interface {}) string {
...@@ -70,6 +75,8 @@ func Name(f interface {}) string { ...@@ -70,6 +75,8 @@ func Name(f interface {}) string {
// XXX SectionReader MultiReader TeeReader // XXX SectionReader MultiReader TeeReader
// XXX bufio.Reader bufio.Writer bufio.Scanner // XXX bufio.Reader bufio.Writer bufio.Scanner
case Wrapper: return Name(f.IOUnwrap())
// if name cannot be determined - let's provide full info // if name cannot be determined - let's provide full info
default: default:
return fmt.Sprintf("%#v", f) return fmt.Sprintf("%#v", f)
......
...@@ -31,9 +31,9 @@ import ( ...@@ -31,9 +31,9 @@ import (
) )
// Reindex rebuilds index for FileStorage file @ path // Reindex rebuilds index for FileStorage file @ path
func Reindex(path string) error { func Reindex(ctx context.Context, path string) error {
// XXX lock path.lock ? // XXX lock path.lock ?
index, err := fs1.BuildIndexForFile(context.Background(), path) index, err := fs1.BuildIndexForFile(ctx, path)
if err != nil { if err != nil {
return err return err
} }
...@@ -84,21 +84,47 @@ func reindexMain(argv []string) { ...@@ -84,21 +84,47 @@ func reindexMain(argv []string) {
// TODO verify-index -quick (only small sanity check) // TODO verify-index -quick (only small sanity check)
// VerifyIndexFor verifies that on-disk index for FileStorage file @ path is correct // VerifyIndexFor verifies that on-disk index for FileStorage file @ path is correct
func VerifyIndexFor(path string) error { func VerifyIndexFor(ctx context.Context, path string) (err error) {
panic("TODO") // XXX lock path.lock ?
/* inverify := false
// XXX open read-only defer func() {
fs, err := fs1.Open(context.Background(), path) // XXX , 0) if inverify {
return // index.Verify provides all context in error
}
xerr.Contextf(&err, "%s: verify index", path)
}
f, err := os.Open(path)
if err != nil {
return err
}
defer func() {
err2 := f.Close() // XXX vs inverify
err = xerr.First(err, err2)
}()
index, err := LoadIndexFile(path + ".index")
if err != nil { if err != nil {
return nil // XXX err ctx return err
}
fi, err := f.Stat()
if err != nil {
return err
}
topPos := fi.Size() // XXX there might be last TxnInprogress transaction
if index.TopPos != topPos {
return fmt.Errorf("topPos mismatch: data=%v index=%v", topPos, index.TopPos)
} }
defer fs.Close() // XXX err
err = fs.VerifyIndex(nil) // XXX - better somehow not here?
fSeq := xbufio.NewSeqReaderAt(f)
inverify = true
err = index.Verify(ctx, fSeq)
return err return err
//fs.Index()
//fs.ComputeIndex
*/
} }
const verifyIdxSummary = "verify database index" const verifyIdxSummary = "verify database index"
......
...@@ -555,7 +555,7 @@ func indexCorrupt(r io.ReaderAt, format string, argv ...interface{}) *IndexCorru ...@@ -555,7 +555,7 @@ func indexCorrupt(r io.ReaderAt, format string, argv ...interface{}) *IndexCorru
// VerifyTail checks index correctness against several newest transactions of FileStorage data in r. // VerifyTail checks index correctness against several newest transactions of FileStorage data in r.
// //
// For ntxn transactions starting from index.TopPos backwards, it verifies // For (XXX max) ntxn transactions starting from index.TopPos backwards, it verifies
// whether oid there have correct entries in the index. // whether oid there have correct entries in the index.
// //
// ntxn=-1 means data range to verify is till start of the file. // ntxn=-1 means data range to verify is till start of the file.
...@@ -585,7 +585,9 @@ func (index *Index) VerifyTail(ctx context.Context, r io.ReaderAt, ntxn int) (oi ...@@ -585,7 +585,9 @@ func (index *Index) VerifyTail(ctx context.Context, r io.ReaderAt, ntxn int) (oi
err := it.NextTxn(LoadNoStrings) err := it.NextTxn(LoadNoStrings)
if err != nil { if err != nil {
err = okEOF(err) if err == io.EOF {
break
}
return nil, err // XXX err ctx return nil, err // XXX err ctx
} }
...@@ -618,10 +620,11 @@ func (index *Index) VerifyTail(ctx context.Context, r io.ReaderAt, ntxn int) (oi ...@@ -618,10 +620,11 @@ func (index *Index) VerifyTail(ctx context.Context, r io.ReaderAt, ntxn int) (oi
} }
} }
// TODO err = EOF -> merge from Verify
return oidChecked, nil return oidChecked, nil
} }
// Verify checks index correctness against FileStorage data in r. // Verify checks index correctness against FileStorage data in r.
// //
// it verifies whether index is exactly the same as if it was build anew for // it verifies whether index is exactly the same as if it was build anew for
...@@ -634,6 +637,8 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt) error { ...@@ -634,6 +637,8 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt) error {
return err return err
} }
// XXX merge this into VerifyTail
// all oids from data were checked to be in index // all oids from data were checked to be in index
// now verify that there is no extra oids in index // now verify that there is no extra oids in index
if len(oidChecked) == index.Len() { if len(oidChecked) == index.Len() {
...@@ -656,3 +661,48 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt) error { ...@@ -656,3 +661,48 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt) error {
return nil return nil
} }
// XXX text
func (index *Index) VerifyTailForFile(ctx context.Context, path string, ntxn int) (oidChecked map[zodb.Oid]struct{}, err error) {
err = index.verifyForFile(path, func(r io.ReaderAt) error {
oidChecked, err = index.VerifyTail(ctx, r, ntxn)
return err
})
return
}
// VerifyForFile verifies index correctness against FileStorage file @ path
// XXX text
func (index *Index) VerifyForFile(ctx context.Context, path string) error {
return index.verifyForFile(path, func(r io.ReaderAt) error {
return index.Verify(ctx, r)
})
}
// common driver for Verify*ForFile
func (index *Index) verifyForFile(path string, check func(r io.ReaderAt) error) error {
f, err := os.Open(path)
if err != nil {
return err
}
defer func() {
err2 := f.Close()
err = xerr.First(err, err2)
}()
fi, err := f.Stat()
if err != nil {
return err
}
topPos := fi.Size() // XXX there might be last TxnInprogress transaction
if index.TopPos != topPos {
return indexCorrupt(f, "topPos mismatch: data=%v index=%v", topPos, index.TopPos)
}
// use IO optimized for sequential access when verifying index
fSeq := xbufio.NewSeqReaderAt(f)
return check(fSeq)
}
...@@ -220,15 +220,27 @@ func TestIndexSaveToPy(t *testing.T) { ...@@ -220,15 +220,27 @@ func TestIndexSaveToPy(t *testing.T) {
} }
} }
func TestIndexBuild(t *testing.T) { func TestIndexBuildVerify(t *testing.T) {
index, err := BuildIndexForFile(context.Background(), "testdata/1.fs") index, err := BuildIndexForFile(context.Background(), "testdata/1.fs")
if err != nil { if err != nil {
t.Fatal(err) t.Fatalf("index build: %v", err)
} }
if !index.Equal(_1fs_index) { if !index.Equal(_1fs_index) {
t.Fatal("computed index differ from expected") t.Fatal("computed index differ from expected")
} }
err = index.VerifyForFile(context.Background(), "testdata/1.fs")
if err != nil {
t.Fatalf("index verify: %v", err)
}
pos0, _ := index.Get(0)
index.Set(0, pos0 + 1)
err = index.VerifyForFile(context.Background(), "testdata/1.fs")
if err == nil {
t.Fatalf("index verify: expected error after tweak")
}
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment