Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
N
neoppod
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
1
Issues
1
List
Boards
Labels
Milestones
Merge Requests
2
Merge Requests
2
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
nexedi
neoppod
Commits
f53dc308
Commit
f53dc308
authored
Jul 28, 2017
by
Kirill Smelkov
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
.
parent
d1054012
Changes
5
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
117 additions
and
18 deletions
+117
-18
go/xcommon/xbufio/seqbuf_ioat.go
go/xcommon/xbufio/seqbuf_ioat.go
+4
-0
go/xcommon/xio/xio.go
go/xcommon/xio/xio.go
+7
-0
go/zodb/storage/fs1/fs1tools/index.go
go/zodb/storage/fs1/fs1tools/index.go
+39
-13
go/zodb/storage/fs1/index.go
go/zodb/storage/fs1/index.go
+53
-3
go/zodb/storage/fs1/index_test.go
go/zodb/storage/fs1/index_test.go
+14
-2
No files found.
go/xcommon/xbufio/seqbuf_ioat.go
View file @
f53dc308
...
@@ -60,6 +60,10 @@ func NewSeqReaderAtSize(r io.ReaderAt, size int) *SeqReaderAt {
...
@@ -60,6 +60,10 @@ func NewSeqReaderAtSize(r io.ReaderAt, size int) *SeqReaderAt {
return
sb
return
sb
}
}
// XXX not yet sure it is good idea
func
(
sb
*
SeqReaderAt
)
IOUnwrap
()
interface
{}
{
return
sb
.
r
}
// // XXX temp
// // XXX temp
// func init() {
// func init() {
...
...
go/xcommon/xio/xio.go
View file @
f53dc308
...
@@ -48,6 +48,11 @@ func (l *LimitedWriter) Write(p []byte) (n int, err error) {
...
@@ -48,6 +48,11 @@ func (l *LimitedWriter) Write(p []byte) (n int, err error) {
func
LimitWriter
(
w
io
.
Writer
,
n
int64
)
io
.
Writer
{
return
&
LimitedWriter
{
w
,
n
}
}
func
LimitWriter
(
w
io
.
Writer
,
n
int64
)
io
.
Writer
{
return
&
LimitedWriter
{
w
,
n
}
}
// XXX not sure it is a good ide
// XXX another option is IOName() - but that requires leaf packages to import xio
type
Wrapper
interface
{
IOUnwrap
()
interface
{}
}
// Name returns a "filename" associated with io.Reader, io.Writer, net.Conn, ...
// Name returns a "filename" associated with io.Reader, io.Writer, net.Conn, ...
func
Name
(
f
interface
{})
string
{
func
Name
(
f
interface
{})
string
{
...
@@ -70,6 +75,8 @@ func Name(f interface {}) string {
...
@@ -70,6 +75,8 @@ func Name(f interface {}) string {
// XXX SectionReader MultiReader TeeReader
// XXX SectionReader MultiReader TeeReader
// XXX bufio.Reader bufio.Writer bufio.Scanner
// XXX bufio.Reader bufio.Writer bufio.Scanner
case
Wrapper
:
return
Name
(
f
.
IOUnwrap
())
// if name cannot be determined - let's provide full info
// if name cannot be determined - let's provide full info
default
:
default
:
return
fmt
.
Sprintf
(
"%#v"
,
f
)
return
fmt
.
Sprintf
(
"%#v"
,
f
)
...
...
go/zodb/storage/fs1/fs1tools/index.go
View file @
f53dc308
...
@@ -31,9 +31,9 @@ import (
...
@@ -31,9 +31,9 @@ import (
)
)
// Reindex rebuilds index for FileStorage file @ path
// Reindex rebuilds index for FileStorage file @ path
func
Reindex
(
path
string
)
error
{
func
Reindex
(
ctx
context
.
Context
,
path
string
)
error
{
// XXX lock path.lock ?
// XXX lock path.lock ?
index
,
err
:=
fs1
.
BuildIndexForFile
(
c
ontext
.
Background
()
,
path
)
index
,
err
:=
fs1
.
BuildIndexForFile
(
c
tx
,
path
)
if
err
!=
nil
{
if
err
!=
nil
{
return
err
return
err
}
}
...
@@ -84,21 +84,47 @@ func reindexMain(argv []string) {
...
@@ -84,21 +84,47 @@ func reindexMain(argv []string) {
// TODO verify-index -quick (only small sanity check)
// TODO verify-index -quick (only small sanity check)
// VerifyIndexFor verifies that on-disk index for FileStorage file @ path is correct
// VerifyIndexFor verifies that on-disk index for FileStorage file @ path is correct
func
VerifyIndexFor
(
path
string
)
error
{
func
VerifyIndexFor
(
ctx
context
.
Context
,
path
string
)
(
err
error
)
{
panic
(
"TODO"
)
// XXX lock path.lock ?
/*
inverify
:=
false
// XXX open read-only
defer
func
()
{
fs, err := fs1.Open(context.Background(), path) // XXX , 0)
if
inverify
{
return
// index.Verify provides all context in error
}
xerr
.
Contextf
(
&
err
,
"%s: verify index"
,
path
)
}
f
,
err
:=
os
.
Open
(
path
)
if
err
!=
nil
{
return
err
}
defer
func
()
{
err2
:=
f
.
Close
()
// XXX vs inverify
err
=
xerr
.
First
(
err
,
err2
)
}()
index
,
err
:=
LoadIndexFile
(
path
+
".index"
)
if
err
!=
nil
{
if
err
!=
nil
{
return nil // XXX err ctx
return
err
}
fi
,
err
:=
f
.
Stat
()
if
err
!=
nil
{
return
err
}
topPos
:=
fi
.
Size
()
// XXX there might be last TxnInprogress transaction
if
index
.
TopPos
!=
topPos
{
return
fmt
.
Errorf
(
"topPos mismatch: data=%v index=%v"
,
topPos
,
index
.
TopPos
)
}
}
defer fs.Close() // XXX err
err = fs.VerifyIndex(nil)
// XXX - better somehow not here?
fSeq
:=
xbufio
.
NewSeqReaderAt
(
f
)
inverify
=
true
err
=
index
.
Verify
(
ctx
,
fSeq
)
return
err
return
err
//fs.Index()
//fs.ComputeIndex
*/
}
}
const
verifyIdxSummary
=
"verify database index"
const
verifyIdxSummary
=
"verify database index"
...
...
go/zodb/storage/fs1/index.go
View file @
f53dc308
...
@@ -555,7 +555,7 @@ func indexCorrupt(r io.ReaderAt, format string, argv ...interface{}) *IndexCorru
...
@@ -555,7 +555,7 @@ func indexCorrupt(r io.ReaderAt, format string, argv ...interface{}) *IndexCorru
// VerifyTail checks index correctness against several newest transactions of FileStorage data in r.
// VerifyTail checks index correctness against several newest transactions of FileStorage data in r.
//
//
// For ntxn transactions starting from index.TopPos backwards, it verifies
// For
(XXX max)
ntxn transactions starting from index.TopPos backwards, it verifies
// whether oid there have correct entries in the index.
// whether oid there have correct entries in the index.
//
//
// ntxn=-1 means data range to verify is till start of the file.
// ntxn=-1 means data range to verify is till start of the file.
...
@@ -585,7 +585,9 @@ func (index *Index) VerifyTail(ctx context.Context, r io.ReaderAt, ntxn int) (oi
...
@@ -585,7 +585,9 @@ func (index *Index) VerifyTail(ctx context.Context, r io.ReaderAt, ntxn int) (oi
err
:=
it
.
NextTxn
(
LoadNoStrings
)
err
:=
it
.
NextTxn
(
LoadNoStrings
)
if
err
!=
nil
{
if
err
!=
nil
{
err
=
okEOF
(
err
)
if
err
==
io
.
EOF
{
break
}
return
nil
,
err
// XXX err ctx
return
nil
,
err
// XXX err ctx
}
}
...
@@ -618,10 +620,11 @@ func (index *Index) VerifyTail(ctx context.Context, r io.ReaderAt, ntxn int) (oi
...
@@ -618,10 +620,11 @@ func (index *Index) VerifyTail(ctx context.Context, r io.ReaderAt, ntxn int) (oi
}
}
}
}
// TODO err = EOF -> merge from Verify
return
oidChecked
,
nil
return
oidChecked
,
nil
}
}
// Verify checks index correctness against FileStorage data in r.
// Verify checks index correctness against FileStorage data in r.
//
//
// it verifies whether index is exactly the same as if it was build anew for
// it verifies whether index is exactly the same as if it was build anew for
...
@@ -634,6 +637,8 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt) error {
...
@@ -634,6 +637,8 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt) error {
return
err
return
err
}
}
// XXX merge this into VerifyTail
// all oids from data were checked to be in index
// all oids from data were checked to be in index
// now verify that there is no extra oids in index
// now verify that there is no extra oids in index
if
len
(
oidChecked
)
==
index
.
Len
()
{
if
len
(
oidChecked
)
==
index
.
Len
()
{
...
@@ -656,3 +661,48 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt) error {
...
@@ -656,3 +661,48 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt) error {
return
nil
return
nil
}
}
// XXX text
func
(
index
*
Index
)
VerifyTailForFile
(
ctx
context
.
Context
,
path
string
,
ntxn
int
)
(
oidChecked
map
[
zodb
.
Oid
]
struct
{},
err
error
)
{
err
=
index
.
verifyForFile
(
path
,
func
(
r
io
.
ReaderAt
)
error
{
oidChecked
,
err
=
index
.
VerifyTail
(
ctx
,
r
,
ntxn
)
return
err
})
return
}
// VerifyForFile verifies index correctness against FileStorage file @ path
// XXX text
func
(
index
*
Index
)
VerifyForFile
(
ctx
context
.
Context
,
path
string
)
error
{
return
index
.
verifyForFile
(
path
,
func
(
r
io
.
ReaderAt
)
error
{
return
index
.
Verify
(
ctx
,
r
)
})
}
// common driver for Verify*ForFile
func
(
index
*
Index
)
verifyForFile
(
path
string
,
check
func
(
r
io
.
ReaderAt
)
error
)
error
{
f
,
err
:=
os
.
Open
(
path
)
if
err
!=
nil
{
return
err
}
defer
func
()
{
err2
:=
f
.
Close
()
err
=
xerr
.
First
(
err
,
err2
)
}()
fi
,
err
:=
f
.
Stat
()
if
err
!=
nil
{
return
err
}
topPos
:=
fi
.
Size
()
// XXX there might be last TxnInprogress transaction
if
index
.
TopPos
!=
topPos
{
return
indexCorrupt
(
f
,
"topPos mismatch: data=%v index=%v"
,
topPos
,
index
.
TopPos
)
}
// use IO optimized for sequential access when verifying index
fSeq
:=
xbufio
.
NewSeqReaderAt
(
f
)
return
check
(
fSeq
)
}
go/zodb/storage/fs1/index_test.go
View file @
f53dc308
...
@@ -220,15 +220,27 @@ func TestIndexSaveToPy(t *testing.T) {
...
@@ -220,15 +220,27 @@ func TestIndexSaveToPy(t *testing.T) {
}
}
}
}
func
TestIndexBuild
(
t
*
testing
.
T
)
{
func
TestIndexBuild
Verify
(
t
*
testing
.
T
)
{
index
,
err
:=
BuildIndexForFile
(
context
.
Background
(),
"testdata/1.fs"
)
index
,
err
:=
BuildIndexForFile
(
context
.
Background
(),
"testdata/1.fs"
)
if
err
!=
nil
{
if
err
!=
nil
{
t
.
Fatal
(
err
)
t
.
Fatal
f
(
"index build: %v"
,
err
)
}
}
if
!
index
.
Equal
(
_1fs_index
)
{
if
!
index
.
Equal
(
_1fs_index
)
{
t
.
Fatal
(
"computed index differ from expected"
)
t
.
Fatal
(
"computed index differ from expected"
)
}
}
err
=
index
.
VerifyForFile
(
context
.
Background
(),
"testdata/1.fs"
)
if
err
!=
nil
{
t
.
Fatalf
(
"index verify: %v"
,
err
)
}
pos0
,
_
:=
index
.
Get
(
0
)
index
.
Set
(
0
,
pos0
+
1
)
err
=
index
.
VerifyForFile
(
context
.
Background
(),
"testdata/1.fs"
)
if
err
==
nil
{
t
.
Fatalf
(
"index verify: expected error after tweak"
)
}
}
}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment