Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
N
neoppod
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
1
Issues
1
List
Boards
Labels
Milestones
Merge Requests
2
Merge Requests
2
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
nexedi
neoppod
Commits
ffde05a5
Commit
ffde05a5
authored
Jul 31, 2017
by
Kirill Smelkov
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
.
parent
a69c7dee
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
56 additions
and
6 deletions
+56
-6
go/zodb/storage/fs1/fs1tools/index.go
go/zodb/storage/fs1/fs1tools/index.go
+31
-3
go/zodb/storage/fs1/index.go
go/zodb/storage/fs1/index.go
+25
-3
No files found.
go/zodb/storage/fs1/fs1tools/index.go
View file @
ffde05a5
...
@@ -82,14 +82,14 @@ func reindexMain(argv []string) {
...
@@ -82,14 +82,14 @@ func reindexMain(argv []string) {
// ----------------------------------------
// ----------------------------------------
// VerifyIndexFor verifies that on-disk index for FileStorage file @ path is correct
// VerifyIndexFor verifies that on-disk index for FileStorage file @ path is correct
func
VerifyIndexFor
(
ctx
context
.
Context
,
path
string
,
ntxn
int
)
(
err
error
)
{
func
VerifyIndexFor
(
ctx
context
.
Context
,
path
string
,
ntxn
int
,
progress
func
(
*
fs1
.
IndexVerifyProgress
)
)
(
err
error
)
{
// XXX lock path.lock ?
// XXX lock path.lock ?
index
,
err
:=
fs1
.
LoadIndexFile
(
path
+
".index"
)
index
,
err
:=
fs1
.
LoadIndexFile
(
path
+
".index"
)
if
err
!=
nil
{
if
err
!=
nil
{
return
err
// XXX err ctx
return
err
// XXX err ctx
}
}
_
,
err
=
index
.
VerifyForFile
(
context
.
Background
(),
path
,
ntxn
)
_
,
err
=
index
.
VerifyForFile
(
context
.
Background
(),
path
,
ntxn
,
progress
)
return
err
return
err
}
}
...
@@ -106,15 +106,18 @@ Verify FileStorage index
...
@@ -106,15 +106,18 @@ Verify FileStorage index
-checkonly <n> only check consistency by verifying against <n>
-checkonly <n> only check consistency by verifying against <n>
last transactions.
last transactions.
-quiet do not show intermediate progress.
-h --help this help text.
-h --help this help text.
`
)
`
)
}
}
func
verifyIdxMain
(
argv
[]
string
)
{
func
verifyIdxMain
(
argv
[]
string
)
{
ntxn
:=
-
1
ntxn
:=
-
1
quiet
:=
false
flags
:=
flag
.
FlagSet
{
Usage
:
func
()
{
verifyIdxUsage
(
os
.
Stderr
)
}}
flags
:=
flag
.
FlagSet
{
Usage
:
func
()
{
verifyIdxUsage
(
os
.
Stderr
)
}}
flags
.
Init
(
""
,
flag
.
ExitOnError
)
flags
.
Init
(
""
,
flag
.
ExitOnError
)
flags
.
IntVar
(
&
ntxn
,
"checkonly"
,
ntxn
,
"check consistency only wrt last <n> transactions"
)
flags
.
IntVar
(
&
ntxn
,
"checkonly"
,
ntxn
,
"check consistency only wrt last <n> transactions"
)
flags
.
BoolVar
(
&
quiet
,
"quiet"
,
quiet
,
"do not show intermediate progress"
)
flags
.
Parse
(
argv
[
1
:
])
flags
.
Parse
(
argv
[
1
:
])
argv
=
flags
.
Args
()
argv
=
flags
.
Args
()
...
@@ -124,8 +127,33 @@ func verifyIdxMain(argv []string) {
...
@@ -124,8 +127,33 @@ func verifyIdxMain(argv []string) {
}
}
storPath
:=
argv
[
0
]
storPath
:=
argv
[
0
]
err
:=
VerifyIndexFor
(
context
.
Background
(),
storPath
,
ntxn
)
// progress display
progress
:=
func
(
p
*
fs1
.
IndexVerifyProgress
)
{
if
p
.
TxnTotal
==
-
1
{
bytesChecked
:=
p
.
Index
.
TopPos
-
p
.
Iter
.
Txnh
.
Pos
bytesAll
:=
p
.
Index
.
TopPos
fmt
.
Printf
(
"Checked data bytes: %.1f%% (%d/%d); #txn: %d, #oid: %d
\n
"
,
100
*
float64
(
bytesChecked
)
/
float64
(
bytesAll
),
// XXX /0 ?
bytesChecked
,
bytesAll
,
p
.
TxnChecked
,
len
(
p
.
OidChecked
))
}
else
{
fmt
.
Printf
(
"Checked data transactions: %.1f%% (%d/%d); #oid: %d
\n
"
,
100
*
float64
(
p
.
TxnChecked
)
/
float64
(
p
.
TxnTotal
),
// XXX /0 ?
p
.
TxnChecked
,
p
.
TxnTotal
,
len
(
p
.
OidChecked
))
}
}
if
quiet
{
progress
=
nil
}
err
:=
VerifyIndexFor
(
context
.
Background
(),
storPath
,
ntxn
,
progress
)
if
err
!=
nil
{
if
err
!=
nil
{
zt
.
Fatal
(
err
)
zt
.
Fatal
(
err
)
}
}
if
!
quiet
{
fmt
.
Println
(
"OK"
)
}
}
}
go/zodb/storage/fs1/index.go
View file @
ffde05a5
...
@@ -553,6 +553,15 @@ func indexCorrupt(r io.ReaderAt, format string, argv ...interface{}) *IndexCorru
...
@@ -553,6 +553,15 @@ func indexCorrupt(r io.ReaderAt, format string, argv ...interface{}) *IndexCorru
return
&
IndexCorruptError
{
DataFileName
:
xio
.
Name
(
r
),
Detail
:
fmt
.
Sprintf
(
format
,
argv
...
)}
return
&
IndexCorruptError
{
DataFileName
:
xio
.
Name
(
r
),
Detail
:
fmt
.
Sprintf
(
format
,
argv
...
)}
}
}
// IndexVerifyProgress is data sent by Index.Verify to progress
type
IndexVerifyProgress
struct
{
TxnTotal
int
// total # of transactions to verify; if = -1 -- whole data
TxnChecked
int
Index
*
Index
// index verification runs for
Iter
*
Iter
// iterator thtough data
OidChecked
map
[
zodb
.
Oid
]
struct
{}
// oid checked so far
}
// Verify checks index correctness against FileStorage data in r.
// Verify checks index correctness against FileStorage data in r.
//
//
// For ntxn transactions starting from index.TopPos backwards, it verifies
// For ntxn transactions starting from index.TopPos backwards, it verifies
...
@@ -568,7 +577,7 @@ func indexCorrupt(r io.ReaderAt, format string, argv ...interface{}) *IndexCorru
...
@@ -568,7 +577,7 @@ func indexCorrupt(r io.ReaderAt, format string, argv ...interface{}) *IndexCorru
// Returned error is either:
// Returned error is either:
// - of type *IndexCorruptError, when data in index was found not to match original data, or
// - of type *IndexCorruptError, when data in index was found not to match original data, or
// - any other error type representing e.g. IO error when reading original data or something else.
// - any other error type representing e.g. IO error when reading original data or something else.
func
(
index
*
Index
)
Verify
(
ctx
context
.
Context
,
r
io
.
ReaderAt
,
ntxn
int
)
(
oidChecked
map
[
zodb
.
Oid
]
struct
{},
err
error
)
{
func
(
index
*
Index
)
Verify
(
ctx
context
.
Context
,
r
io
.
ReaderAt
,
ntxn
int
,
progress
func
(
*
IndexVerifyProgress
)
)
(
oidChecked
map
[
zodb
.
Oid
]
struct
{},
err
error
)
{
defer
func
()
{
defer
func
()
{
if
_
,
ok
:=
err
.
(
*
IndexCorruptError
);
ok
{
if
_
,
ok
:=
err
.
(
*
IndexCorruptError
);
ok
{
return
// leave it as is
return
// leave it as is
...
@@ -581,6 +590,14 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int) (oidChe
...
@@ -581,6 +590,14 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int) (oidChe
wholeData
:=
false
wholeData
:=
false
it
:=
Iterate
(
r
,
index
.
TopPos
,
IterBackward
)
it
:=
Iterate
(
r
,
index
.
TopPos
,
IterBackward
)
pd
:=
&
IndexVerifyProgress
{
TxnTotal
:
ntxn
,
Index
:
index
,
Iter
:
it
,
OidChecked
:
oidChecked
,
}
for
i
:=
0
;
ntxn
==
-
1
||
i
<
ntxn
;
i
++
{
for
i
:=
0
;
ntxn
==
-
1
||
i
<
ntxn
;
i
++
{
// check ctx cancel once per transaction
// check ctx cancel once per transaction
select
{
select
{
...
@@ -625,6 +642,11 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int) (oidChe
...
@@ -625,6 +642,11 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int) (oidChe
it
.
Datah
.
Oid
,
it
.
Datah
.
Pos
,
dataPos
)
it
.
Datah
.
Oid
,
it
.
Datah
.
Pos
,
dataPos
)
}
}
}
}
if
progress
!=
nil
{
pd
.
TxnChecked
=
i
progress
(
pd
)
}
}
}
// all oids from data were checked to be in index
// all oids from data were checked to be in index
...
@@ -652,7 +674,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int) (oidChe
...
@@ -652,7 +674,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int) (oidChe
// VerifyForFile checks index correctness against FileStorage data in file @ path
// VerifyForFile checks index correctness against FileStorage data in file @ path
//
//
// See Verify for semantic description.
// See Verify for semantic description.
func
(
index
*
Index
)
VerifyForFile
(
ctx
context
.
Context
,
path
string
,
ntxn
int
)
(
oidChecked
map
[
zodb
.
Oid
]
struct
{},
err
error
)
{
func
(
index
*
Index
)
VerifyForFile
(
ctx
context
.
Context
,
path
string
,
ntxn
int
,
progress
func
(
*
IndexVerifyProgress
)
)
(
oidChecked
map
[
zodb
.
Oid
]
struct
{},
err
error
)
{
f
,
err
:=
os
.
Open
(
path
)
f
,
err
:=
os
.
Open
(
path
)
if
err
!=
nil
{
if
err
!=
nil
{
return
nil
,
err
return
nil
,
err
...
@@ -676,5 +698,5 @@ func (index *Index) VerifyForFile(ctx context.Context, path string, ntxn int) (o
...
@@ -676,5 +698,5 @@ func (index *Index) VerifyForFile(ctx context.Context, path string, ntxn int) (o
// use IO optimized for sequential access when verifying index
// use IO optimized for sequential access when verifying index
fSeq
:=
xbufio
.
NewSeqReaderAt
(
f
)
fSeq
:=
xbufio
.
NewSeqReaderAt
(
f
)
return
index
.
Verify
(
ctx
,
fSeq
,
ntxn
)
return
index
.
Verify
(
ctx
,
fSeq
,
ntxn
,
progress
)
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment