Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
N
neo
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
2
Merge Requests
2
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Jobs
Commits
Open sidebar
Kirill Smelkov
neo
Commits
15af0624
Commit
15af0624
authored
Nov 08, 2017
by
Kirill Smelkov
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
.
parent
f71eebf7
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
157 additions
and
170 deletions
+157
-170
go/zodb/storage/fs1/index.go
go/zodb/storage/fs1/index.go
+157
-170
No files found.
go/zodb/storage/fs1/index.go
View file @
15af0624
...
...
@@ -80,8 +80,8 @@ func IndexNew() *Index {
// oid[6:8]oid[6:8]oid[6:8]...pos[2:8]pos[2:8]pos[2:8]...
const
(
oidPrefixMask
zodb
.
Oid
=
(
1
<<
64
-
1
)
^
(
1
<<
16
-
1
)
// 0xffffffffffff0000
posInvalidMask
uint64
=
(
1
<<
64
-
1
)
^
(
1
<<
48
-
1
)
// 0xffff000000000000
oidPrefixMask
zodb
.
Oid
=
(
1
<<
64
-
1
)
^
(
1
<<
16
-
1
)
// 0xffffffffffff0000
posInvalidMask
uint64
=
(
1
<<
64
-
1
)
^
(
1
<<
48
-
1
)
// 0xffff000000000000
posValidMask
uint64
=
1
<<
48
-
1
// 0x0000ffffffffffff
)
...
...
@@ -95,15 +95,26 @@ func (e *IndexSaveError) Error() string {
}
// Save saves index to a writer
func
(
fsi
*
Index
)
Save
(
w
io
.
Writer
)
error
{
var
err
error
func
(
fsi
*
Index
)
Save
(
w
io
.
Writer
)
(
err
error
)
{
defer
func
()
{
if
err
==
nil
{
return
}
if
_
,
ok
:=
err
.
(
*
pickle
.
TypeError
);
ok
{
panic
(
err
)
// all our types are expected to be supported by pickle
}
// otherwise it is an error returned by writer, which should already
// have filename & op as context.
err
=
&
IndexSaveError
{
err
}
}()
{
p
:=
pickle
.
NewEncoder
(
w
)
err
=
p
.
Encode
(
fsi
.
TopPos
)
if
err
!=
nil
{
goto
out
return
err
}
var
oidb
[
8
]
byte
...
...
@@ -128,7 +139,7 @@ func (fsi *Index) Save(w io.Writer) error {
t
[
1
]
=
bytes
.
Join
([][]
byte
{
oidBuf
,
posBuf
},
nil
)
err
=
p
.
Encode
(
pickle
.
Tuple
(
t
[
:
]))
if
err
!=
nil
{
goto
out
return
err
}
oidPrefixCur
=
oidPrefix
...
...
@@ -141,9 +152,8 @@ func (fsi *Index) Save(w io.Writer) error {
}
// check pos does not overflow 6 bytes
if
uint64
(
pos
)
&
posInvalidMask
!=
0
{
err
=
fmt
.
Errorf
(
"entry position too large: 0x%x"
,
pos
)
goto
out
if
uint64
(
pos
)
&
posInvalidMask
!=
0
{
return
fmt
.
Errorf
(
"entry position too large: 0x%x"
,
pos
)
}
binary
.
BigEndian
.
PutUint64
(
oidb
[
:
],
uint64
(
oid
))
...
...
@@ -155,20 +165,7 @@ func (fsi *Index) Save(w io.Writer) error {
}
err
=
p
.
Encode
(
pickle
.
None
{})
}
out
:
if
err
==
nil
{
return
err
}
if
_
,
ok
:=
err
.
(
*
pickle
.
TypeError
);
ok
{
panic
(
err
)
// all our types are expected to be supported by pickle
}
// otherwise it is an error returned by writer, which should already
// have filename & op as context.
return
&
IndexSaveError
{
err
}
}
// SaveFile saves index to a file @ path.
...
...
@@ -178,9 +175,9 @@ out:
// updated only with complete index data.
func
(
fsi
*
Index
)
SaveFile
(
path
string
)
error
{
dir
,
name
:=
filepath
.
Dir
(
path
),
filepath
.
Base
(
path
)
f
,
err
:=
ioutil
.
TempFile
(
dir
,
name
+
".tmp"
)
f
,
err
:=
ioutil
.
TempFile
(
dir
,
name
+
".tmp"
)
if
err
!=
nil
{
return
&
IndexSaveError
{
err
}
// XXX needed?
return
&
IndexSaveError
{
err
}
}
// use buffering for f (ogórek does not buffer itself on encoding)
...
...
@@ -193,14 +190,14 @@ func (fsi *Index) SaveFile(path string) error {
os
.
Remove
(
f
.
Name
())
err
=
err1
if
err
==
nil
{
err
=
&
IndexSaveError
{
xerr
.
First
(
err2
,
err3
)}
// XXX needed?
err
=
&
IndexSaveError
{
xerr
.
First
(
err2
,
err3
)}
}
return
err
}
err
=
os
.
Rename
(
f
.
Name
(),
path
)
if
err
!=
nil
{
return
&
IndexSaveError
{
err
}
// XXX needed?
return
&
IndexSaveError
{
err
}
}
return
nil
...
...
@@ -242,8 +239,12 @@ func xint64(xv interface{}) (v int64, ok bool) {
// LoadIndex loads index from a reader
func
LoadIndex
(
r
io
.
Reader
)
(
fsi
*
Index
,
err
error
)
{
var
picklePos
int64
defer
func
()
{
if
err
!=
nil
{
err
=
&
IndexLoadError
{
xio
.
Name
(
r
),
picklePos
,
err
}
}
}()
{
var
ok
bool
var
xtopPos
,
xv
interface
{}
...
...
@@ -254,38 +255,35 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
picklePos
=
xr
.
InputOffset
()
xtopPos
,
err
=
p
.
Decode
()
if
err
!=
nil
{
goto
out
return
nil
,
err
}
topPos
,
ok
:=
xint64
(
xtopPos
)
if
!
ok
{
err
=
fmt
.
Errorf
(
"topPos is %T:%v (expected int64)"
,
xtopPos
,
xtopPos
)
goto
out
return
nil
,
fmt
.
Errorf
(
"topPos is %T:%v (expected int64)"
,
xtopPos
,
xtopPos
)
}
fsi
=
IndexNew
()
fsi
.
TopPos
=
topPos
var
oidb
[
8
]
byte
loop
:
loop
:
for
{
// load/decode next entry
var
v
pickle
.
Tuple
picklePos
=
xr
.
InputOffset
()
xv
,
err
=
p
.
Decode
()
if
err
!=
nil
{
goto
out
return
nil
,
err
}
switch
xv
:=
xv
.
(
type
)
{
default
:
err
=
fmt
.
Errorf
(
"invalid entry: type %T"
,
xv
)
goto
out
return
nil
,
fmt
.
Errorf
(
"invalid entry: type %T"
,
xv
)
case
pickle
.
None
:
break
loop
// we accept tuple or list
// XXX accept only tuple ?
case
pickle
.
Tuple
:
v
=
xv
case
[]
interface
{}
:
...
...
@@ -294,20 +292,17 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
// unpack entry tuple -> oidPrefix, fsBucket
if
len
(
v
)
!=
2
{
err
=
fmt
.
Errorf
(
"invalid entry: len = %i"
,
len
(
v
))
goto
out
return
nil
,
fmt
.
Errorf
(
"invalid entry: len = %i"
,
len
(
v
))
}
// decode oidPrefix
xoidPrefixStr
:=
v
[
0
]
oidPrefixStr
,
ok
:=
xoidPrefixStr
.
(
string
)
if
!
ok
{
err
=
fmt
.
Errorf
(
"invalid oidPrefix: type %T"
,
xoidPrefixStr
)
goto
out
return
nil
,
fmt
.
Errorf
(
"invalid oidPrefix: type %T"
,
xoidPrefixStr
)
}
if
l
:=
len
(
oidPrefixStr
);
l
!=
6
{
err
=
fmt
.
Errorf
(
"invalid oidPrefix: len = %i"
,
l
)
goto
out
return
nil
,
fmt
.
Errorf
(
"invalid oidPrefix: len = %i"
,
l
)
}
copy
(
oidb
[
:
],
oidPrefixStr
)
oidPrefix
:=
zodb
.
Oid
(
binary
.
BigEndian
.
Uint64
(
oidb
[
:
]))
...
...
@@ -316,12 +311,10 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
xkvStr
:=
v
[
1
]
kvStr
,
ok
:=
xkvStr
.
(
string
)
if
!
ok
{
err
=
fmt
.
Errorf
(
"invalid fsBucket: type %T"
,
xkvStr
)
goto
out
return
nil
,
fmt
.
Errorf
(
"invalid fsBucket: type %T"
,
xkvStr
)
}
if
l
:=
len
(
kvStr
);
l
%
8
!=
0
{
err
=
fmt
.
Errorf
(
"invalid fsBucket: len = %i"
,
l
)
goto
out
if
l
:=
len
(
kvStr
);
l
%
8
!=
0
{
return
nil
,
fmt
.
Errorf
(
"invalid fsBucket: len = %i"
,
l
)
}
// load btree from fsBucket entries
...
...
@@ -331,7 +324,7 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
oidBuf
:=
kvBuf
[
:
n
*
2
]
posBuf
:=
kvBuf
[
n
*
2
-
2
:
]
// NOTE starting 2 bytes behind
for
i
:=
0
;
i
<
n
;
i
++
{
for
i
:=
0
;
i
<
n
;
i
++
{
oid
:=
zodb
.
Oid
(
binary
.
BigEndian
.
Uint16
(
oidBuf
[
i
*
2
:
]))
oid
|=
oidPrefix
pos
:=
int64
(
binary
.
BigEndian
.
Uint64
(
posBuf
[
i
*
6
:
])
&
posValidMask
)
...
...
@@ -339,14 +332,8 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
fsi
.
Set
(
oid
,
pos
)
}
}
}
out
:
if
err
==
nil
{
return
fsi
,
err
}
return
nil
,
&
IndexLoadError
{
xio
.
Name
(
r
),
picklePos
,
err
}
return
fsi
,
nil
}
// LoadIndexFile loads index from a file @ path.
...
...
@@ -441,7 +428,7 @@ type IndexUpdateProgress struct {
// On success returned error is nil and index.TopPos is set to either:
// - topPos (if it is != -1), or
// - r's position at which read got EOF (if topPos=-1).
func
(
index
*
Index
)
Update
(
ctx
context
.
Context
,
r
io
.
ReaderAt
,
topPos
int64
,
progress
func
(
*
IndexUpdateProgress
))
(
err
error
)
{
func
(
index
*
Index
)
Update
(
ctx
context
.
Context
,
r
io
.
ReaderAt
,
topPos
int64
,
progress
func
(
*
IndexUpdateProgress
))
(
err
error
)
{
defer
xerr
.
Contextf
(
&
err
,
"%s: reindex %v..%v"
,
xio
.
Name
(
r
),
index
.
TopPos
,
topPos
)
if
topPos
>=
0
&&
index
.
TopPos
>
topPos
{
...
...
@@ -485,7 +472,7 @@ func (index *Index) Update(ctx context.Context, r io.ReaderAt, topPos int64, pro
// check for topPos overlapping txn & whether we are done.
// topPos=-1 will never match here
if
it
.
Txnh
.
Pos
<
topPos
&&
(
it
.
Txnh
.
Pos
+
it
.
Txnh
.
Len
)
>
topPos
{
if
it
.
Txnh
.
Pos
<
topPos
&&
(
it
.
Txnh
.
Pos
+
it
.
Txnh
.
Len
)
>
topPos
{
return
fmt
.
Errorf
(
"transaction %v @%v overlaps topPos boundary"
,
it
.
Txnh
.
Tid
,
it
.
Txnh
.
Pos
)
}
...
...
@@ -547,7 +534,7 @@ func BuildIndex(ctx context.Context, r io.ReaderAt, progress func(*IndexUpdatePr
func
BuildIndexForFile
(
ctx
context
.
Context
,
path
string
,
progress
func
(
*
IndexUpdateProgress
))
(
index
*
Index
,
err
error
)
{
f
,
err
:=
os
.
Open
(
path
)
if
err
!=
nil
{
return
IndexNew
(),
err
// XXX add err ctx?
return
IndexNew
(),
err
}
defer
func
()
{
...
...
@@ -637,7 +624,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int, progres
wholeData
=
true
break
}
return
oidChecked
,
err
// XXX err ctx
return
oidChecked
,
err
}
for
{
...
...
@@ -646,7 +633,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int, progres
if
err
==
io
.
EOF
{
break
}
return
oidChecked
,
err
// XXX err ctx
return
oidChecked
,
err
}
// if oid was already checked - do not check index anymore
...
...
@@ -700,7 +687,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int, progres
// VerifyForFile checks index correctness against FileStorage data in file @ path
//
// See Verify for semantic description.
func
(
index
*
Index
)
VerifyForFile
(
ctx
context
.
Context
,
path
string
,
ntxn
int
,
progress
func
(
*
IndexVerifyProgress
))
(
oidChecked
map
[
zodb
.
Oid
]
struct
{},
err
error
)
{
func
(
index
*
Index
)
VerifyForFile
(
ctx
context
.
Context
,
path
string
,
ntxn
int
,
progress
func
(
*
IndexVerifyProgress
))
(
oidChecked
map
[
zodb
.
Oid
]
struct
{},
err
error
)
{
f
,
err
:=
os
.
Open
(
path
)
if
err
!=
nil
{
return
nil
,
err
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment