Commit 15af0624 authored by Kirill Smelkov's avatar Kirill Smelkov

.

parent f71eebf7
...@@ -80,8 +80,8 @@ func IndexNew() *Index { ...@@ -80,8 +80,8 @@ func IndexNew() *Index {
// oid[6:8]oid[6:8]oid[6:8]...pos[2:8]pos[2:8]pos[2:8]... // oid[6:8]oid[6:8]oid[6:8]...pos[2:8]pos[2:8]pos[2:8]...
const ( const (
oidPrefixMask zodb.Oid = (1<<64-1) ^ (1<<16 - 1) // 0xffffffffffff0000 oidPrefixMask zodb.Oid = (1<<64 - 1) ^ (1<<16 - 1) // 0xffffffffffff0000
posInvalidMask uint64 = (1<<64-1) ^ (1<<48 - 1) // 0xffff000000000000 posInvalidMask uint64 = (1<<64 - 1) ^ (1<<48 - 1) // 0xffff000000000000
posValidMask uint64 = 1<<48 - 1 // 0x0000ffffffffffff posValidMask uint64 = 1<<48 - 1 // 0x0000ffffffffffff
) )
...@@ -95,15 +95,26 @@ func (e *IndexSaveError) Error() string { ...@@ -95,15 +95,26 @@ func (e *IndexSaveError) Error() string {
} }
// Save saves index to a writer // Save saves index to a writer
func (fsi *Index) Save(w io.Writer) error { func (fsi *Index) Save(w io.Writer) (err error) {
var err error defer func() {
if err == nil {
return
}
if _, ok := err.(*pickle.TypeError); ok {
panic(err) // all our types are expected to be supported by pickle
}
// otherwise it is an error returned by writer, which should already
// have filename & op as context.
err = &IndexSaveError{err}
}()
{
p := pickle.NewEncoder(w) p := pickle.NewEncoder(w)
err = p.Encode(fsi.TopPos) err = p.Encode(fsi.TopPos)
if err != nil { if err != nil {
goto out return err
} }
var oidb [8]byte var oidb [8]byte
...@@ -128,7 +139,7 @@ func (fsi *Index) Save(w io.Writer) error { ...@@ -128,7 +139,7 @@ func (fsi *Index) Save(w io.Writer) error {
t[1] = bytes.Join([][]byte{oidBuf, posBuf}, nil) t[1] = bytes.Join([][]byte{oidBuf, posBuf}, nil)
err = p.Encode(pickle.Tuple(t[:])) err = p.Encode(pickle.Tuple(t[:]))
if err != nil { if err != nil {
goto out return err
} }
oidPrefixCur = oidPrefix oidPrefixCur = oidPrefix
...@@ -141,9 +152,8 @@ func (fsi *Index) Save(w io.Writer) error { ...@@ -141,9 +152,8 @@ func (fsi *Index) Save(w io.Writer) error {
} }
// check pos does not overflow 6 bytes // check pos does not overflow 6 bytes
if uint64(pos) & posInvalidMask != 0 { if uint64(pos)&posInvalidMask != 0 {
err = fmt.Errorf("entry position too large: 0x%x", pos) return fmt.Errorf("entry position too large: 0x%x", pos)
goto out
} }
binary.BigEndian.PutUint64(oidb[:], uint64(oid)) binary.BigEndian.PutUint64(oidb[:], uint64(oid))
...@@ -155,20 +165,7 @@ func (fsi *Index) Save(w io.Writer) error { ...@@ -155,20 +165,7 @@ func (fsi *Index) Save(w io.Writer) error {
} }
err = p.Encode(pickle.None{}) err = p.Encode(pickle.None{})
}
out:
if err == nil {
return err return err
}
if _, ok := err.(*pickle.TypeError); ok {
panic(err) // all our types are expected to be supported by pickle
}
// otherwise it is an error returned by writer, which should already
// have filename & op as context.
return &IndexSaveError{err}
} }
// SaveFile saves index to a file @ path. // SaveFile saves index to a file @ path.
...@@ -178,9 +175,9 @@ out: ...@@ -178,9 +175,9 @@ out:
// updated only with complete index data. // updated only with complete index data.
func (fsi *Index) SaveFile(path string) error { func (fsi *Index) SaveFile(path string) error {
dir, name := filepath.Dir(path), filepath.Base(path) dir, name := filepath.Dir(path), filepath.Base(path)
f, err := ioutil.TempFile(dir, name + ".tmp") f, err := ioutil.TempFile(dir, name+".tmp")
if err != nil { if err != nil {
return &IndexSaveError{err} // XXX needed? return &IndexSaveError{err}
} }
// use buffering for f (ogórek does not buffer itself on encoding) // use buffering for f (ogórek does not buffer itself on encoding)
...@@ -193,14 +190,14 @@ func (fsi *Index) SaveFile(path string) error { ...@@ -193,14 +190,14 @@ func (fsi *Index) SaveFile(path string) error {
os.Remove(f.Name()) os.Remove(f.Name())
err = err1 err = err1
if err == nil { if err == nil {
err = &IndexSaveError{xerr.First(err2, err3)} // XXX needed? err = &IndexSaveError{xerr.First(err2, err3)}
} }
return err return err
} }
err = os.Rename(f.Name(), path) err = os.Rename(f.Name(), path)
if err != nil { if err != nil {
return &IndexSaveError{err} // XXX needed? return &IndexSaveError{err}
} }
return nil return nil
...@@ -242,8 +239,12 @@ func xint64(xv interface{}) (v int64, ok bool) { ...@@ -242,8 +239,12 @@ func xint64(xv interface{}) (v int64, ok bool) {
// LoadIndex loads index from a reader // LoadIndex loads index from a reader
func LoadIndex(r io.Reader) (fsi *Index, err error) { func LoadIndex(r io.Reader) (fsi *Index, err error) {
var picklePos int64 var picklePos int64
defer func() {
if err != nil {
err = &IndexLoadError{xio.Name(r), picklePos, err}
}
}()
{
var ok bool var ok bool
var xtopPos, xv interface{} var xtopPos, xv interface{}
...@@ -254,38 +255,35 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) { ...@@ -254,38 +255,35 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
picklePos = xr.InputOffset() picklePos = xr.InputOffset()
xtopPos, err = p.Decode() xtopPos, err = p.Decode()
if err != nil { if err != nil {
goto out return nil, err
} }
topPos, ok := xint64(xtopPos) topPos, ok := xint64(xtopPos)
if !ok { if !ok {
err = fmt.Errorf("topPos is %T:%v (expected int64)", xtopPos, xtopPos) return nil, fmt.Errorf("topPos is %T:%v (expected int64)", xtopPos, xtopPos)
goto out
} }
fsi = IndexNew() fsi = IndexNew()
fsi.TopPos = topPos fsi.TopPos = topPos
var oidb [8]byte var oidb [8]byte
loop: loop:
for { for {
// load/decode next entry // load/decode next entry
var v pickle.Tuple var v pickle.Tuple
picklePos = xr.InputOffset() picklePos = xr.InputOffset()
xv, err = p.Decode() xv, err = p.Decode()
if err != nil { if err != nil {
goto out return nil, err
} }
switch xv := xv.(type) { switch xv := xv.(type) {
default: default:
err = fmt.Errorf("invalid entry: type %T", xv) return nil, fmt.Errorf("invalid entry: type %T", xv)
goto out
case pickle.None: case pickle.None:
break loop break loop
// we accept tuple or list // we accept tuple or list
// XXX accept only tuple ?
case pickle.Tuple: case pickle.Tuple:
v = xv v = xv
case []interface{}: case []interface{}:
...@@ -294,20 +292,17 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) { ...@@ -294,20 +292,17 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
// unpack entry tuple -> oidPrefix, fsBucket // unpack entry tuple -> oidPrefix, fsBucket
if len(v) != 2 { if len(v) != 2 {
err = fmt.Errorf("invalid entry: len = %i", len(v)) return nil, fmt.Errorf("invalid entry: len = %i", len(v))
goto out
} }
// decode oidPrefix // decode oidPrefix
xoidPrefixStr := v[0] xoidPrefixStr := v[0]
oidPrefixStr, ok := xoidPrefixStr.(string) oidPrefixStr, ok := xoidPrefixStr.(string)
if !ok { if !ok {
err = fmt.Errorf("invalid oidPrefix: type %T", xoidPrefixStr) return nil, fmt.Errorf("invalid oidPrefix: type %T", xoidPrefixStr)
goto out
} }
if l := len(oidPrefixStr); l != 6 { if l := len(oidPrefixStr); l != 6 {
err = fmt.Errorf("invalid oidPrefix: len = %i", l) return nil, fmt.Errorf("invalid oidPrefix: len = %i", l)
goto out
} }
copy(oidb[:], oidPrefixStr) copy(oidb[:], oidPrefixStr)
oidPrefix := zodb.Oid(binary.BigEndian.Uint64(oidb[:])) oidPrefix := zodb.Oid(binary.BigEndian.Uint64(oidb[:]))
...@@ -316,12 +311,10 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) { ...@@ -316,12 +311,10 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
xkvStr := v[1] xkvStr := v[1]
kvStr, ok := xkvStr.(string) kvStr, ok := xkvStr.(string)
if !ok { if !ok {
err = fmt.Errorf("invalid fsBucket: type %T", xkvStr) return nil, fmt.Errorf("invalid fsBucket: type %T", xkvStr)
goto out
} }
if l := len(kvStr); l % 8 != 0 { if l := len(kvStr); l%8 != 0 {
err = fmt.Errorf("invalid fsBucket: len = %i", l) return nil, fmt.Errorf("invalid fsBucket: len = %i", l)
goto out
} }
// load btree from fsBucket entries // load btree from fsBucket entries
...@@ -331,7 +324,7 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) { ...@@ -331,7 +324,7 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
oidBuf := kvBuf[:n*2] oidBuf := kvBuf[:n*2]
posBuf := kvBuf[n*2-2:] // NOTE starting 2 bytes behind posBuf := kvBuf[n*2-2:] // NOTE starting 2 bytes behind
for i:=0; i<n; i++ { for i := 0; i < n; i++ {
oid := zodb.Oid(binary.BigEndian.Uint16(oidBuf[i*2:])) oid := zodb.Oid(binary.BigEndian.Uint16(oidBuf[i*2:]))
oid |= oidPrefix oid |= oidPrefix
pos := int64(binary.BigEndian.Uint64(posBuf[i*6:]) & posValidMask) pos := int64(binary.BigEndian.Uint64(posBuf[i*6:]) & posValidMask)
...@@ -339,14 +332,8 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) { ...@@ -339,14 +332,8 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
fsi.Set(oid, pos) fsi.Set(oid, pos)
} }
} }
}
out:
if err == nil {
return fsi, err
}
return nil, &IndexLoadError{xio.Name(r), picklePos, err} return fsi, nil
} }
// LoadIndexFile loads index from a file @ path. // LoadIndexFile loads index from a file @ path.
...@@ -441,7 +428,7 @@ type IndexUpdateProgress struct { ...@@ -441,7 +428,7 @@ type IndexUpdateProgress struct {
// On success returned error is nil and index.TopPos is set to either: // On success returned error is nil and index.TopPos is set to either:
// - topPos (if it is != -1), or // - topPos (if it is != -1), or
// - r's position at which read got EOF (if topPos=-1). // - r's position at which read got EOF (if topPos=-1).
func (index *Index) Update(ctx context.Context, r io.ReaderAt, topPos int64, progress func (*IndexUpdateProgress)) (err error) { func (index *Index) Update(ctx context.Context, r io.ReaderAt, topPos int64, progress func(*IndexUpdateProgress)) (err error) {
defer xerr.Contextf(&err, "%s: reindex %v..%v", xio.Name(r), index.TopPos, topPos) defer xerr.Contextf(&err, "%s: reindex %v..%v", xio.Name(r), index.TopPos, topPos)
if topPos >= 0 && index.TopPos > topPos { if topPos >= 0 && index.TopPos > topPos {
...@@ -485,7 +472,7 @@ func (index *Index) Update(ctx context.Context, r io.ReaderAt, topPos int64, pro ...@@ -485,7 +472,7 @@ func (index *Index) Update(ctx context.Context, r io.ReaderAt, topPos int64, pro
// check for topPos overlapping txn & whether we are done. // check for topPos overlapping txn & whether we are done.
// topPos=-1 will never match here // topPos=-1 will never match here
if it.Txnh.Pos < topPos && (it.Txnh.Pos + it.Txnh.Len) > topPos { if it.Txnh.Pos < topPos && (it.Txnh.Pos+it.Txnh.Len) > topPos {
return fmt.Errorf("transaction %v @%v overlaps topPos boundary", return fmt.Errorf("transaction %v @%v overlaps topPos boundary",
it.Txnh.Tid, it.Txnh.Pos) it.Txnh.Tid, it.Txnh.Pos)
} }
...@@ -547,7 +534,7 @@ func BuildIndex(ctx context.Context, r io.ReaderAt, progress func(*IndexUpdatePr ...@@ -547,7 +534,7 @@ func BuildIndex(ctx context.Context, r io.ReaderAt, progress func(*IndexUpdatePr
func BuildIndexForFile(ctx context.Context, path string, progress func(*IndexUpdateProgress)) (index *Index, err error) { func BuildIndexForFile(ctx context.Context, path string, progress func(*IndexUpdateProgress)) (index *Index, err error) {
f, err := os.Open(path) f, err := os.Open(path)
if err != nil { if err != nil {
return IndexNew(), err // XXX add err ctx? return IndexNew(), err
} }
defer func() { defer func() {
...@@ -637,7 +624,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int, progres ...@@ -637,7 +624,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int, progres
wholeData = true wholeData = true
break break
} }
return oidChecked, err // XXX err ctx return oidChecked, err
} }
for { for {
...@@ -646,7 +633,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int, progres ...@@ -646,7 +633,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int, progres
if err == io.EOF { if err == io.EOF {
break break
} }
return oidChecked, err // XXX err ctx return oidChecked, err
} }
// if oid was already checked - do not check index anymore // if oid was already checked - do not check index anymore
...@@ -700,7 +687,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int, progres ...@@ -700,7 +687,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int, progres
// VerifyForFile checks index correctness against FileStorage data in file @ path // VerifyForFile checks index correctness against FileStorage data in file @ path
// //
// See Verify for semantic description. // See Verify for semantic description.
func (index *Index) VerifyForFile(ctx context.Context, path string, ntxn int, progress func (*IndexVerifyProgress)) (oidChecked map[zodb.Oid]struct{}, err error) { func (index *Index) VerifyForFile(ctx context.Context, path string, ntxn int, progress func(*IndexVerifyProgress)) (oidChecked map[zodb.Oid]struct{}, err error) {
f, err := os.Open(path) f, err := os.Open(path)
if err != nil { if err != nil {
return nil, err return nil, err
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment