Commit 15af0624 authored by Kirill Smelkov's avatar Kirill Smelkov

.

parent f71eebf7
......@@ -80,8 +80,8 @@ func IndexNew() *Index {
// oid[6:8]oid[6:8]oid[6:8]...pos[2:8]pos[2:8]pos[2:8]...
const (
oidPrefixMask zodb.Oid = (1<<64-1) ^ (1<<16 - 1) // 0xffffffffffff0000
posInvalidMask uint64 = (1<<64-1) ^ (1<<48 - 1) // 0xffff000000000000
oidPrefixMask zodb.Oid = (1<<64 - 1) ^ (1<<16 - 1) // 0xffffffffffff0000
posInvalidMask uint64 = (1<<64 - 1) ^ (1<<48 - 1) // 0xffff000000000000
posValidMask uint64 = 1<<48 - 1 // 0x0000ffffffffffff
)
......@@ -95,15 +95,26 @@ func (e *IndexSaveError) Error() string {
}
// Save saves index to a writer
func (fsi *Index) Save(w io.Writer) error {
var err error
func (fsi *Index) Save(w io.Writer) (err error) {
defer func() {
if err == nil {
return
}
if _, ok := err.(*pickle.TypeError); ok {
panic(err) // all our types are expected to be supported by pickle
}
// otherwise it is an error returned by writer, which should already
// have filename & op as context.
err = &IndexSaveError{err}
}()
{
p := pickle.NewEncoder(w)
err = p.Encode(fsi.TopPos)
if err != nil {
goto out
return err
}
var oidb [8]byte
......@@ -128,7 +139,7 @@ func (fsi *Index) Save(w io.Writer) error {
t[1] = bytes.Join([][]byte{oidBuf, posBuf}, nil)
err = p.Encode(pickle.Tuple(t[:]))
if err != nil {
goto out
return err
}
oidPrefixCur = oidPrefix
......@@ -141,9 +152,8 @@ func (fsi *Index) Save(w io.Writer) error {
}
// check pos does not overflow 6 bytes
if uint64(pos) & posInvalidMask != 0 {
err = fmt.Errorf("entry position too large: 0x%x", pos)
goto out
if uint64(pos)&posInvalidMask != 0 {
return fmt.Errorf("entry position too large: 0x%x", pos)
}
binary.BigEndian.PutUint64(oidb[:], uint64(oid))
......@@ -155,20 +165,7 @@ func (fsi *Index) Save(w io.Writer) error {
}
err = p.Encode(pickle.None{})
}
out:
if err == nil {
return err
}
if _, ok := err.(*pickle.TypeError); ok {
panic(err) // all our types are expected to be supported by pickle
}
// otherwise it is an error returned by writer, which should already
// have filename & op as context.
return &IndexSaveError{err}
}
// SaveFile saves index to a file @ path.
......@@ -178,9 +175,9 @@ out:
// updated only with complete index data.
func (fsi *Index) SaveFile(path string) error {
dir, name := filepath.Dir(path), filepath.Base(path)
f, err := ioutil.TempFile(dir, name + ".tmp")
f, err := ioutil.TempFile(dir, name+".tmp")
if err != nil {
return &IndexSaveError{err} // XXX needed?
return &IndexSaveError{err}
}
// use buffering for f (ogórek does not buffer itself on encoding)
......@@ -193,14 +190,14 @@ func (fsi *Index) SaveFile(path string) error {
os.Remove(f.Name())
err = err1
if err == nil {
err = &IndexSaveError{xerr.First(err2, err3)} // XXX needed?
err = &IndexSaveError{xerr.First(err2, err3)}
}
return err
}
err = os.Rename(f.Name(), path)
if err != nil {
return &IndexSaveError{err} // XXX needed?
return &IndexSaveError{err}
}
return nil
......@@ -242,8 +239,12 @@ func xint64(xv interface{}) (v int64, ok bool) {
// LoadIndex loads index from a reader
func LoadIndex(r io.Reader) (fsi *Index, err error) {
var picklePos int64
defer func() {
if err != nil {
err = &IndexLoadError{xio.Name(r), picklePos, err}
}
}()
{
var ok bool
var xtopPos, xv interface{}
......@@ -254,38 +255,35 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
picklePos = xr.InputOffset()
xtopPos, err = p.Decode()
if err != nil {
goto out
return nil, err
}
topPos, ok := xint64(xtopPos)
if !ok {
err = fmt.Errorf("topPos is %T:%v (expected int64)", xtopPos, xtopPos)
goto out
return nil, fmt.Errorf("topPos is %T:%v (expected int64)", xtopPos, xtopPos)
}
fsi = IndexNew()
fsi.TopPos = topPos
var oidb [8]byte
loop:
loop:
for {
// load/decode next entry
var v pickle.Tuple
picklePos = xr.InputOffset()
xv, err = p.Decode()
if err != nil {
goto out
return nil, err
}
switch xv := xv.(type) {
default:
err = fmt.Errorf("invalid entry: type %T", xv)
goto out
return nil, fmt.Errorf("invalid entry: type %T", xv)
case pickle.None:
break loop
// we accept tuple or list
// XXX accept only tuple ?
case pickle.Tuple:
v = xv
case []interface{}:
......@@ -294,20 +292,17 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
// unpack entry tuple -> oidPrefix, fsBucket
if len(v) != 2 {
err = fmt.Errorf("invalid entry: len = %i", len(v))
goto out
return nil, fmt.Errorf("invalid entry: len = %i", len(v))
}
// decode oidPrefix
xoidPrefixStr := v[0]
oidPrefixStr, ok := xoidPrefixStr.(string)
if !ok {
err = fmt.Errorf("invalid oidPrefix: type %T", xoidPrefixStr)
goto out
return nil, fmt.Errorf("invalid oidPrefix: type %T", xoidPrefixStr)
}
if l := len(oidPrefixStr); l != 6 {
err = fmt.Errorf("invalid oidPrefix: len = %i", l)
goto out
return nil, fmt.Errorf("invalid oidPrefix: len = %i", l)
}
copy(oidb[:], oidPrefixStr)
oidPrefix := zodb.Oid(binary.BigEndian.Uint64(oidb[:]))
......@@ -316,12 +311,10 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
xkvStr := v[1]
kvStr, ok := xkvStr.(string)
if !ok {
err = fmt.Errorf("invalid fsBucket: type %T", xkvStr)
goto out
return nil, fmt.Errorf("invalid fsBucket: type %T", xkvStr)
}
if l := len(kvStr); l % 8 != 0 {
err = fmt.Errorf("invalid fsBucket: len = %i", l)
goto out
if l := len(kvStr); l%8 != 0 {
return nil, fmt.Errorf("invalid fsBucket: len = %i", l)
}
// load btree from fsBucket entries
......@@ -331,7 +324,7 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
oidBuf := kvBuf[:n*2]
posBuf := kvBuf[n*2-2:] // NOTE starting 2 bytes behind
for i:=0; i<n; i++ {
for i := 0; i < n; i++ {
oid := zodb.Oid(binary.BigEndian.Uint16(oidBuf[i*2:]))
oid |= oidPrefix
pos := int64(binary.BigEndian.Uint64(posBuf[i*6:]) & posValidMask)
......@@ -339,14 +332,8 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
fsi.Set(oid, pos)
}
}
}
out:
if err == nil {
return fsi, err
}
return nil, &IndexLoadError{xio.Name(r), picklePos, err}
return fsi, nil
}
// LoadIndexFile loads index from a file @ path.
......@@ -441,7 +428,7 @@ type IndexUpdateProgress struct {
// On success returned error is nil and index.TopPos is set to either:
// - topPos (if it is != -1), or
// - r's position at which read got EOF (if topPos=-1).
func (index *Index) Update(ctx context.Context, r io.ReaderAt, topPos int64, progress func (*IndexUpdateProgress)) (err error) {
func (index *Index) Update(ctx context.Context, r io.ReaderAt, topPos int64, progress func(*IndexUpdateProgress)) (err error) {
defer xerr.Contextf(&err, "%s: reindex %v..%v", xio.Name(r), index.TopPos, topPos)
if topPos >= 0 && index.TopPos > topPos {
......@@ -485,7 +472,7 @@ func (index *Index) Update(ctx context.Context, r io.ReaderAt, topPos int64, pro
// check for topPos overlapping txn & whether we are done.
// topPos=-1 will never match here
if it.Txnh.Pos < topPos && (it.Txnh.Pos + it.Txnh.Len) > topPos {
if it.Txnh.Pos < topPos && (it.Txnh.Pos+it.Txnh.Len) > topPos {
return fmt.Errorf("transaction %v @%v overlaps topPos boundary",
it.Txnh.Tid, it.Txnh.Pos)
}
......@@ -547,7 +534,7 @@ func BuildIndex(ctx context.Context, r io.ReaderAt, progress func(*IndexUpdatePr
func BuildIndexForFile(ctx context.Context, path string, progress func(*IndexUpdateProgress)) (index *Index, err error) {
f, err := os.Open(path)
if err != nil {
return IndexNew(), err // XXX add err ctx?
return IndexNew(), err
}
defer func() {
......@@ -637,7 +624,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int, progres
wholeData = true
break
}
return oidChecked, err // XXX err ctx
return oidChecked, err
}
for {
......@@ -646,7 +633,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int, progres
if err == io.EOF {
break
}
return oidChecked, err // XXX err ctx
return oidChecked, err
}
// if oid was already checked - do not check index anymore
......@@ -700,7 +687,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int, progres
// VerifyForFile checks index correctness against FileStorage data in file @ path
//
// See Verify for semantic description.
func (index *Index) VerifyForFile(ctx context.Context, path string, ntxn int, progress func (*IndexVerifyProgress)) (oidChecked map[zodb.Oid]struct{}, err error) {
func (index *Index) VerifyForFile(ctx context.Context, path string, ntxn int, progress func(*IndexVerifyProgress)) (oidChecked map[zodb.Oid]struct{}, err error) {
f, err := os.Open(path)
if err != nil {
return nil, err
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment