Skip to content

N
neoppod
Commit 34c4d915 authored by Levin Zimmermann's avatar Levin Zimmermann
Browse files
Options

proto: Update 'Compression' to int to support different compression algorithms 

With nexedi/neoppod@fd80cc30 NEO/py added support to encode the compression
algorithm with the 'Compression' parameter. Before this, compression could
only be true (= with compression) or false (= without compression). Now
the absence of compression is encoded with 0. Any other number than 0
encodes a compression algorithm. The mapping is currently:

	1 = zlib

In the future, 2 could mean zstd [1].

[1] https://github.com/facebook/zstd/issues/1134
parent 33fab6e3
Show whitespace changes
Inline Side-by-side
Showing with 168 additions and 132 deletions
go/neo/client.go
View file @ 34c4d915
...@@ -381,7 +381,10 @@ func (c *Client) Load(ctx context.Context, xid zodb.Xid) (buf *mem.Buf, serial z ...@@ -381,7 +381,10 @@ func (c *Client) Load(ctx context.Context, xid zodb.Xid) (buf *mem.Buf, serial z
} }
} }
if resp.Compression { if resp.Compression != 0 {
if resp.Compression != 1 {
return nil, 0, fmt.Errorf("unsupported compression algorithm: %v", resp.Compression)
}
buf2 := &mem.Buf{Data: nil} buf2 := &mem.Buf{Data: nil}
udata, err := xzlib.Decompress(buf.Data) udata, err := xzlib.Decompress(buf.Data)
buf.Release() buf.Release()
......
go/neo/neo_test.go
View file @ 34c4d915
...@@ -257,7 +257,7 @@ func _TestMasterStorage(t0 *tEnv) { ...@@ -257,7 +257,7 @@ func _TestMasterStorage(t0 *tEnv) {
Oid: xid1.Oid, Oid: xid1.Oid,
Serial: serial1, Serial: serial1,
NextSerial: proto.INVALID_TID, NextSerial: proto.INVALID_TID,
Compression: false, Compression: 0,
Data: buf1, Data: buf1,
DataSerial: 0, // XXX DataSerial: 0, // XXX
Checksum: sha1.Sum(buf1.Data), Checksum: sha1.Sum(buf1.Data),
...@@ -292,7 +292,7 @@ func _TestMasterStorage(t0 *tEnv) { ...@@ -292,7 +292,7 @@ func _TestMasterStorage(t0 *tEnv) {
Oid: xid1prev.Oid, Oid: xid1prev.Oid,
Serial: serial1prev, Serial: serial1prev,
NextSerial: serial1, NextSerial: serial1,
Compression: false, Compression: 0,
Data: buf1prev, Data: buf1prev,
DataSerial: 0, // XXX DataSerial: 0, // XXX
Checksum: sha1.Sum(buf1prev.Data), Checksum: sha1.Sum(buf1prev.Data),
......
go/neo/proto/proto.go
View file @ 34c4d915
...@@ -336,6 +336,11 @@ func (a *Address) neoDecodeN(b []byte) (uint64, bool) { ...@@ -336,6 +336,11 @@ func (a *Address) neoDecodeN(b []byte) (uint64, bool) {
// Checksum is a SHA1 hash. // Checksum is a SHA1 hash.
type Checksum [20]byte type Checksum [20]byte
// Compression is an integer that tells the compression algorithm:
// 0 = no compression algorithm is used
// 1 = zlib is used
type Compression uint64
// PTid is Partition Table identifier. // PTid is Partition Table identifier.
// //
// Zero value means "invalid id" (<-> None in py.PPTID, nil in msgpack) // Zero value means "invalid id" (<-> None in py.PPTID, nil in msgpack)
...@@ -715,7 +720,7 @@ type AnswerRebaseObject struct { ...@@ -715,7 +720,7 @@ type AnswerRebaseObject struct {
Serial zodb.Tid Serial zodb.Tid
ConflictSerial zodb.Tid ConflictSerial zodb.Tid
// FIXME POption('data') // FIXME POption('data')
Compression bool Compression Compression
Checksum Checksum Checksum Checksum
Data *mem.Buf Data *mem.Buf
} }
...@@ -729,7 +734,7 @@ type AnswerRebaseObject struct { ...@@ -729,7 +734,7 @@ type AnswerRebaseObject struct {
type StoreObject struct { type StoreObject struct {
Oid zodb.Oid Oid zodb.Oid
Serial zodb.Tid Serial zodb.Tid
Compression bool Compression Compression
Checksum Checksum Checksum Checksum
Data []byte // TODO -> msg.Buf, separately (for writev) Data []byte // TODO -> msg.Buf, separately (for writev)
DataSerial zodb.Tid DataSerial zodb.Tid
...@@ -784,7 +789,7 @@ type AnswerObject struct { ...@@ -784,7 +789,7 @@ type AnswerObject struct {
Oid zodb.Oid Oid zodb.Oid
Serial zodb.Tid Serial zodb.Tid
NextSerial zodb.Tid NextSerial zodb.Tid
Compression bool Compression Compression
Checksum Checksum Checksum Checksum
Data *mem.Buf // TODO encode -> separately (for writev) Data *mem.Buf // TODO encode -> separately (for writev)
DataSerial zodb.Tid DataSerial zodb.Tid
...@@ -1219,7 +1224,7 @@ type AddTransaction struct { ...@@ -1219,7 +1224,7 @@ type AddTransaction struct {
type AddObject struct { type AddObject struct {
Oid zodb.Oid Oid zodb.Oid
Serial zodb.Tid Serial zodb.Tid
Compression bool Compression Compression
Checksum Checksum Checksum Checksum
Data *mem.Buf Data *mem.Buf
DataSerial zodb.Tid DataSerial zodb.Tid
......
go/neo/proto/zproto-marshal.go
View file @ 34c4d915
...@@ -4980,18 +4980,18 @@ func (*AnswerRebaseObject) neoMsgCode() uint16 { ...@@ -4980,18 +4980,18 @@ func (*AnswerRebaseObject) neoMsgCode() uint16 {
} }
func (p *AnswerRebaseObject) neoMsgEncodedLenN() int { func (p *AnswerRebaseObject) neoMsgEncodedLenN() int {
return 41 + len(p.Data.XData()) return 48 + len(p.Data.XData())
} }
func (p *AnswerRebaseObject) neoMsgEncodeN(data []byte) { func (p *AnswerRebaseObject) neoMsgEncodeN(data []byte) {
binary.BigEndian.PutUint64(data[0:], uint64(p.Serial)) binary.BigEndian.PutUint64(data[0:], uint64(p.Serial))
binary.BigEndian.PutUint64(data[8:], uint64(p.ConflictSerial)) binary.BigEndian.PutUint64(data[8:], uint64(p.ConflictSerial))
(data[16:])[0] = bool2byte(p.Compression) binary.BigEndian.PutUint64(data[16:], p.Compression)
copy(data[17:], p.Checksum[:]) copy(data[24:], p.Checksum[:])
{ {
l := uint32(len(p.Data.XData())) l := uint32(len(p.Data.XData()))
binary.BigEndian.PutUint32(data[37:], l) binary.BigEndian.PutUint32(data[44:], l)
data = data[41:] data = data[48:]
copy(data, p.Data.XData()) copy(data, p.Data.XData())
data = data[l:] data = data[l:]
} }
...@@ -4999,16 +4999,16 @@ func (p *AnswerRebaseObject) neoMsgEncodeN(data []byte) { ...@@ -4999,16 +4999,16 @@ func (p *AnswerRebaseObject) neoMsgEncodeN(data []byte) {
func (p *AnswerRebaseObject) neoMsgDecodeN(data []byte) (int, error) { func (p *AnswerRebaseObject) neoMsgDecodeN(data []byte) (int, error) {
var nread uint64 var nread uint64
if len(data) < 41 { if len(data) < 48 {
goto overflow goto overflow
} }
p.Serial = zodb.Tid(binary.BigEndian.Uint64(data[0 : 0+8])) p.Serial = zodb.Tid(binary.BigEndian.Uint64(data[0 : 0+8]))
p.ConflictSerial = zodb.Tid(binary.BigEndian.Uint64(data[8 : 8+8])) p.ConflictSerial = zodb.Tid(binary.BigEndian.Uint64(data[8 : 8+8]))
p.Compression = byte2bool((data[16 : 16+1])[0]) p.Compression = binary.BigEndian.Uint64(data[16 : 16+8])
copy(p.Checksum[:], data[17:37]) copy(p.Checksum[:], data[24:44])
{ {
l := binary.BigEndian.Uint32(data[37 : 37+4]) l := binary.BigEndian.Uint32(data[44 : 44+4])
data = data[41:] data = data[48:]
if uint64(len(data)) < uint64(l) { if uint64(len(data)) < uint64(l) {
goto overflow goto overflow
} }
...@@ -5017,7 +5017,7 @@ func (p *AnswerRebaseObject) neoMsgDecodeN(data []byte) (int, error) { ...@@ -5017,7 +5017,7 @@ func (p *AnswerRebaseObject) neoMsgDecodeN(data []byte) (int, error) {
copy(p.Data.Data, data[:l]) copy(p.Data.Data, data[:l])
data = data[l:] data = data[l:]
} }
return 41 + int(nread), nil return 48 + int(nread), nil
overflow: overflow:
return 0, ErrDecodeOverflow return 0, ErrDecodeOverflow
...@@ -5035,7 +5035,7 @@ func (p *AnswerRebaseObject) neoMsgEncodedLenM() int { ...@@ -5035,7 +5035,7 @@ func (p *AnswerRebaseObject) neoMsgEncodedLenM() int {
} else { } else {
size += 1 + 1 + 8 // mbin8 + 8 + [8]data size += 1 + 1 + 8 // mbin8 + 8 + [8]data
} }
return 24 + msgpack.BinHeadSize(len(p.Data.XData())) + len(p.Data.XData()) + size return 23 + msgpack.Uint64Size(p.Compression) + msgpack.BinHeadSize(len(p.Data.XData())) + len(p.Data.XData()) + size
} }
func (p *AnswerRebaseObject) neoMsgEncodeM(data []byte) { func (p *AnswerRebaseObject) neoMsgEncodeM(data []byte) {
...@@ -5058,14 +5058,17 @@ func (p *AnswerRebaseObject) neoMsgEncodeM(data []byte) { ...@@ -5058,14 +5058,17 @@ func (p *AnswerRebaseObject) neoMsgEncodeM(data []byte) {
binary.BigEndian.PutUint64(data[2:], uint64(p.ConflictSerial)) binary.BigEndian.PutUint64(data[2:], uint64(p.ConflictSerial))
data = data[10:] data = data[10:]
} }
data[0] = byte(msgpack.Bool(p.Compression)) {
data[1] = byte(msgpack.Bin8) n := msgpack.PutUint64(data[0:], p.Compression)
data[2] = 20 data = data[0+n:]
copy(data[3:], p.Checksum[:]) }
data[0] = byte(msgpack.Bin8)
data[1] = 20
copy(data[2:], p.Checksum[:])
{ {
l := len(p.Data.XData()) l := len(p.Data.XData())
n := msgpack.PutBinHead(data[23:], l) n := msgpack.PutBinHead(data[22:], l)
data = data[23+n:] data = data[22+n:]
copy(data, p.Data.XData()) copy(data, p.Data.XData())
data = data[l:] data = data[l:]
} }
...@@ -5073,7 +5076,7 @@ func (p *AnswerRebaseObject) neoMsgEncodeM(data []byte) { ...@@ -5073,7 +5076,7 @@ func (p *AnswerRebaseObject) neoMsgEncodeM(data []byte) {
func (p *AnswerRebaseObject) neoMsgDecodeM(data []byte) (int, error) { func (p *AnswerRebaseObject) neoMsgDecodeM(data []byte) (int, error) {
var nread uint64 var nread uint64
if len(data) < 44 { if len(data) < 21 {
goto overflow goto overflow
} }
if op, opOk := msgpack.Op(data[0]), msgpack.FixArray_4|5; op != opOk { if op, opOk := msgpack.Op(data[0]), msgpack.FixArray_4|5; op != opOk {
...@@ -5112,22 +5115,26 @@ func (p *AnswerRebaseObject) neoMsgDecodeM(data []byte) (int, error) { ...@@ -5112,22 +5115,26 @@ func (p *AnswerRebaseObject) neoMsgDecodeM(data []byte) (int, error) {
p.ConflictSerial = zodb.Tid(binary.BigEndian.Uint64(data[2:])) p.ConflictSerial = zodb.Tid(binary.BigEndian.Uint64(data[2:]))
data = data[10:] data = data[10:]
} }
switch op := msgpack.Op(data[0]); op { {
default: v, tail, err := msgp.ReadUint64Bytes(data)
return 0, mdecodeOpErr("AnswerRebaseObject.Compression", op, msgpack.True, msgpack.False) if err != nil {
case msgpack.True: return 0, mdecodeErr("AnswerRebaseObject.Compression", err)
p.Compression = true
case msgpack.False:
p.Compression = false
} }
if op := msgpack.Op(data[1]); op != msgpack.Bin8 { p.Compression = v
nread += uint64(len(data) - len(tail))
data = tail
}
if len(data) < 22 {
goto overflow
}
if op := msgpack.Op(data[0]); op != msgpack.Bin8 {
return 0, mdecodeOpErr("AnswerRebaseObject.Checksum", op, msgpack.Bin8) return 0, mdecodeOpErr("AnswerRebaseObject.Checksum", op, msgpack.Bin8)
} }
if l := data[2]; l != 20 { if l := data[1]; l != 20 {
return 0, mdecodeLen8Err("AnswerRebaseObject.Checksum", l, 20) return 0, mdecodeLen8Err("AnswerRebaseObject.Checksum", l, 20)
} }
copy(p.Checksum[:], data[3:23]) copy(p.Checksum[:], data[2:22])
data = data[23:] data = data[22:]
{ {
b, tail, err := msgp.ReadBytesZC(data) b, tail, err := msgp.ReadBytesZC(data)
if err != nil { if err != nil {
...@@ -5138,7 +5145,7 @@ func (p *AnswerRebaseObject) neoMsgDecodeM(data []byte) (int, error) { ...@@ -5138,7 +5145,7 @@ func (p *AnswerRebaseObject) neoMsgDecodeM(data []byte) (int, error) {
nread += uint64(len(data) - len(tail)) nread += uint64(len(data) - len(tail))
data = tail data = tail
} }
return 44 + int(nread), nil return 43 + int(nread), nil
overflow: overflow:
return 0, ErrDecodeOverflow return 0, ErrDecodeOverflow
...@@ -5151,18 +5158,18 @@ func (*StoreObject) neoMsgCode() uint16 { ...@@ -5151,18 +5158,18 @@ func (*StoreObject) neoMsgCode() uint16 {
} }
func (p *StoreObject) neoMsgEncodedLenN() int { func (p *StoreObject) neoMsgEncodedLenN() int {
return 57 + len(p.Data) return 64 + len(p.Data)
} }
func (p *StoreObject) neoMsgEncodeN(data []byte) { func (p *StoreObject) neoMsgEncodeN(data []byte) {
binary.BigEndian.PutUint64(data[0:], uint64(p.Oid)) binary.BigEndian.PutUint64(data[0:], uint64(p.Oid))
binary.BigEndian.PutUint64(data[8:], uint64(p.Serial)) binary.BigEndian.PutUint64(data[8:], uint64(p.Serial))
(data[16:])[0] = bool2byte(p.Compression) binary.BigEndian.PutUint64(data[16:], p.Compression)
copy(data[17:], p.Checksum[:]) copy(data[24:], p.Checksum[:])
{ {
l := uint32(len(p.Data)) l := uint32(len(p.Data))
binary.BigEndian.PutUint32(data[37:], l) binary.BigEndian.PutUint32(data[44:], l)
data = data[41:] data = data[48:]
copy(data, p.Data) copy(data, p.Data)
data = data[l:] data = data[l:]
} }
...@@ -5172,16 +5179,16 @@ func (p *StoreObject) neoMsgEncodeN(data []byte) { ...@@ -5172,16 +5179,16 @@ func (p *StoreObject) neoMsgEncodeN(data []byte) {
func (p *StoreObject) neoMsgDecodeN(data []byte) (int, error) { func (p *StoreObject) neoMsgDecodeN(data []byte) (int, error) {
var nread uint64 var nread uint64
if len(data) < 41 { if len(data) < 48 {
goto overflow goto overflow
} }
p.Oid = zodb.Oid(binary.BigEndian.Uint64(data[0 : 0+8])) p.Oid = zodb.Oid(binary.BigEndian.Uint64(data[0 : 0+8]))
p.Serial = zodb.Tid(binary.BigEndian.Uint64(data[8 : 8+8])) p.Serial = zodb.Tid(binary.BigEndian.Uint64(data[8 : 8+8]))
p.Compression = byte2bool((data[16 : 16+1])[0]) p.Compression = binary.BigEndian.Uint64(data[16 : 16+8])
copy(p.Checksum[:], data[17:37]) copy(p.Checksum[:], data[24:44])
{ {
l := binary.BigEndian.Uint32(data[37 : 37+4]) l := binary.BigEndian.Uint32(data[44 : 44+4])
data = data[41:] data = data[48:]
if uint64(len(data)) < 16+uint64(l) { if uint64(len(data)) < 16+uint64(l) {
goto overflow goto overflow
} }
...@@ -5192,7 +5199,7 @@ func (p *StoreObject) neoMsgDecodeN(data []byte) (int, error) { ...@@ -5192,7 +5199,7 @@ func (p *StoreObject) neoMsgDecodeN(data []byte) (int, error) {
} }
p.DataSerial = zodb.Tid(binary.BigEndian.Uint64(data[0 : 0+8])) p.DataSerial = zodb.Tid(binary.BigEndian.Uint64(data[0 : 0+8]))
p.Tid = zodb.Tid(binary.BigEndian.Uint64(data[8 : 8+8])) p.Tid = zodb.Tid(binary.BigEndian.Uint64(data[8 : 8+8]))
return 41 + int(nread), nil return 48 + int(nread), nil
overflow: overflow:
return 0, ErrDecodeOverflow return 0, ErrDecodeOverflow
...@@ -5220,7 +5227,7 @@ func (p *StoreObject) neoMsgEncodedLenM() int { ...@@ -5220,7 +5227,7 @@ func (p *StoreObject) neoMsgEncodedLenM() int {
} else { } else {
size += 1 + 1 + 8 // mbin8 + 8 + [8]data size += 1 + 1 + 8 // mbin8 + 8 + [8]data
} }
return 24 + msgpack.BinHeadSize(len(p.Data)) + len(p.Data) + size return 23 + msgpack.Uint64Size(p.Compression) + msgpack.BinHeadSize(len(p.Data)) + len(p.Data) + size
} }
func (p *StoreObject) neoMsgEncodeM(data []byte) { func (p *StoreObject) neoMsgEncodeM(data []byte) {
...@@ -5243,14 +5250,17 @@ func (p *StoreObject) neoMsgEncodeM(data []byte) { ...@@ -5243,14 +5250,17 @@ func (p *StoreObject) neoMsgEncodeM(data []byte) {
binary.BigEndian.PutUint64(data[2:], uint64(p.Serial)) binary.BigEndian.PutUint64(data[2:], uint64(p.Serial))
data = data[10:] data = data[10:]
} }
data[0] = byte(msgpack.Bool(p.Compression)) {
data[1] = byte(msgpack.Bin8) n := msgpack.PutUint64(data[0:], p.Compression)
data[2] = 20 data = data[0+n:]
copy(data[3:], p.Checksum[:]) }
data[0] = byte(msgpack.Bin8)
data[1] = 20
copy(data[2:], p.Checksum[:])
{ {
l := len(p.Data) l := len(p.Data)
n := msgpack.PutBinHead(data[23:], l) n := msgpack.PutBinHead(data[22:], l)
data = data[23+n:] data = data[22+n:]
copy(data, p.Data) copy(data, p.Data)
data = data[l:] data = data[l:]
} }
...@@ -5276,7 +5286,7 @@ func (p *StoreObject) neoMsgEncodeM(data []byte) { ...@@ -5276,7 +5286,7 @@ func (p *StoreObject) neoMsgEncodeM(data []byte) {
func (p *StoreObject) neoMsgDecodeM(data []byte) (int, error) { func (p *StoreObject) neoMsgDecodeM(data []byte) (int, error) {
var nread uint64 var nread uint64
if len(data) < 44 { if len(data) < 21 {
goto overflow goto overflow
} }
if op, opOk := msgpack.Op(data[0]), msgpack.FixArray_4|7; op != opOk { if op, opOk := msgpack.Op(data[0]), msgpack.FixArray_4|7; op != opOk {
...@@ -5315,22 +5325,26 @@ func (p *StoreObject) neoMsgDecodeM(data []byte) (int, error) { ...@@ -5315,22 +5325,26 @@ func (p *StoreObject) neoMsgDecodeM(data []byte) (int, error) {
p.Serial = zodb.Tid(binary.BigEndian.Uint64(data[2:])) p.Serial = zodb.Tid(binary.BigEndian.Uint64(data[2:]))
data = data[10:] data = data[10:]
} }
switch op := msgpack.Op(data[0]); op { {
default: v, tail, err := msgp.ReadUint64Bytes(data)
return 0, mdecodeOpErr("StoreObject.Compression", op, msgpack.True, msgpack.False) if err != nil {
case msgpack.True: return 0, mdecodeErr("StoreObject.Compression", err)
p.Compression = true
case msgpack.False:
p.Compression = false
} }
if op := msgpack.Op(data[1]); op != msgpack.Bin8 { p.Compression = v
nread += uint64(len(data) - len(tail))
data = tail
}
if len(data) < 22 {
goto overflow
}
if op := msgpack.Op(data[0]); op != msgpack.Bin8 {
return 0, mdecodeOpErr("StoreObject.Checksum", op, msgpack.Bin8) return 0, mdecodeOpErr("StoreObject.Checksum", op, msgpack.Bin8)
} }
if l := data[2]; l != 20 { if l := data[1]; l != 20 {
return 0, mdecodeLen8Err("StoreObject.Checksum", l, 20) return 0, mdecodeLen8Err("StoreObject.Checksum", l, 20)
} }
copy(p.Checksum[:], data[3:23]) copy(p.Checksum[:], data[2:22])
data = data[23:] data = data[22:]
{ {
b, tail, err := msgp.ReadBytesZC(data) b, tail, err := msgp.ReadBytesZC(data)
if err != nil { if err != nil {
...@@ -5370,7 +5384,7 @@ func (p *StoreObject) neoMsgDecodeM(data []byte) (int, error) { ...@@ -5370,7 +5384,7 @@ func (p *StoreObject) neoMsgDecodeM(data []byte) (int, error) {
p.Tid = zodb.Tid(binary.BigEndian.Uint64(data[2:])) p.Tid = zodb.Tid(binary.BigEndian.Uint64(data[2:]))
data = data[10:] data = data[10:]
} }
return 64 + int(nread), nil return 63 + int(nread), nil
overflow: overflow:
return 0, ErrDecodeOverflow return 0, ErrDecodeOverflow
...@@ -6170,19 +6184,19 @@ func (*AnswerObject) neoMsgCode() uint16 { ...@@ -6170,19 +6184,19 @@ func (*AnswerObject) neoMsgCode() uint16 {
} }
func (p *AnswerObject) neoMsgEncodedLenN() int { func (p *AnswerObject) neoMsgEncodedLenN() int {
return 57 + len(p.Data.XData()) return 64 + len(p.Data.XData())
} }
func (p *AnswerObject) neoMsgEncodeN(data []byte) { func (p *AnswerObject) neoMsgEncodeN(data []byte) {
binary.BigEndian.PutUint64(data[0:], uint64(p.Oid)) binary.BigEndian.PutUint64(data[0:], uint64(p.Oid))
binary.BigEndian.PutUint64(data[8:], uint64(p.Serial)) binary.BigEndian.PutUint64(data[8:], uint64(p.Serial))
binary.BigEndian.PutUint64(data[16:], uint64(p.NextSerial)) binary.BigEndian.PutUint64(data[16:], uint64(p.NextSerial))
(data[24:])[0] = bool2byte(p.Compression) binary.BigEndian.PutUint64(data[24:], p.Compression)
copy(data[25:], p.Checksum[:]) copy(data[32:], p.Checksum[:])
{ {
l := uint32(len(p.Data.XData())) l := uint32(len(p.Data.XData()))
binary.BigEndian.PutUint32(data[45:], l) binary.BigEndian.PutUint32(data[52:], l)
data = data[49:] data = data[56:]
copy(data, p.Data.XData()) copy(data, p.Data.XData())
data = data[l:] data = data[l:]
} }
...@@ -6191,17 +6205,17 @@ func (p *AnswerObject) neoMsgEncodeN(data []byte) { ...@@ -6191,17 +6205,17 @@ func (p *AnswerObject) neoMsgEncodeN(data []byte) {
func (p *AnswerObject) neoMsgDecodeN(data []byte) (int, error) { func (p *AnswerObject) neoMsgDecodeN(data []byte) (int, error) {
var nread uint64 var nread uint64
if len(data) < 49 { if len(data) < 56 {
goto overflow goto overflow
} }
p.Oid = zodb.Oid(binary.BigEndian.Uint64(data[0 : 0+8])) p.Oid = zodb.Oid(binary.BigEndian.Uint64(data[0 : 0+8]))
p.Serial = zodb.Tid(binary.BigEndian.Uint64(data[8 : 8+8])) p.Serial = zodb.Tid(binary.BigEndian.Uint64(data[8 : 8+8]))
p.NextSerial = zodb.Tid(binary.BigEndian.Uint64(data[16 : 16+8])) p.NextSerial = zodb.Tid(binary.BigEndian.Uint64(data[16 : 16+8]))
p.Compression = byte2bool((data[24 : 24+1])[0]) p.Compression = binary.BigEndian.Uint64(data[24 : 24+8])
copy(p.Checksum[:], data[25:45]) copy(p.Checksum[:], data[32:52])
{ {
l := binary.BigEndian.Uint32(data[45 : 45+4]) l := binary.BigEndian.Uint32(data[52 : 52+4])
data = data[49:] data = data[56:]
if uint64(len(data)) < 8+uint64(l) { if uint64(len(data)) < 8+uint64(l) {
goto overflow goto overflow
} }
...@@ -6211,7 +6225,7 @@ func (p *AnswerObject) neoMsgDecodeN(data []byte) (int, error) { ...@@ -6211,7 +6225,7 @@ func (p *AnswerObject) neoMsgDecodeN(data []byte) (int, error) {
data = data[l:] data = data[l:]
} }
p.DataSerial = zodb.Tid(binary.BigEndian.Uint64(data[0 : 0+8])) p.DataSerial = zodb.Tid(binary.BigEndian.Uint64(data[0 : 0+8]))
return 49 + int(nread), nil return 56 + int(nread), nil
overflow: overflow:
return 0, ErrDecodeOverflow return 0, ErrDecodeOverflow
...@@ -6239,7 +6253,7 @@ func (p *AnswerObject) neoMsgEncodedLenM() int { ...@@ -6239,7 +6253,7 @@ func (p *AnswerObject) neoMsgEncodedLenM() int {
} else { } else {
size += 1 + 1 + 8 // mbin8 + 8 + [8]data size += 1 + 1 + 8 // mbin8 + 8 + [8]data
} }
return 24 + msgpack.BinHeadSize(len(p.Data.XData())) + len(p.Data.XData()) + size return 23 + msgpack.Uint64Size(p.Compression) + msgpack.BinHeadSize(len(p.Data.XData())) + len(p.Data.XData()) + size
} }
func (p *AnswerObject) neoMsgEncodeM(data []byte) { func (p *AnswerObject) neoMsgEncodeM(data []byte) {
...@@ -6271,14 +6285,17 @@ func (p *AnswerObject) neoMsgEncodeM(data []byte) { ...@@ -6271,14 +6285,17 @@ func (p *AnswerObject) neoMsgEncodeM(data []byte) {
binary.BigEndian.PutUint64(data[2:], uint64(p.NextSerial)) binary.BigEndian.PutUint64(data[2:], uint64(p.NextSerial))
data = data[10:] data = data[10:]
} }
data[0] = byte(msgpack.Bool(p.Compression)) {
data[1] = byte(msgpack.Bin8) n := msgpack.PutUint64(data[0:], p.Compression)
data[2] = 20 data = data[0+n:]
copy(data[3:], p.Checksum[:]) }
data[0] = byte(msgpack.Bin8)
data[1] = 20
copy(data[2:], p.Checksum[:])
{ {
l := len(p.Data.XData()) l := len(p.Data.XData())
n := msgpack.PutBinHead(data[23:], l) n := msgpack.PutBinHead(data[22:], l)
data = data[23+n:] data = data[22+n:]
copy(data, p.Data.XData()) copy(data, p.Data.XData())
data = data[l:] data = data[l:]
} }
...@@ -6295,7 +6312,7 @@ func (p *AnswerObject) neoMsgEncodeM(data []byte) { ...@@ -6295,7 +6312,7 @@ func (p *AnswerObject) neoMsgEncodeM(data []byte) {
func (p *AnswerObject) neoMsgDecodeM(data []byte) (int, error) { func (p *AnswerObject) neoMsgDecodeM(data []byte) (int, error) {
var nread uint64 var nread uint64
if len(data) < 54 { if len(data) < 31 {
goto overflow goto overflow
} }
if op, opOk := msgpack.Op(data[0]), msgpack.FixArray_4|7; op != opOk { if op, opOk := msgpack.Op(data[0]), msgpack.FixArray_4|7; op != opOk {
...@@ -6347,22 +6364,26 @@ func (p *AnswerObject) neoMsgDecodeM(data []byte) (int, error) { ...@@ -6347,22 +6364,26 @@ func (p *AnswerObject) neoMsgDecodeM(data []byte) (int, error) {
p.NextSerial = zodb.Tid(binary.BigEndian.Uint64(data[2:])) p.NextSerial = zodb.Tid(binary.BigEndian.Uint64(data[2:]))
data = data[10:] data = data[10:]
} }
switch op := msgpack.Op(data[0]); op { {
default: v, tail, err := msgp.ReadUint64Bytes(data)
return 0, mdecodeOpErr("AnswerObject.Compression", op, msgpack.True, msgpack.False) if err != nil {
case msgpack.True: return 0, mdecodeErr("AnswerObject.Compression", err)
p.Compression = true
case msgpack.False:
p.Compression = false
} }
if op := msgpack.Op(data[1]); op != msgpack.Bin8 { p.Compression = v
nread += uint64(len(data) - len(tail))
data = tail
}
if len(data) < 22 {
goto overflow
}
if op := msgpack.Op(data[0]); op != msgpack.Bin8 {
return 0, mdecodeOpErr("AnswerObject.Checksum", op, msgpack.Bin8) return 0, mdecodeOpErr("AnswerObject.Checksum", op, msgpack.Bin8)
} }
if l := data[2]; l != 20 { if l := data[1]; l != 20 {
return 0, mdecodeLen8Err("AnswerObject.Checksum", l, 20) return 0, mdecodeLen8Err("AnswerObject.Checksum", l, 20)
} }
copy(p.Checksum[:], data[3:23]) copy(p.Checksum[:], data[2:22])
data = data[23:] data = data[22:]
{ {
b, tail, err := msgp.ReadBytesZC(data) b, tail, err := msgp.ReadBytesZC(data)
if err != nil { if err != nil {
...@@ -6389,7 +6410,7 @@ func (p *AnswerObject) neoMsgDecodeM(data []byte) (int, error) { ...@@ -6389,7 +6410,7 @@ func (p *AnswerObject) neoMsgDecodeM(data []byte) (int, error) {
p.DataSerial = zodb.Tid(binary.BigEndian.Uint64(data[2:])) p.DataSerial = zodb.Tid(binary.BigEndian.Uint64(data[2:]))
data = data[10:] data = data[10:]
} }
return 64 + int(nread), nil return 63 + int(nread), nil
overflow: overflow:
return 0, ErrDecodeOverflow return 0, ErrDecodeOverflow
...@@ -13210,18 +13231,18 @@ func (*AddObject) neoMsgCode() uint16 { ...@@ -13210,18 +13231,18 @@ func (*AddObject) neoMsgCode() uint16 {
} }
func (p *AddObject) neoMsgEncodedLenN() int { func (p *AddObject) neoMsgEncodedLenN() int {
return 49 + len(p.Data.XData()) return 56 + len(p.Data.XData())
} }
func (p *AddObject) neoMsgEncodeN(data []byte) { func (p *AddObject) neoMsgEncodeN(data []byte) {
binary.BigEndian.PutUint64(data[0:], uint64(p.Oid)) binary.BigEndian.PutUint64(data[0:], uint64(p.Oid))
binary.BigEndian.PutUint64(data[8:], uint64(p.Serial)) binary.BigEndian.PutUint64(data[8:], uint64(p.Serial))
(data[16:])[0] = bool2byte(p.Compression) binary.BigEndian.PutUint64(data[16:], p.Compression)
copy(data[17:], p.Checksum[:]) copy(data[24:], p.Checksum[:])
{ {
l := uint32(len(p.Data.XData())) l := uint32(len(p.Data.XData()))
binary.BigEndian.PutUint32(data[37:], l) binary.BigEndian.PutUint32(data[44:], l)
data = data[41:] data = data[48:]
copy(data, p.Data.XData()) copy(data, p.Data.XData())
data = data[l:] data = data[l:]
} }
...@@ -13230,16 +13251,16 @@ func (p *AddObject) neoMsgEncodeN(data []byte) { ...@@ -13230,16 +13251,16 @@ func (p *AddObject) neoMsgEncodeN(data []byte) {
func (p *AddObject) neoMsgDecodeN(data []byte) (int, error) { func (p *AddObject) neoMsgDecodeN(data []byte) (int, error) {
var nread uint64 var nread uint64
if len(data) < 41 { if len(data) < 48 {
goto overflow goto overflow
} }
p.Oid = zodb.Oid(binary.BigEndian.Uint64(data[0 : 0+8])) p.Oid = zodb.Oid(binary.BigEndian.Uint64(data[0 : 0+8]))
p.Serial = zodb.Tid(binary.BigEndian.Uint64(data[8 : 8+8])) p.Serial = zodb.Tid(binary.BigEndian.Uint64(data[8 : 8+8]))
p.Compression = byte2bool((data[16 : 16+1])[0]) p.Compression = binary.BigEndian.Uint64(data[16 : 16+8])
copy(p.Checksum[:], data[17:37]) copy(p.Checksum[:], data[24:44])
{ {
l := binary.BigEndian.Uint32(data[37 : 37+4]) l := binary.BigEndian.Uint32(data[44 : 44+4])
data = data[41:] data = data[48:]
if uint64(len(data)) < 8+uint64(l) { if uint64(len(data)) < 8+uint64(l) {
goto overflow goto overflow
} }
...@@ -13249,7 +13270,7 @@ func (p *AddObject) neoMsgDecodeN(data []byte) (int, error) { ...@@ -13249,7 +13270,7 @@ func (p *AddObject) neoMsgDecodeN(data []byte) (int, error) {
data = data[l:] data = data[l:]
} }
p.DataSerial = zodb.Tid(binary.BigEndian.Uint64(data[0 : 0+8])) p.DataSerial = zodb.Tid(binary.BigEndian.Uint64(data[0 : 0+8]))
return 41 + int(nread), nil return 48 + int(nread), nil
overflow: overflow:
return 0, ErrDecodeOverflow return 0, ErrDecodeOverflow
...@@ -13272,7 +13293,7 @@ func (p *AddObject) neoMsgEncodedLenM() int { ...@@ -13272,7 +13293,7 @@ func (p *AddObject) neoMsgEncodedLenM() int {
} else { } else {
size += 1 + 1 + 8 // mbin8 + 8 + [8]data size += 1 + 1 + 8 // mbin8 + 8 + [8]data
} }
return 24 + msgpack.BinHeadSize(len(p.Data.XData())) + len(p.Data.XData()) + size return 23 + msgpack.Uint64Size(p.Compression) + msgpack.BinHeadSize(len(p.Data.XData())) + len(p.Data.XData()) + size
} }
func (p *AddObject) neoMsgEncodeM(data []byte) { func (p *AddObject) neoMsgEncodeM(data []byte) {
...@@ -13295,14 +13316,17 @@ func (p *AddObject) neoMsgEncodeM(data []byte) { ...@@ -13295,14 +13316,17 @@ func (p *AddObject) neoMsgEncodeM(data []byte) {
binary.BigEndian.PutUint64(data[2:], uint64(p.Serial)) binary.BigEndian.PutUint64(data[2:], uint64(p.Serial))
data = data[10:] data = data[10:]
} }
data[0] = byte(msgpack.Bool(p.Compression)) {
data[1] = byte(msgpack.Bin8) n := msgpack.PutUint64(data[0:], p.Compression)
data[2] = 20 data = data[0+n:]
copy(data[3:], p.Checksum[:]) }
data[0] = byte(msgpack.Bin8)
data[1] = 20
copy(data[2:], p.Checksum[:])
{ {
l := len(p.Data.XData()) l := len(p.Data.XData())
n := msgpack.PutBinHead(data[23:], l) n := msgpack.PutBinHead(data[22:], l)
data = data[23+n:] data = data[22+n:]
copy(data, p.Data.XData()) copy(data, p.Data.XData())
data = data[l:] data = data[l:]
} }
...@@ -13319,7 +13343,7 @@ func (p *AddObject) neoMsgEncodeM(data []byte) { ...@@ -13319,7 +13343,7 @@ func (p *AddObject) neoMsgEncodeM(data []byte) {
func (p *AddObject) neoMsgDecodeM(data []byte) (int, error) { func (p *AddObject) neoMsgDecodeM(data []byte) (int, error) {
var nread uint64 var nread uint64
if len(data) < 44 { if len(data) < 21 {
goto overflow goto overflow
} }
if op, opOk := msgpack.Op(data[0]), msgpack.FixArray_4|6; op != opOk { if op, opOk := msgpack.Op(data[0]), msgpack.FixArray_4|6; op != opOk {
...@@ -13358,22 +13382,26 @@ func (p *AddObject) neoMsgDecodeM(data []byte) (int, error) { ...@@ -13358,22 +13382,26 @@ func (p *AddObject) neoMsgDecodeM(data []byte) (int, error) {
p.Serial = zodb.Tid(binary.BigEndian.Uint64(data[2:])) p.Serial = zodb.Tid(binary.BigEndian.Uint64(data[2:]))
data = data[10:] data = data[10:]
} }
switch op := msgpack.Op(data[0]); op { {
default: v, tail, err := msgp.ReadUint64Bytes(data)
return 0, mdecodeOpErr("AddObject.Compression", op, msgpack.True, msgpack.False) if err != nil {
case msgpack.True: return 0, mdecodeErr("AddObject.Compression", err)
p.Compression = true
case msgpack.False:
p.Compression = false
} }
if op := msgpack.Op(data[1]); op != msgpack.Bin8 { p.Compression = v
nread += uint64(len(data) - len(tail))
data = tail
}
if len(data) < 22 {
goto overflow
}
if op := msgpack.Op(data[0]); op != msgpack.Bin8 {
return 0, mdecodeOpErr("AddObject.Checksum", op, msgpack.Bin8) return 0, mdecodeOpErr("AddObject.Checksum", op, msgpack.Bin8)
} }
if l := data[2]; l != 20 { if l := data[1]; l != 20 {
return 0, mdecodeLen8Err("AddObject.Checksum", l, 20) return 0, mdecodeLen8Err("AddObject.Checksum", l, 20)
} }
copy(p.Checksum[:], data[3:23]) copy(p.Checksum[:], data[2:22])
data = data[23:] data = data[22:]
{ {
b, tail, err := msgp.ReadBytesZC(data) b, tail, err := msgp.ReadBytesZC(data)
if err != nil { if err != nil {
...@@ -13400,7 +13428,7 @@ func (p *AddObject) neoMsgDecodeM(data []byte) (int, error) { ...@@ -13400,7 +13428,7 @@ func (p *AddObject) neoMsgDecodeM(data []byte) (int, error) {
p.DataSerial = zodb.Tid(binary.BigEndian.Uint64(data[2:])) p.DataSerial = zodb.Tid(binary.BigEndian.Uint64(data[2:]))
data = data[10:] data = data[10:]
} }
return 54 + int(nread), nil return 53 + int(nread), nil
overflow: overflow:
return 0, ErrDecodeOverflow return 0, ErrDecodeOverflow
......
go/neo/storage/fs1/fs1.go
View file @ 34c4d915
...@@ -87,7 +87,7 @@ func (f *Backend) Load(ctx context.Context, xid zodb.Xid) (*proto.AnswerObject, ...@@ -87,7 +87,7 @@ func (f *Backend) Load(ctx context.Context, xid zodb.Xid) (*proto.AnswerObject,
Serial: serial, Serial: serial,
NextSerial: nextSerial, NextSerial: nextSerial,
Compression: false, Compression: 0,
Data: buf, Data: buf,
Checksum: xsha1.NEOSum(buf.Data), // XXX computing every time Checksum: xsha1.NEOSum(buf.Data), // XXX computing every time
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7