Commit 7f96ad77 authored by Levin Zimmermann's avatar Levin Zimmermann Committed by Kirill Smelkov

go/neo/proto/RowInfo: Fix representation on the wire

Some NEO protocol packets have the field 'RowList'. This field contains
information about each row of a partition table. In NEO/go the information
of each row is represented with the 'RowInfo' type [1]. This type is defined
as a struct with the field ‘CellList’. ‘CellList’ is defined as a list of
'CellInfo' [1] (e.g. an entry for each cell).

NEO/go {en,de}codes struct types with ‘genStructHead’ (structures in
golang are encoded as arrays in msgpack) [2]. From the 'RowList'
definition, the msgpack decoder currently expects the following msgpack
array structure:

    ArrayHeader (RowList)

        ArrayHeader (RowInfo)

            ArrayHeader (CellList)

                ArrayHeader (CellInfo)

                    int32 (NID)
                    enum (State)

However NEO/py actually sends:

    ArrayHeader (RowList)

        ArrayHeader (CellList)

            ArrayHeader (CellInfo)

                int32 (NID)
                enum (State)

In other words, currently the NEO/go msgpack encoder expects one more
nesting, which NEO/py doesn’t provide (and which also doesn’t seem to
be necessary as the outer nesting would always only contain one
element). We could adjust the msgpack {en,de}coder to introduce an
exception for the 'RowInfo' type, however as the protocol definition in
'proto.go' aims to transparently reflect the structure of the packets on
the wire, it seems to be more appropriate to fix this straight in the
protocol definition. This is also less error-prone as we don't have to
fix all the different positions of the encoder, decoder & sizer and it's
less code (particularly if 'RowInfo' doesn't stay the only case for such
an issue).

[1] https://lab.nexedi.com/kirr/neo/-/blob/1ad088c8/go/neo/proto/proto.go#L391-394
[2] https://lab.nexedi.com/kirr/neo/-/blob/1ad088c8/go/neo/proto/protogen.go#L1770-1775

--------

kirr: I've applied the following interdiff to the original patch of
levin.zimmermann/neoppod@c93d5dbc :

    --- a/go/neo/neo_test.go
    +++ b/go/neo/neo_test.go
    @@ -100,7 +100,7 @@ func _TestMasterStorage(t0 *tEnv) {
                    PTid:           1,
                    NumReplicas:    0,
                    RowList:        []proto.RowInfo{
    -                       proto.RowInfo{proto.CellInfo{proto.NID(proto.STORAGE, 1), proto.UP_TO_DATE}},
    +                       {proto.CellInfo{proto.NID(proto.STORAGE, 1), proto.UP_TO_DATE}},
                    },
            }))

    @@ -173,7 +173,7 @@ func _TestMasterStorage(t0 *tEnv) {
                    PTid:           1,
                    NumReplicas:    0,
                    RowList:        []proto.RowInfo{
    -                       proto.RowInfo{proto.CellInfo{proto.NID(proto.STORAGE, 1), proto.UP_TO_DATE}},
    +                       {proto.CellInfo{proto.NID(proto.STORAGE, 1), proto.UP_TO_DATE}},
                    },
            }))

    --- a/go/neo/proto/proto_test.go
    +++ b/go/neo/proto/proto_test.go
    @@ -210,9 +210,9 @@ func TestMsgMarshal(t *testing.T) {
                            PTid:        0x0102030405060708,
                            NumReplicas: 34,
                            RowList: []RowInfo{
    -                               {CellInfo{11, UP_TO_DATE}, CellInfo{17, OUT_OF_DATE}},
    -                               {CellInfo{11, FEEDING}},
    -                               {CellInfo{11, CORRUPTED}, CellInfo{15, DISCARDED}, CellInfo{23, UP_TO_DATE}},
    +                               {{11, UP_TO_DATE}, {17, OUT_OF_DATE}},
    +                               {{11, FEEDING}},
    +                               {{11, CORRUPTED}, {15, DISCARDED}, {23, UP_TO_DATE}},
                            },
                            },

    @@ -229,9 +229,9 @@ func TestMsgMarshal(t *testing.T) {
                            hex("cf0102030405060708") +
                            hex("22") +
                            hex("93") +
    -                               hex("92"+"920bd40001"+"9211d40000") +
    -                               hex("91"+"920bd40002") +
    -                               hex("93"+"920bd40003"+"920fd40004"+"9217d40001"),
    +                               hex("92"+"920bd40401"+"9211d40400") +
    +                               hex("91"+"920bd40402") +
    +                               hex("93"+"920bd40403"+"920fd40404"+"9217d40401"),
                    },

                    // map[Oid]struct {Tid,Tid,bool}

for cosmetics and because the tests were failing as

    --- FAIL: TestMsgMarshal (0.00s)
        proto_test.go:106: M/proto.AnswerPartitionTable: encode result unexpected:
        proto_test.go:107:  have: 93cf0102030405060708229392920bd404019211d4040091920bd4040293920bd40403920fd404049217d40401
        proto_test.go:108:  want: 93cf0102030405060708229392920bd400019211d4000091920bd4000293920bd40003920fd400049217d40001

/reviewed-by @kirr
/reviewed-on !6
parent d75f4ac2
......@@ -385,9 +385,7 @@ type CellInfo struct {
}
//neo:proto typeonly
type RowInfo struct {
CellList []CellInfo
}
type RowInfo []CellInfo
......
......@@ -210,9 +210,9 @@ func TestMsgMarshal(t *testing.T) {
PTid: 0x0102030405060708,
NumReplicas: 34,
RowList: []RowInfo{
{[]CellInfo{{11, UP_TO_DATE}, {17, OUT_OF_DATE}}},
{[]CellInfo{{11, FEEDING}}},
{[]CellInfo{{11, CORRUPTED}, {15, DISCARDED}, {23, UP_TO_DATE}}},
{{11, UP_TO_DATE}, {17, OUT_OF_DATE}},
{{11, FEEDING}},
{{11, CORRUPTED}, {15, DISCARDED}, {23, UP_TO_DATE}},
},
},
......@@ -229,9 +229,9 @@ func TestMsgMarshal(t *testing.T) {
hex("cf0102030405060708") +
hex("22") +
hex("93") +
hex("91"+"92"+"920bd40401"+"9211d40400") +
hex("91"+"91"+"920bd40402") +
hex("91"+"93"+"920bd40403"+"920fd40404"+"9217d40401"),
hex("92"+"920bd40401"+"9211d40400") +
hex("91"+"920bd40402") +
hex("93"+"920bd40403"+"920fd40404"+"9217d40401"),
},
// map[Oid]struct {Tid,Tid,bool}
......
......@@ -1477,7 +1477,7 @@ func (p *AnswerPartitionTable) neoMsgEncodedLenN() int {
var size int
for i := 0; i < len(p.RowList); i++ {
a := &p.RowList[i]
size += len((*a).CellList) * 5
size += len((*a)) * 5
}
return 16 + len(p.RowList)*4 + size
}
......@@ -1492,11 +1492,11 @@ func (p *AnswerPartitionTable) neoMsgEncodeN(data []byte) {
for i := 0; i < l; i++ {
a := &p.RowList[i]
{
l := len((*a).CellList)
l := len((*a))
binary.BigEndian.PutUint32(data[0:], uint32(l))
data = data[4:]
for i := 0; i < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
binary.BigEndian.PutUint32(data[0:], uint32(int32((*a).NID)))
(data[4:])[0] = uint8(int8((*a).State))
data = data[5:]
......@@ -1529,9 +1529,9 @@ func (p *AnswerPartitionTable) neoMsgDecodeN(data []byte) (int, error) {
goto overflow
}
nread += uint64(l) * 5
(*a).CellList = make([]CellInfo, l)
(*a) = make([]CellInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
(*a).NID = NodeID(int32(binary.BigEndian.Uint32(data[0 : 0+4])))
(*a).State = CellState(int8((data[4 : 4+1])[0]))
data = data[5:]
......@@ -1550,13 +1550,13 @@ func (p *AnswerPartitionTable) neoMsgEncodedLenM() int {
var size int
for i := 0; i < len(p.RowList); i++ {
a := &p.RowList[i]
for i := 0; i < len((*a).CellList); i++ {
a := &(*a).CellList[i]
for i := 0; i < len((*a)); i++ {
a := &(*a)[i]
size += msgpack.Int32Size(int32((*a).NID))
}
size += msgpack.ArrayHeadSize(len((*a).CellList)) + len((*a).CellList)*4
size += msgpack.ArrayHeadSize(len((*a))) + len((*a))*4
}
return 1 + msgpack.Uint64Size(uint64(p.PTid)) + msgpack.Uint32Size(p.NumReplicas) + msgpack.ArrayHeadSize(len(p.RowList)) + len(p.RowList)*1 + size
return 1 + msgpack.Uint64Size(uint64(p.PTid)) + msgpack.Uint32Size(p.NumReplicas) + msgpack.ArrayHeadSize(len(p.RowList)) + size
}
func (p *AnswerPartitionTable) neoMsgEncodeM(data []byte) {
......@@ -1575,13 +1575,12 @@ func (p *AnswerPartitionTable) neoMsgEncodeM(data []byte) {
data = data[0+n:]
for i := 0; i < l; i++ {
a := &p.RowList[i]
data[0] = byte(msgpack.FixArray_4 | 1)
{
l := len((*a).CellList)
n := msgpack.PutArrayHead(data[1:], l)
data = data[1+n:]
l := len((*a))
n := msgpack.PutArrayHead(data[0:], l)
data = data[0+n:]
for i := 0; i < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
data[0] = byte(msgpack.FixArray_4 | 2)
{
n := msgpack.PutInt32(data[1:], int32((*a).NID))
......@@ -1637,23 +1636,16 @@ func (p *AnswerPartitionTable) neoMsgDecodeM(data []byte) (int, error) {
p.RowList = make([]RowInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &p.RowList[i]
if len(data) < 1 {
goto overflow
}
if op, opOk := msgpack.Op(data[0]), msgpack.FixArray_4|1; op != opOk {
return 0, &mstructDecodeError{"AnswerPartitionTable", op, opOk}
}
{
data = data[1:]
l, tail, err := msgp.ReadArrayHeaderBytes(data)
if err != nil {
return 0, mdecodeErr("AnswerPartitionTable.CellList", err)
return 0, mdecodeErr("AnswerPartitionTable", err)
}
nread += uint64(len(data) - len(tail))
data = tail
(*a).CellList = make([]CellInfo, l)
(*a) = make([]CellInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
if len(data) < 1 {
goto overflow
}
......@@ -1691,7 +1683,6 @@ func (p *AnswerPartitionTable) neoMsgDecodeM(data []byte) (int, error) {
nread += uint64(l) * 4
}
}
nread += uint64(l) * 1
}
return 1 + int(nread), nil
......@@ -1709,7 +1700,7 @@ func (p *SendPartitionTable) neoMsgEncodedLenN() int {
var size int
for i := 0; i < len(p.RowList); i++ {
a := &p.RowList[i]
size += len((*a).CellList) * 5
size += len((*a)) * 5
}
return 16 + len(p.RowList)*4 + size
}
......@@ -1724,11 +1715,11 @@ func (p *SendPartitionTable) neoMsgEncodeN(data []byte) {
for i := 0; i < l; i++ {
a := &p.RowList[i]
{
l := len((*a).CellList)
l := len((*a))
binary.BigEndian.PutUint32(data[0:], uint32(l))
data = data[4:]
for i := 0; i < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
binary.BigEndian.PutUint32(data[0:], uint32(int32((*a).NID)))
(data[4:])[0] = uint8(int8((*a).State))
data = data[5:]
......@@ -1761,9 +1752,9 @@ func (p *SendPartitionTable) neoMsgDecodeN(data []byte) (int, error) {
goto overflow
}
nread += uint64(l) * 5
(*a).CellList = make([]CellInfo, l)
(*a) = make([]CellInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
(*a).NID = NodeID(int32(binary.BigEndian.Uint32(data[0 : 0+4])))
(*a).State = CellState(int8((data[4 : 4+1])[0]))
data = data[5:]
......@@ -1782,13 +1773,13 @@ func (p *SendPartitionTable) neoMsgEncodedLenM() int {
var size int
for i := 0; i < len(p.RowList); i++ {
a := &p.RowList[i]
for i := 0; i < len((*a).CellList); i++ {
a := &(*a).CellList[i]
for i := 0; i < len((*a)); i++ {
a := &(*a)[i]
size += msgpack.Int32Size(int32((*a).NID))
}
size += msgpack.ArrayHeadSize(len((*a).CellList)) + len((*a).CellList)*4
size += msgpack.ArrayHeadSize(len((*a))) + len((*a))*4
}
return 1 + msgpack.Uint64Size(uint64(p.PTid)) + msgpack.Uint32Size(p.NumReplicas) + msgpack.ArrayHeadSize(len(p.RowList)) + len(p.RowList)*1 + size
return 1 + msgpack.Uint64Size(uint64(p.PTid)) + msgpack.Uint32Size(p.NumReplicas) + msgpack.ArrayHeadSize(len(p.RowList)) + size
}
func (p *SendPartitionTable) neoMsgEncodeM(data []byte) {
......@@ -1807,13 +1798,12 @@ func (p *SendPartitionTable) neoMsgEncodeM(data []byte) {
data = data[0+n:]
for i := 0; i < l; i++ {
a := &p.RowList[i]
data[0] = byte(msgpack.FixArray_4 | 1)
{
l := len((*a).CellList)
n := msgpack.PutArrayHead(data[1:], l)
data = data[1+n:]
l := len((*a))
n := msgpack.PutArrayHead(data[0:], l)
data = data[0+n:]
for i := 0; i < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
data[0] = byte(msgpack.FixArray_4 | 2)
{
n := msgpack.PutInt32(data[1:], int32((*a).NID))
......@@ -1869,23 +1859,16 @@ func (p *SendPartitionTable) neoMsgDecodeM(data []byte) (int, error) {
p.RowList = make([]RowInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &p.RowList[i]
if len(data) < 1 {
goto overflow
}
if op, opOk := msgpack.Op(data[0]), msgpack.FixArray_4|1; op != opOk {
return 0, &mstructDecodeError{"SendPartitionTable", op, opOk}
}
{
data = data[1:]
l, tail, err := msgp.ReadArrayHeaderBytes(data)
if err != nil {
return 0, mdecodeErr("SendPartitionTable.CellList", err)
return 0, mdecodeErr("SendPartitionTable", err)
}
nread += uint64(len(data) - len(tail))
data = tail
(*a).CellList = make([]CellInfo, l)
(*a) = make([]CellInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
if len(data) < 1 {
goto overflow
}
......@@ -1923,7 +1906,6 @@ func (p *SendPartitionTable) neoMsgDecodeM(data []byte) (int, error) {
nread += uint64(l) * 4
}
}
nread += uint64(l) * 1
}
return 1 + int(nread), nil
......@@ -5945,7 +5927,7 @@ func (p *AnswerPartitionList) neoMsgEncodedLenN() int {
var size int
for i := 0; i < len(p.RowList); i++ {
a := &p.RowList[i]
size += len((*a).CellList) * 5
size += len((*a)) * 5
}
return 16 + len(p.RowList)*4 + size
}
......@@ -5960,11 +5942,11 @@ func (p *AnswerPartitionList) neoMsgEncodeN(data []byte) {
for i := 0; i < l; i++ {
a := &p.RowList[i]
{
l := len((*a).CellList)
l := len((*a))
binary.BigEndian.PutUint32(data[0:], uint32(l))
data = data[4:]
for i := 0; i < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
binary.BigEndian.PutUint32(data[0:], uint32(int32((*a).NID)))
(data[4:])[0] = uint8(int8((*a).State))
data = data[5:]
......@@ -5997,9 +5979,9 @@ func (p *AnswerPartitionList) neoMsgDecodeN(data []byte) (int, error) {
goto overflow
}
nread += uint64(l) * 5
(*a).CellList = make([]CellInfo, l)
(*a) = make([]CellInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
(*a).NID = NodeID(int32(binary.BigEndian.Uint32(data[0 : 0+4])))
(*a).State = CellState(int8((data[4 : 4+1])[0]))
data = data[5:]
......@@ -6018,13 +6000,13 @@ func (p *AnswerPartitionList) neoMsgEncodedLenM() int {
var size int
for i := 0; i < len(p.RowList); i++ {
a := &p.RowList[i]
for i := 0; i < len((*a).CellList); i++ {
a := &(*a).CellList[i]
for i := 0; i < len((*a)); i++ {
a := &(*a)[i]
size += msgpack.Int32Size(int32((*a).NID))
}
size += msgpack.ArrayHeadSize(len((*a).CellList)) + len((*a).CellList)*4
size += msgpack.ArrayHeadSize(len((*a))) + len((*a))*4
}
return 1 + msgpack.Uint64Size(uint64(p.PTid)) + msgpack.Uint32Size(p.NumReplicas) + msgpack.ArrayHeadSize(len(p.RowList)) + len(p.RowList)*1 + size
return 1 + msgpack.Uint64Size(uint64(p.PTid)) + msgpack.Uint32Size(p.NumReplicas) + msgpack.ArrayHeadSize(len(p.RowList)) + size
}
func (p *AnswerPartitionList) neoMsgEncodeM(data []byte) {
......@@ -6043,13 +6025,12 @@ func (p *AnswerPartitionList) neoMsgEncodeM(data []byte) {
data = data[0+n:]
for i := 0; i < l; i++ {
a := &p.RowList[i]
data[0] = byte(msgpack.FixArray_4 | 1)
{
l := len((*a).CellList)
n := msgpack.PutArrayHead(data[1:], l)
data = data[1+n:]
l := len((*a))
n := msgpack.PutArrayHead(data[0:], l)
data = data[0+n:]
for i := 0; i < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
data[0] = byte(msgpack.FixArray_4 | 2)
{
n := msgpack.PutInt32(data[1:], int32((*a).NID))
......@@ -6105,23 +6086,16 @@ func (p *AnswerPartitionList) neoMsgDecodeM(data []byte) (int, error) {
p.RowList = make([]RowInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &p.RowList[i]
if len(data) < 1 {
goto overflow
}
if op, opOk := msgpack.Op(data[0]), msgpack.FixArray_4|1; op != opOk {
return 0, &mstructDecodeError{"AnswerPartitionList", op, opOk}
}
{
data = data[1:]
l, tail, err := msgp.ReadArrayHeaderBytes(data)
if err != nil {
return 0, mdecodeErr("AnswerPartitionList.CellList", err)
return 0, mdecodeErr("AnswerPartitionList", err)
}
nread += uint64(len(data) - len(tail))
data = tail
(*a).CellList = make([]CellInfo, l)
(*a) = make([]CellInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
if len(data) < 1 {
goto overflow
}
......@@ -6159,7 +6133,6 @@ func (p *AnswerPartitionList) neoMsgDecodeM(data []byte) (int, error) {
nread += uint64(l) * 4
}
}
nread += uint64(l) * 1
}
return 1 + int(nread), nil
......@@ -6797,7 +6770,7 @@ func (p *AnswerTweakPartitionTable) neoMsgEncodedLenN() int {
var size int
for i := 0; i < len(p.RowList); i++ {
a := &p.RowList[i]
size += len((*a).CellList) * 5
size += len((*a)) * 5
}
return 5 + len(p.RowList)*4 + size
}
......@@ -6811,11 +6784,11 @@ func (p *AnswerTweakPartitionTable) neoMsgEncodeN(data []byte) {
for i := 0; i < l; i++ {
a := &p.RowList[i]
{
l := len((*a).CellList)
l := len((*a))
binary.BigEndian.PutUint32(data[0:], uint32(l))
data = data[4:]
for i := 0; i < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
binary.BigEndian.PutUint32(data[0:], uint32(int32((*a).NID)))
(data[4:])[0] = uint8(int8((*a).State))
data = data[5:]
......@@ -6847,9 +6820,9 @@ func (p *AnswerTweakPartitionTable) neoMsgDecodeN(data []byte) (int, error) {
goto overflow
}
nread += uint64(l) * 5
(*a).CellList = make([]CellInfo, l)
(*a) = make([]CellInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
(*a).NID = NodeID(int32(binary.BigEndian.Uint32(data[0 : 0+4])))
(*a).State = CellState(int8((data[4 : 4+1])[0]))
data = data[5:]
......@@ -6868,13 +6841,13 @@ func (p *AnswerTweakPartitionTable) neoMsgEncodedLenM() int {
var size int
for i := 0; i < len(p.RowList); i++ {
a := &p.RowList[i]
for i := 0; i < len((*a).CellList); i++ {
a := &(*a).CellList[i]
for i := 0; i < len((*a)); i++ {
a := &(*a)[i]
size += msgpack.Int32Size(int32((*a).NID))
}
size += msgpack.ArrayHeadSize(len((*a).CellList)) + len((*a).CellList)*4
size += msgpack.ArrayHeadSize(len((*a))) + len((*a))*4
}
return 2 + msgpack.ArrayHeadSize(len(p.RowList)) + len(p.RowList)*1 + size
return 2 + msgpack.ArrayHeadSize(len(p.RowList)) + size
}
func (p *AnswerTweakPartitionTable) neoMsgEncodeM(data []byte) {
......@@ -6886,13 +6859,12 @@ func (p *AnswerTweakPartitionTable) neoMsgEncodeM(data []byte) {
data = data[2+n:]
for i := 0; i < l; i++ {
a := &p.RowList[i]
data[0] = byte(msgpack.FixArray_4 | 1)
{
l := len((*a).CellList)
n := msgpack.PutArrayHead(data[1:], l)
data = data[1+n:]
l := len((*a))
n := msgpack.PutArrayHead(data[0:], l)
data = data[0+n:]
for i := 0; i < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
data[0] = byte(msgpack.FixArray_4 | 2)
{
n := msgpack.PutInt32(data[1:], int32((*a).NID))
......@@ -6938,23 +6910,16 @@ func (p *AnswerTweakPartitionTable) neoMsgDecodeM(data []byte) (int, error) {
p.RowList = make([]RowInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &p.RowList[i]
if len(data) < 1 {
goto overflow
}
if op, opOk := msgpack.Op(data[0]), msgpack.FixArray_4|1; op != opOk {
return 0, &mstructDecodeError{"AnswerTweakPartitionTable", op, opOk}
}
{
data = data[1:]
l, tail, err := msgp.ReadArrayHeaderBytes(data)
if err != nil {
return 0, mdecodeErr("AnswerTweakPartitionTable.CellList", err)
return 0, mdecodeErr("AnswerTweakPartitionTable", err)
}
nread += uint64(len(data) - len(tail))
data = tail
(*a).CellList = make([]CellInfo, l)
(*a) = make([]CellInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
if len(data) < 1 {
goto overflow
}
......@@ -6992,7 +6957,6 @@ func (p *AnswerTweakPartitionTable) neoMsgDecodeM(data []byte) (int, error) {
nread += uint64(l) * 4
}
}
nread += uint64(l) * 1
}
return 2 + int(nread), nil
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment