Skip to content

N
neoppod
Commit 6f02bc97 authored by Levin Zimmermann's avatar Levin Zimmermann
Browse files
Options

go/neo/proto/RowInfo: Fix representation on the wire 

Some NEO protocol packets have the field 'RowList'. This field contains
information about each row of a partition table. In NEO/go the information
of each row is represented with the 'RowInfo' type [1]. This type is defined
as a struct with the field ‘CellList’. ‘CellList’ is defined as a list of
'CellInfo' [1] (e.g. an entry for each cell).

NEO/go {en,de}codes struct types with ‘genStructHead’ (structures in
golang are encoded as arrays in msgpack) [2]. From the 'RowList'
definition, the msgpack decoder currently expects the following msgpack
array structure:

    ArrayHeader (RowList)

        ArrayHeader (RowInfo)

            ArrayHeader (CellList)

                ArrayHeader (CellInfo)

                    int32 (NID)
                    enum (State)

However NEO/py actually sends:

    ArrayHeader (RowList)

        ArrayHeader (CellList)

            ArrayHeader (CellInfo)

                int32 (NID)
                enum (State)

In other words, currently the NEO/go msgpack encoder expects one more
nesting, which NEO/py doesn’t provide (and which also doesn’t seem to
be necessary as the outer nesting would always only contain one
element). We could adjust the msgpack {en,de}coder to introduce an
exception for the 'RowInfo' type, however as the protocol definition in
'proto.go' aims to transparently reflect the structure of the packets on
the wire, it seems to be more appropriate to fix this straight in the
protocol definition. This is also less error-prone as we don't have to
fix all the different positions of the encoder, decoder & sizer and it's
less code (particularly if 'RowInfo' doesn't stay the only case for such
an issue).

[1] https://lab.nexedi.com/kirr/neo/-/blob/1ad088c8/go/neo/proto/proto.go#L391-394
[2] https://lab.nexedi.com/kirr/neo/-/blob/1ad088c8/go/neo/proto/protogen.go#L1770-1775
parent c5b08706
Hide whitespace changes
Inline Side-by-side
Showing with 71 additions and 109 deletions
go/neo/neo_test.go
View file @ 6f02bc97
......@@ -100,7 +100,7 @@ func _TestMasterStorage(t0 *tEnv) {
PTid: 1,
NumReplicas: 0,
RowList: []proto.RowInfo{
{[]proto.CellInfo{{proto.NID(proto.STORAGE, 1), proto.UP_TO_DATE}}},
proto.RowInfo{proto.CellInfo{proto.NID(proto.STORAGE, 1), proto.UP_TO_DATE}},
},
}))
......@@ -173,7 +173,7 @@ func _TestMasterStorage(t0 *tEnv) {
PTid: 1,
NumReplicas: 0,
RowList: []proto.RowInfo{
{[]proto.CellInfo{{proto.NID(proto.STORAGE, 1), proto.UP_TO_DATE}}},
proto.RowInfo{proto.CellInfo{proto.NID(proto.STORAGE, 1), proto.UP_TO_DATE}},
},
}))
......
go/neo/proto/proto.go
View file @ 6f02bc97
......@@ -389,9 +389,7 @@ type CellInfo struct {
}
//neo:proto typeonly
type RowInfo struct {
CellList []CellInfo
}
type RowInfo []CellInfo
......
go/neo/proto/proto_test.go
View file @ 6f02bc97
......@@ -210,9 +210,9 @@ func TestMsgMarshal(t *testing.T) {
PTid: 0x0102030405060708,
NumReplicas: 34,
RowList: []RowInfo{
{[]CellInfo{{11, UP_TO_DATE}, {17, OUT_OF_DATE}}},
{[]CellInfo{{11, FEEDING}}},
{[]CellInfo{{11, CORRUPTED}, {15, DISCARDED}, {23, UP_TO_DATE}}},
{CellInfo{11, UP_TO_DATE}, CellInfo{17, OUT_OF_DATE}},
{CellInfo{11, FEEDING}},
{CellInfo{11, CORRUPTED}, CellInfo{15, DISCARDED}, CellInfo{23, UP_TO_DATE}},
},
},
......@@ -229,9 +229,9 @@ func TestMsgMarshal(t *testing.T) {
hex("cf0102030405060708") +
hex("22") +
hex("93") +
hex("91"+"92"+"920bd40001"+"9211d40000") +
hex("91"+"91"+"920bd40002") +
hex("91"+"93"+"920bd40003"+"920fd40004"+"9217d40001"),
hex("92"+"920bd40001"+"9211d40000") +
hex("91"+"920bd40002") +
hex("93"+"920bd40003"+"920fd40004"+"9217d40001"),
},
// map[Oid]struct {Tid,Tid,bool}
......
go/neo/proto/zproto-marshal.go
View file @ 6f02bc97
......@@ -1537,7 +1537,7 @@ func (p *AnswerPartitionTable) neoMsgEncodedLenN() int {
var size int
for i := 0; i < len(p.RowList); i++ {
a := &p.RowList[i]
size += len((*a).CellList) * 5
size += len((*a)) * 5
}
return 16 + len(p.RowList)*4 + size
}
......@@ -1552,11 +1552,11 @@ func (p *AnswerPartitionTable) neoMsgEncodeN(data []byte) {
for i := 0; i < l; i++ {
a := &p.RowList[i]
{
l := len((*a).CellList)
l := len((*a))
binary.BigEndian.PutUint32(data[0:], uint32(l))
data = data[4:]
for i := 0; i < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
binary.BigEndian.PutUint32(data[0:], uint32(int32((*a).NID)))
(data[4:])[0] = uint8(int8((*a).State))
data = data[5:]
......@@ -1589,9 +1589,9 @@ func (p *AnswerPartitionTable) neoMsgDecodeN(data []byte) (int, error) {
goto overflow
}
nread += uint64(l) * 5
(*a).CellList = make([]CellInfo, l)
(*a) = make([]CellInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
(*a).NID = NodeID(int32(binary.BigEndian.Uint32(data[0 : 0+4])))
(*a).State = CellState(int8((data[4 : 4+1])[0]))
data = data[5:]
......@@ -1610,11 +1610,11 @@ func (p *AnswerPartitionTable) neoMsgEncodedLenM() int {
var size int
for i := 0; i < len(p.RowList); i++ {
a := &p.RowList[i]
for i := 0; i < len((*a).CellList); i++ {
a := &(*a).CellList[i]
for i := 0; i < len((*a)); i++ {
a := &(*a)[i]
size += msgpack.ArrayHeadSize(reflect.TypeOf((*a)).NumField()) + msgpack.Int32Size(int32((*a).NID))
}
size += msgpack.ArrayHeadSize(reflect.TypeOf((*a)).NumField()) + msgpack.ArrayHeadSize(len((*a).CellList)) + len((*a).CellList)*3
size += msgpack.ArrayHeadSize(len((*a))) + len((*a))*3
}
return msgpack.ArrayHeadSize(reflect.TypeOf((*p)).NumField()) + msgpack.Uint64Size(uint64(p.PTid)) + msgpack.Uint32Size(p.NumReplicas) + msgpack.ArrayHeadSize(len(p.RowList)) + size
}
......@@ -1635,13 +1635,12 @@ func (p *AnswerPartitionTable) neoMsgEncodeM(data []byte) {
data = data[0+n:]
for i := 0; i < l; i++ {
a := &p.RowList[i]
data[0] = byte(msgpack.FixArray_4 | 1)
{
l := len((*a).CellList)
n := msgpack.PutArrayHead(data[1:], l)
data = data[1+n:]
l := len((*a))
n := msgpack.PutArrayHead(data[0:], l)
data = data[0+n:]
for i := 0; i < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
data[0] = byte(msgpack.FixArray_4 | 2)
{
n := msgpack.PutInt32(data[1:], int32((*a).NID))
......@@ -1698,24 +1697,16 @@ func (p *AnswerPartitionTable) neoMsgDecodeM(data []byte) (int, error) {
p.RowList = make([]RowInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &p.RowList[i]
{
_, tail, err := msgp.ReadArrayHeaderBytes(data)
if err != nil {
return 0, mdecodeErr("AnswerPartitionTable", err)
}
nread += uint64(len(data) - len(tail))
data = tail
}
{
l, tail, err := msgp.ReadArrayHeaderBytes(data)
if err != nil {
return 0, mdecodeErr("AnswerPartitionTable.CellList", err)
return 0, mdecodeErr("AnswerPartitionTable", err)
}
nread += uint64(len(data) - len(tail))
data = tail
(*a).CellList = make([]CellInfo, l)
(*a) = make([]CellInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
{
_, tail, err := msgp.ReadArrayHeaderBytes(data)
if err != nil {
......@@ -1771,7 +1762,7 @@ func (p *SendPartitionTable) neoMsgEncodedLenN() int {
var size int
for i := 0; i < len(p.RowList); i++ {
a := &p.RowList[i]
size += len((*a).CellList) * 5
size += len((*a)) * 5
}
return 16 + len(p.RowList)*4 + size
}
......@@ -1786,11 +1777,11 @@ func (p *SendPartitionTable) neoMsgEncodeN(data []byte) {
for i := 0; i < l; i++ {
a := &p.RowList[i]
{
l := len((*a).CellList)
l := len((*a))
binary.BigEndian.PutUint32(data[0:], uint32(l))
data = data[4:]
for i := 0; i < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
binary.BigEndian.PutUint32(data[0:], uint32(int32((*a).NID)))
(data[4:])[0] = uint8(int8((*a).State))
data = data[5:]
......@@ -1823,9 +1814,9 @@ func (p *SendPartitionTable) neoMsgDecodeN(data []byte) (int, error) {
goto overflow
}
nread += uint64(l) * 5
(*a).CellList = make([]CellInfo, l)
(*a) = make([]CellInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
(*a).NID = NodeID(int32(binary.BigEndian.Uint32(data[0 : 0+4])))
(*a).State = CellState(int8((data[4 : 4+1])[0]))
data = data[5:]
......@@ -1844,11 +1835,11 @@ func (p *SendPartitionTable) neoMsgEncodedLenM() int {
var size int
for i := 0; i < len(p.RowList); i++ {
a := &p.RowList[i]
for i := 0; i < len((*a).CellList); i++ {
a := &(*a).CellList[i]
for i := 0; i < len((*a)); i++ {
a := &(*a)[i]
size += msgpack.ArrayHeadSize(reflect.TypeOf((*a)).NumField()) + msgpack.Int32Size(int32((*a).NID))
}
size += msgpack.ArrayHeadSize(reflect.TypeOf((*a)).NumField()) + msgpack.ArrayHeadSize(len((*a).CellList)) + len((*a).CellList)*3
size += msgpack.ArrayHeadSize(len((*a))) + len((*a))*3
}
return msgpack.ArrayHeadSize(reflect.TypeOf((*p)).NumField()) + msgpack.Uint64Size(uint64(p.PTid)) + msgpack.Uint32Size(p.NumReplicas) + msgpack.ArrayHeadSize(len(p.RowList)) + size
}
......@@ -1869,13 +1860,12 @@ func (p *SendPartitionTable) neoMsgEncodeM(data []byte) {
data = data[0+n:]
for i := 0; i < l; i++ {
a := &p.RowList[i]
data[0] = byte(msgpack.FixArray_4 | 1)
{
l := len((*a).CellList)
n := msgpack.PutArrayHead(data[1:], l)
data = data[1+n:]
l := len((*a))
n := msgpack.PutArrayHead(data[0:], l)
data = data[0+n:]
for i := 0; i < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
data[0] = byte(msgpack.FixArray_4 | 2)
{
n := msgpack.PutInt32(data[1:], int32((*a).NID))
......@@ -1932,24 +1922,16 @@ func (p *SendPartitionTable) neoMsgDecodeM(data []byte) (int, error) {
p.RowList = make([]RowInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &p.RowList[i]
{
_, tail, err := msgp.ReadArrayHeaderBytes(data)
if err != nil {
return 0, mdecodeErr("SendPartitionTable", err)
}
nread += uint64(len(data) - len(tail))
data = tail
}
{
l, tail, err := msgp.ReadArrayHeaderBytes(data)
if err != nil {
return 0, mdecodeErr("SendPartitionTable.CellList", err)
return 0, mdecodeErr("SendPartitionTable", err)
}
nread += uint64(len(data) - len(tail))
data = tail
(*a).CellList = make([]CellInfo, l)
(*a) = make([]CellInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
{
_, tail, err := msgp.ReadArrayHeaderBytes(data)
if err != nil {
......@@ -6185,7 +6167,7 @@ func (p *AnswerPartitionList) neoMsgEncodedLenN() int {
var size int
for i := 0; i < len(p.RowList); i++ {
a := &p.RowList[i]
size += len((*a).CellList) * 5
size += len((*a)) * 5
}
return 16 + len(p.RowList)*4 + size
}
......@@ -6200,11 +6182,11 @@ func (p *AnswerPartitionList) neoMsgEncodeN(data []byte) {
for i := 0; i < l; i++ {
a := &p.RowList[i]
{
l := len((*a).CellList)
l := len((*a))
binary.BigEndian.PutUint32(data[0:], uint32(l))
data = data[4:]
for i := 0; i < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
binary.BigEndian.PutUint32(data[0:], uint32(int32((*a).NID)))
(data[4:])[0] = uint8(int8((*a).State))
data = data[5:]
......@@ -6237,9 +6219,9 @@ func (p *AnswerPartitionList) neoMsgDecodeN(data []byte) (int, error) {
goto overflow
}
nread += uint64(l) * 5
(*a).CellList = make([]CellInfo, l)
(*a) = make([]CellInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
(*a).NID = NodeID(int32(binary.BigEndian.Uint32(data[0 : 0+4])))
(*a).State = CellState(int8((data[4 : 4+1])[0]))
data = data[5:]
......@@ -6258,11 +6240,11 @@ func (p *AnswerPartitionList) neoMsgEncodedLenM() int {
var size int
for i := 0; i < len(p.RowList); i++ {
a := &p.RowList[i]
for i := 0; i < len((*a).CellList); i++ {
a := &(*a).CellList[i]
for i := 0; i < len((*a)); i++ {
a := &(*a)[i]
size += msgpack.ArrayHeadSize(reflect.TypeOf((*a)).NumField()) + msgpack.Int32Size(int32((*a).NID))
}
size += msgpack.ArrayHeadSize(reflect.TypeOf((*a)).NumField()) + msgpack.ArrayHeadSize(len((*a).CellList)) + len((*a).CellList)*3
size += msgpack.ArrayHeadSize(len((*a))) + len((*a))*3
}
return msgpack.ArrayHeadSize(reflect.TypeOf((*p)).NumField()) + msgpack.Uint64Size(uint64(p.PTid)) + msgpack.Uint32Size(p.NumReplicas) + msgpack.ArrayHeadSize(len(p.RowList)) + size
}
......@@ -6283,13 +6265,12 @@ func (p *AnswerPartitionList) neoMsgEncodeM(data []byte) {
data = data[0+n:]
for i := 0; i < l; i++ {
a := &p.RowList[i]
data[0] = byte(msgpack.FixArray_4 | 1)
{
l := len((*a).CellList)
n := msgpack.PutArrayHead(data[1:], l)
data = data[1+n:]
l := len((*a))
n := msgpack.PutArrayHead(data[0:], l)
data = data[0+n:]
for i := 0; i < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
data[0] = byte(msgpack.FixArray_4 | 2)
{
n := msgpack.PutInt32(data[1:], int32((*a).NID))
......@@ -6346,24 +6327,16 @@ func (p *AnswerPartitionList) neoMsgDecodeM(data []byte) (int, error) {
p.RowList = make([]RowInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &p.RowList[i]
{
_, tail, err := msgp.ReadArrayHeaderBytes(data)
if err != nil {
return 0, mdecodeErr("AnswerPartitionList", err)
}
nread += uint64(len(data) - len(tail))
data = tail
}
{
l, tail, err := msgp.ReadArrayHeaderBytes(data)
if err != nil {
return 0, mdecodeErr("AnswerPartitionList.CellList", err)
return 0, mdecodeErr("AnswerPartitionList", err)
}
nread += uint64(len(data) - len(tail))
data = tail
(*a).CellList = make([]CellInfo, l)
(*a) = make([]CellInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
{
_, tail, err := msgp.ReadArrayHeaderBytes(data)
if err != nil {
......@@ -7068,7 +7041,7 @@ func (p *AnswerTweakPartitionTable) neoMsgEncodedLenN() int {
var size int
for i := 0; i < len(p.RowList); i++ {
a := &p.RowList[i]
size += len((*a).CellList) * 5
size += len((*a)) * 5
}
return 5 + len(p.RowList)*4 + size
}
......@@ -7082,11 +7055,11 @@ func (p *AnswerTweakPartitionTable) neoMsgEncodeN(data []byte) {
for i := 0; i < l; i++ {
a := &p.RowList[i]
{
l := len((*a).CellList)
l := len((*a))
binary.BigEndian.PutUint32(data[0:], uint32(l))
data = data[4:]
for i := 0; i < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
binary.BigEndian.PutUint32(data[0:], uint32(int32((*a).NID)))
(data[4:])[0] = uint8(int8((*a).State))
data = data[5:]
......@@ -7118,9 +7091,9 @@ func (p *AnswerTweakPartitionTable) neoMsgDecodeN(data []byte) (int, error) {
goto overflow
}
nread += uint64(l) * 5
(*a).CellList = make([]CellInfo, l)
(*a) = make([]CellInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
(*a).NID = NodeID(int32(binary.BigEndian.Uint32(data[0 : 0+4])))
(*a).State = CellState(int8((data[4 : 4+1])[0]))
data = data[5:]
......@@ -7139,11 +7112,11 @@ func (p *AnswerTweakPartitionTable) neoMsgEncodedLenM() int {
var size int
for i := 0; i < len(p.RowList); i++ {
a := &p.RowList[i]
for i := 0; i < len((*a).CellList); i++ {
a := &(*a).CellList[i]
for i := 0; i < len((*a)); i++ {
a := &(*a)[i]
size += msgpack.ArrayHeadSize(reflect.TypeOf((*a)).NumField()) + msgpack.Int32Size(int32((*a).NID))
}
size += msgpack.ArrayHeadSize(reflect.TypeOf((*a)).NumField()) + msgpack.ArrayHeadSize(len((*a).CellList)) + len((*a).CellList)*3
size += msgpack.ArrayHeadSize(len((*a))) + len((*a))*3
}
return 1 + msgpack.ArrayHeadSize(reflect.TypeOf((*p)).NumField()) + msgpack.ArrayHeadSize(len(p.RowList)) + size
}
......@@ -7157,13 +7130,12 @@ func (p *AnswerTweakPartitionTable) neoMsgEncodeM(data []byte) {
data = data[2+n:]
for i := 0; i < l; i++ {
a := &p.RowList[i]
data[0] = byte(msgpack.FixArray_4 | 1)
{
l := len((*a).CellList)
n := msgpack.PutArrayHead(data[1:], l)
data = data[1+n:]
l := len((*a))
n := msgpack.PutArrayHead(data[0:], l)
data = data[0+n:]
for i := 0; i < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
data[0] = byte(msgpack.FixArray_4 | 2)
{
n := msgpack.PutInt32(data[1:], int32((*a).NID))
......@@ -7214,24 +7186,16 @@ func (p *AnswerTweakPartitionTable) neoMsgDecodeM(data []byte) (int, error) {
p.RowList = make([]RowInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &p.RowList[i]
{
_, tail, err := msgp.ReadArrayHeaderBytes(data)
if err != nil {
return 0, mdecodeErr("AnswerTweakPartitionTable", err)
}
nread += uint64(len(data) - len(tail))
data = tail
}
{
l, tail, err := msgp.ReadArrayHeaderBytes(data)
if err != nil {
return 0, mdecodeErr("AnswerTweakPartitionTable.CellList", err)
return 0, mdecodeErr("AnswerTweakPartitionTable", err)
}
nread += uint64(len(data) - len(tail))
data = tail
(*a).CellList = make([]CellInfo, l)
(*a) = make([]CellInfo, l)
for i := 0; uint32(i) < l; i++ {
a := &(*a).CellList[i]
a := &(*a)[i]
{
_, tail, err := msgp.ReadArrayHeaderBytes(data)
if err != nil {
......
go/neo/xneo/parttab.go
View file @ 6f02bc97
......@@ -260,7 +260,7 @@ func (pt *PartitionTable) ToMsg() *proto.SendPartitionTable {
cellv[j] = cell.CellInfo
}
rowv[i] = proto.RowInfo{CellList: cellv}
rowv[i] = cellv
}
msg.RowList = rowv
......@@ -280,7 +280,7 @@ func PartTabFromMsg(msg *proto.SendPartitionTable) *PartitionTable {
}
//pt.tab[i] = append(pt.tab[i], row.CellList...)
for _, cell := range row.CellList {
for _, cell := range row {
pt.tab[i] = append(pt.tab[i], Cell{cell})
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7