Commit e8143557 authored by Kirill Smelkov's avatar Kirill Smelkov

go/neo/proto: MessagePack support for messages (draft)

This patch adds proto.Encoding('M') and teaches it to encode/decode
messages via MessagePack by the rules defined in

	nexedi/neoppod@9d0bf97a
	( nexedi/neoppod!11 )

It only adds support for messages serialization, without changing
proto.go to match changes in e.g. enums reordering, and without adding
support for MessagePack at link layer in neonet.

M-encoding was only tested for NEO/go-NEO/go, and was not yet tested for
NEO/go-NEO/py interoperation. There will be likely small mistakes
discovered on my side that should be hopefully easy to fix step by step
once we get to that phase.
parent a8249bd9
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package proto
// runtime glue for msgpack support
import (
"fmt"
"github.com/tinylib/msgp/msgp"
"lab.nexedi.com/kirr/neo/go/neo/internal/msgpack"
)
// mstructDecodeError represents decode error when decoder was expecting
// tuple<nfield> for structure named path.
type mstructDecodeError struct {
path string // "Type.field.field"
op msgpack.Op // op we got
opOk msgpack.Op // op expected
}
func (e *mstructDecodeError) Error() string {
return fmt.Sprintf("decode: M: struct %s: got opcode %02x; expect %02x", e.path, e.op, e.opOk)
}
// mdecodeErr is called to normilize error when msgp.ReadXXX returns err when decoding path.
func mdecodeErr(path string, err error) error {
if err == msgp.ErrShortBytes {
return ErrDecodeOverflow
}
return &mdecodeError{path, err}
}
type mdecodeError struct {
path string // "Type.field.field"
err error
}
func (e *mdecodeError) Error() string {
return fmt.Sprintf("decode: M: %s: %s", e.path, e.err)
}
// mOpError represents decode error when decoder faces unexpected operation.
type mOpError struct {
op msgpack.Op // op we got
expectedv []msgpack.Op // expected was any op from expectedv
}
func (e *mOpError) Error() string {
return fmt.Sprintf("got opcode %02x; expected any from %02x", e.op, e.expectedv)
}
func mdecodeOpErr(path string, op msgpack.Op, expectedv ...msgpack.Op) error {
return mdecodeErr(path+"/op", &mOpError{op, expectedv})
}
// mLen8Error represents decode error when decoder faces unexpected length in Bin8.
type mLen8Error struct {
l, lOk byte // len we got and expected
}
func (e *mLen8Error) Error() string {
return fmt.Sprintf("expected length %d; got %d", e.lOk, e.l)
}
func mdecodeLen8Err(path string, l, lOk uint8) error {
return mdecodeErr(path+"/len", &mLen8Error{l, lOk})
}
func mdecodeEnumTypeErr(path string, enumType, enumTypeOk byte) error {
return mdecodeErr(path+"/enumType",
fmt.Errorf("expected %d; got %d", enumTypeOk, enumType))
}
func mdecodeEnumValueErr(path string, v byte) error {
return mdecodeErr(path, fmt.Errorf("invalid enum payload %02x", v))
}
...@@ -129,6 +129,10 @@ type Msg interface { ...@@ -129,6 +129,10 @@ type Msg interface {
neoMsgEncodeN(buf []byte) neoMsgEncodeN(buf []byte)
neoMsgDecodeN(data []byte) (nread int, err error) neoMsgDecodeN(data []byte) (nread int, err error)
// M encoding (via MessagePack)
neoMsgEncodedLenM() int
neoMsgEncodeM(buf []byte)
neoMsgDecodeM(data []byte) (nread int, err error)
} }
// Encoding represents messages encoding. // Encoding represents messages encoding.
...@@ -139,6 +143,7 @@ func (e Encoding) MsgEncodedLen(msg Msg) int { ...@@ -139,6 +143,7 @@ func (e Encoding) MsgEncodedLen(msg Msg) int {
switch e { switch e {
default: panic("bug") default: panic("bug")
case 'N': return msg.neoMsgEncodedLenN() case 'N': return msg.neoMsgEncodedLenN()
case 'M': return msg.neoMsgEncodedLenM()
} }
} }
...@@ -149,6 +154,7 @@ func (e Encoding) MsgEncode(msg Msg, buf []byte) { ...@@ -149,6 +154,7 @@ func (e Encoding) MsgEncode(msg Msg, buf []byte) {
switch e { switch e {
default: panic("bug") default: panic("bug")
case 'N': msg.neoMsgEncodeN(buf) case 'N': msg.neoMsgEncodeN(buf)
case 'M': msg.neoMsgEncodeM(buf)
} }
} }
...@@ -157,6 +163,7 @@ func (e Encoding) MsgDecode(msg Msg, data []byte) (nread int, err error) { ...@@ -157,6 +163,7 @@ func (e Encoding) MsgDecode(msg Msg, data []byte) (nread int, err error) {
switch e { switch e {
default: panic("bug") default: panic("bug")
case 'N': return msg.neoMsgDecodeN(data) case 'N': return msg.neoMsgDecodeN(data)
case 'M': return msg.neoMsgDecodeM(data)
} }
} }
...@@ -166,6 +173,7 @@ var ErrDecodeOverflow = errors.New("decode: buffer overflow") ...@@ -166,6 +173,7 @@ var ErrDecodeOverflow = errors.New("decode: buffer overflow")
// ---- messages ---- // ---- messages ----
//neo:proto enum
type ErrorCode uint32 type ErrorCode uint32
const ( const (
ACK ErrorCode = iota ACK ErrorCode = iota
...@@ -183,6 +191,7 @@ const ( ...@@ -183,6 +191,7 @@ const (
INCOMPLETE_TRANSACTION INCOMPLETE_TRANSACTION
) )
//neo:proto enum
type ClusterState int8 type ClusterState int8
const ( const (
// The cluster is initially in the RECOVERING state, and it goes back to // The cluster is initially in the RECOVERING state, and it goes back to
...@@ -216,6 +225,7 @@ const ( ...@@ -216,6 +225,7 @@ const (
STOPPING_BACKUP STOPPING_BACKUP
) )
//neo:proto enum
type NodeType int8 type NodeType int8
const ( const (
MASTER NodeType = iota MASTER NodeType = iota
...@@ -224,6 +234,7 @@ const ( ...@@ -224,6 +234,7 @@ const (
ADMIN ADMIN
) )
//neo:proto enum
type NodeState int8 type NodeState int8
const ( const (
UNKNOWN NodeState = iota //short: U // XXX tag prefix name ? UNKNOWN NodeState = iota //short: U // XXX tag prefix name ?
...@@ -232,6 +243,7 @@ const ( ...@@ -232,6 +243,7 @@ const (
PENDING //short: P PENDING //short: P
) )
//neo:proto enum
type CellState int8 type CellState int8
const ( const (
// Write-only cell. Last transactions are missing because storage is/was down // Write-only cell. Last transactions are missing because storage is/was down
......
...@@ -164,12 +164,19 @@ func TestMsgMarshal(t *testing.T) { ...@@ -164,12 +164,19 @@ func TestMsgMarshal(t *testing.T) {
var testv = []struct { var testv = []struct {
msg Msg msg Msg
encodedN string // []byte encodedN string // []byte
encodedM string // []byte
}{ }{
// empty // empty
{&Ping{}, ""}, {&Ping{},
"",
"\x90",
},
// uint32, string // uint32(N)/enum(M), string
{&Error{Code: 0x01020304, Message: "hello"}, "\x01\x02\x03\x04\x00\x00\x00\x05hello"}, {&Error{Code: 0x00000045, Message: "hello"},
"\x00\x00\x00\x45\x00\x00\x00\x05hello",
hex("92") + hex("d40045") + "\xc4\x05hello",
},
// Oid, Tid, bool, Checksum, []byte // Oid, Tid, bool, Checksum, []byte
{&StoreObject{ {&StoreObject{
...@@ -185,7 +192,18 @@ func TestMsgMarshal(t *testing.T) { ...@@ -185,7 +192,18 @@ func TestMsgMarshal(t *testing.T) {
hex("01020304050607080a0b0c0d0e0f010200") + hex("01020304050607080a0b0c0d0e0f010200") +
hex("0102030405060708090a0b0c0d0e0f1011121314") + hex("0102030405060708090a0b0c0d0e0f1011121314") +
hex("0000000b") + "hello world" + hex("0000000b") + "hello world" +
hex("0a0b0c0d0e0f01030a0b0c0d0e0f0104")}, hex("0a0b0c0d0e0f01030a0b0c0d0e0f0104"),
// M
hex("97") +
hex("c408") + hex("0102030405060708") +
hex("c408") + hex("0a0b0c0d0e0f0102") +
hex("c2") +
hex("c414") + hex("0102030405060708090a0b0c0d0e0f1011121314") +
hex("c40b") + "hello world" +
hex("c408") + hex("0a0b0c0d0e0f0103") +
hex("c408") + hex("0a0b0c0d0e0f0104"),
},
// PTid, [] (of [] of {UUID, CellState}) // PTid, [] (of [] of {UUID, CellState})
{&AnswerPartitionTable{ {&AnswerPartitionTable{
...@@ -205,6 +223,15 @@ func TestMsgMarshal(t *testing.T) { ...@@ -205,6 +223,15 @@ func TestMsgMarshal(t *testing.T) {
hex("000000020000000b010000001100") + hex("000000020000000b010000001100") +
hex("000000010000000b02") + hex("000000010000000b02") +
hex("000000030000000b030000000f040000001701"), hex("000000030000000b030000000f040000001701"),
// M
hex("93") +
hex("cf0102030405060708") +
hex("22") +
hex("93") +
hex("91"+"92"+"920bd40401"+"9211d40400") +
hex("91"+"91"+"920bd40402") +
hex("91"+"93"+"920bd40403"+"920fd40404"+"9217d40401"),
}, },
// map[Oid]struct {Tid,Tid,bool} // map[Oid]struct {Tid,Tid,bool}
...@@ -226,6 +253,14 @@ func TestMsgMarshal(t *testing.T) { ...@@ -226,6 +253,14 @@ func TestMsgMarshal(t *testing.T) {
u64(2) + u64(7) + u64(1) + hex("01") + u64(2) + u64(7) + u64(1) + hex("01") +
u64(5) + u64(4) + u64(3) + hex("01") + u64(5) + u64(4) + u64(3) + hex("01") +
u64(8) + u64(7) + u64(1) + hex("00"), u64(8) + u64(7) + u64(1) + hex("00"),
// M
hex("91") +
hex("84") +
hex("c408")+u64(1) + hex("93") + hex("c408")+u64(1) + hex("c408")+u64(0) + hex("c2") +
hex("c408")+u64(2) + hex("93") + hex("c408")+u64(7) + hex("c408")+u64(1) + hex("c3") +
hex("c408")+u64(5) + hex("93") + hex("c408")+u64(4) + hex("c408")+u64(3) + hex("c3") +
hex("c408")+u64(8) + hex("93") + hex("c408")+u64(7) + hex("c408")+u64(1) + hex("c2"),
}, },
// map[uint32]UUID + trailing ... // map[uint32]UUID + trailing ...
...@@ -247,12 +282,28 @@ func TestMsgMarshal(t *testing.T) { ...@@ -247,12 +282,28 @@ func TestMsgMarshal(t *testing.T) {
u32(4) + u32(17) + u32(4) + u32(17) +
u32(7) + u32(3) + u32(7) + u32(3) +
u64(23) + u64(128), u64(23) + u64(128),
// M
hex("93") +
hex("84") +
hex("01" + "07") +
hex("02" + "09") +
hex("04" + "11") +
hex("07" + "03") +
hex("c408") + u64(23) +
hex("c408") + u64(128),
}, },
// uint32, []uint32 // uint32, []uint32
{&PartitionCorrupted{7, []NodeUUID{1, 3, 9, 4}}, {&PartitionCorrupted{7, []NodeUUID{1, 3, 9, 4}},
// N // N
u32(7) + u32(4) + u32(1) + u32(3) + u32(9) + u32(4), u32(7) + u32(4) + u32(1) + u32(3) + u32(9) + u32(4),
// M
hex("92") +
hex("07") +
hex("94") +
hex("01030904"),
}, },
// uint32, Address, string, IdTime // uint32, Address, string, IdTime
...@@ -264,6 +315,16 @@ func TestMsgMarshal(t *testing.T) { ...@@ -264,6 +315,16 @@ func TestMsgMarshal(t *testing.T) {
hex("3fbf9add1091c895") + hex("3fbf9add1091c895") +
u32(2) + u32(5)+"room1" + u32(7)+"rack234" + u32(2) + u32(5)+"room1" + u32(7)+"rack234" +
u32(3) + u32(3)+u32(4)+u32(5), u32(3) + u32(3)+u32(4)+u32(5),
// M
hex("97") +
hex("d40202") +
hex("11") +
hex("92") + hex("c409")+"localhost" + hex("cd")+u16(7777) +
hex("c406")+"myname" +
hex("cb" + "3fbf9add1091c895") +
hex("92") + hex("c405")+"room1" + hex("c407")+"rack234" +
hex("93") + hex("030405"),
}, },
// IdTime, empty Address, int32 // IdTime, empty Address, int32
...@@ -273,12 +334,26 @@ func TestMsgMarshal(t *testing.T) { ...@@ -273,12 +334,26 @@ func TestMsgMarshal(t *testing.T) {
hex("41d66b15517b469d") + u32(1) + hex("41d66b15517b469d") + u32(1) +
u8(2) + u32(0) /* <- ø Address */ + hex("e0000001") + u8(2) + u8(2) + u32(0) /* <- ø Address */ + hex("e0000001") + u8(2) +
hex("41d66b15517b3d04"), hex("41d66b15517b3d04"),
// M
hex("92") +
hex("cb" + "41d66b15517b469d") +
hex("91") +
hex("95") +
hex("d40202") +
hex("92" + "c400"+"" + "00") +
hex("d2" + "e0000001") +
hex("d40302") +
hex("cb" + "41d66b15517b3d04"),
}, },
// empty IdTime // empty IdTime
{&NotifyNodeInformation{IdTimeNone, []NodeInfo{}}, {&NotifyNodeInformation{IdTimeNone, []NodeInfo{}},
// N // N
hex("ffffffffffffffff") + hex("00000000"), hex("ffffffffffffffff") + hex("00000000"),
// M
hex("92") +
hex("cb" + "fff0000000000000") + // XXX nan/-inf not handled yet
hex("90"),
}, },
// TODO we need tests for: // TODO we need tests for:
...@@ -288,6 +363,7 @@ func TestMsgMarshal(t *testing.T) { ...@@ -288,6 +363,7 @@ func TestMsgMarshal(t *testing.T) {
for _, tt := range testv { for _, tt := range testv {
testMsgMarshal(t, 'N', tt.msg, tt.encodedN) testMsgMarshal(t, 'N', tt.msg, tt.encodedN)
testMsgMarshal(t, 'M', tt.msg, tt.encodedM)
} }
} }
...@@ -295,11 +371,14 @@ func TestMsgMarshal(t *testing.T) { ...@@ -295,11 +371,14 @@ func TestMsgMarshal(t *testing.T) {
// this way we additionally lightly check encode / decode overflow behaviour for all types. // this way we additionally lightly check encode / decode overflow behaviour for all types.
func TestMsgMarshalAllOverflowLightly(t *testing.T) { func TestMsgMarshalAllOverflowLightly(t *testing.T) {
for _, typ := range msgTypeRegistry { for _, typ := range msgTypeRegistry {
for _, enc := range []Encoding{'N'} { for _, enc := range []Encoding{'N', 'M'} {
// zero-value for a type // zero-value for a type
msg := reflect.New(typ).Interface().(Msg) msg := reflect.New(typ).Interface().(Msg)
l := enc.MsgEncodedLen(msg) l := enc.MsgEncodedLen(msg)
zerol := make([]byte, l) zerol := make([]byte, l)
if enc != 'N' { // M-encoding of zero-value is not all zeros
enc.MsgEncode(msg, zerol)
}
// decoding will turn nil slice & map into empty allocated ones. // decoding will turn nil slice & map into empty allocated ones.
// we need it so that reflect.DeepEqual works for msg encode/decode comparison // we need it so that reflect.DeepEqual works for msg encode/decode comparison
n, err := enc.MsgDecode(msg, zerol) n, err := enc.MsgDecode(msg, zerol)
...@@ -313,6 +392,7 @@ func TestMsgMarshalAllOverflowLightly(t *testing.T) { ...@@ -313,6 +392,7 @@ func TestMsgMarshalAllOverflowLightly(t *testing.T) {
} }
// Verify overflow handling on decodeN len checks // Verify overflow handling on decodeN len checks
// TODO + M-variants with big len too?
func TestMsgDecodeLenOverflowN(t *testing.T) { func TestMsgDecodeLenOverflowN(t *testing.T) {
enc := Encoding('N') enc := Encoding('N')
......
...@@ -27,7 +27,7 @@ For every type the following methods are generated in accordance with neo.Msg in ...@@ -27,7 +27,7 @@ For every type the following methods are generated in accordance with neo.Msg in
neoMsgCode() uint16 neoMsgCode() uint16
; E stands for 'N' encoding ; E stands for 'N' and 'M' encodings
neoMsgEncodedLen<E>() int neoMsgEncodedLen<E>() int
neoMsgEncode<E>(buf []byte) neoMsgEncode<E>(buf []byte)
neoMsgDecode<E>(data []byte) (nread int, err error) neoMsgDecode<E>(data []byte) (nread int, err error)
...@@ -80,6 +80,8 @@ import ( ...@@ -80,6 +80,8 @@ import (
"os" "os"
"sort" "sort"
"strings" "strings"
"lab.nexedi.com/kirr/neo/go/neo/internal/msgpack"
) )
// parsed & typechecked input // parsed & typechecked input
...@@ -126,6 +128,10 @@ var zodbOid types.Type ...@@ -126,6 +128,10 @@ var zodbOid types.Type
var neo_customCodecN *types.Interface // type of neo.customCodecN var neo_customCodecN *types.Interface // type of neo.customCodecN
var memBuf types.Type // type of mem.Buf var memBuf types.Type // type of mem.Buf
// registry of enums
var enumRegistry = map[types.Type]int{} // type -> enum type serial
// bytes.Buffer + bell & whistles // bytes.Buffer + bell & whistles
type Buffer struct { type Buffer struct {
bytes.Buffer bytes.Buffer
...@@ -188,6 +194,7 @@ func loadPkg(pkgPath string, sources ...string) *types.Package { ...@@ -188,6 +194,7 @@ func loadPkg(pkgPath string, sources ...string) *types.Package {
type Annotation struct { type Annotation struct {
typeonly bool typeonly bool
answer bool answer bool
enum bool
} }
// parse checks doc for specific comment annotations and, if present, loads them. // parse checks doc for specific comment annotations and, if present, loads them.
...@@ -218,6 +225,12 @@ func (a *Annotation) parse(doc *ast.CommentGroup) { ...@@ -218,6 +225,12 @@ func (a *Annotation) parse(doc *ast.CommentGroup) {
} }
a.answer = true a.answer = true
case "enum":
if a.enum {
log.Fatalf("%v: duplicate `enum`", cpos)
}
a.enum = true
default: default:
log.Fatalf("%v: unknown neo:proto directive %q", cpos, arg) log.Fatalf("%v: unknown neo:proto directive %q", cpos, arg)
} }
...@@ -303,6 +316,9 @@ import ( ...@@ -303,6 +316,9 @@ import (
"reflect" "reflect"
"sort" "sort"
"github.com/tinylib/msgp/msgp"
"lab.nexedi.com/kirr/neo/go/neo/internal/msgpack"
"lab.nexedi.com/kirr/go123/mem" "lab.nexedi.com/kirr/go123/mem"
"lab.nexedi.com/kirr/neo/go/zodb" "lab.nexedi.com/kirr/neo/go/zodb"
)`) )`)
...@@ -312,6 +328,7 @@ import ( ...@@ -312,6 +328,7 @@ import (
// go over message types declaration and generate marshal code for them // go over message types declaration and generate marshal code for them
buf.emit("// messages marshalling\n") buf.emit("// messages marshalling\n")
msgSerial := 0 msgSerial := 0
enumSerial := 0
for _, decl := range f.Decls { for _, decl := range f.Decls {
// we look for types (which can be only under GenDecl) // we look for types (which can be only under GenDecl)
gendecl, ok := decl.(*ast.GenDecl) gendecl, ok := decl.(*ast.GenDecl)
...@@ -332,16 +349,21 @@ import ( ...@@ -332,16 +349,21 @@ import (
typespec := spec.(*ast.TypeSpec) // must be because tok = TYPE typespec := spec.(*ast.TypeSpec) // must be because tok = TYPE
typename := typespec.Name.Name typename := typespec.Name.Name
// we are only interested in struct types
if _, ok := typespec.Type.(*ast.StructType); !ok {
continue
}
// `//neo:proto ...` annotation for this particular type // `//neo:proto ...` annotation for this particular type
specAnnotation := declAnnotation // inheriting from decl specAnnotation := declAnnotation // inheriting from decl
specAnnotation.parse(typespec.Doc) specAnnotation.parse(typespec.Doc)
// type only -> don't generate message interface for it // remember enum types
if specAnnotation.enum {
typ := typeInfo.Defs[typespec.Name].Type()
enumRegistry[typ]= enumSerial
enumSerial++
}
// messages are only struct types without typeonly annotation
if _, ok := typespec.Type.(*ast.StructType); !ok {
continue
}
if specAnnotation.typeonly { if specAnnotation.typeonly {
continue continue
} }
...@@ -366,6 +388,11 @@ import ( ...@@ -366,6 +388,11 @@ import (
buf.WriteString(generateCodecCode(typespec, &encoderN{})) buf.WriteString(generateCodecCode(typespec, &encoderN{}))
buf.WriteString(generateCodecCode(typespec, &decoderN{})) buf.WriteString(generateCodecCode(typespec, &decoderN{}))
// TODO keep all M routines separate from N for code locality ?
buf.WriteString(generateCodecCode(typespec, &sizerM{}))
buf.WriteString(generateCodecCode(typespec, &encoderM{}))
buf.WriteString(generateCodecCode(typespec, &decoderM{}))
msgTypeRegistry[msgCode] = typename msgTypeRegistry[msgCode] = typename
msgSerial++ msgSerial++
} }
...@@ -431,6 +458,11 @@ func typeSizeFixed(encoding byte, typ types.Type) (wireSize int, ok bool) { ...@@ -431,6 +458,11 @@ func typeSizeFixed(encoding byte, typ types.Type) (wireSize int, ok bool) {
var size SymSize var size SymSize
switch encoding { switch encoding {
case 'M':
s := &sizerM{}
codegenType("x", typ, nil, s)
size = s.size
case 'N': case 'N':
s := &sizerN{} s := &sizerN{}
codegenType("x", typ, nil, s) codegenType("x", typ, nil, s)
...@@ -493,6 +525,7 @@ type CodeGenCustomize interface { ...@@ -493,6 +525,7 @@ type CodeGenCustomize interface {
// X reports encoding=X // X reports encoding=X
type N struct{}; func (_ *N) encoding() byte { return 'N' } type N struct{}; func (_ *N) encoding() byte { return 'N' }
type M struct{}; func (_ *M) encoding() byte { return 'M' }
// common part of codegenerators // common part of codegenerators
type commonCodeGen struct { type commonCodeGen struct {
...@@ -526,6 +559,12 @@ func (c *commonCodeGen) var_(varname string) string { ...@@ -526,6 +559,12 @@ func (c *commonCodeGen) var_(varname string) string {
return varname return varname
} }
// pathName returns name representing path or assignto.
func (c *commonCodeGen) pathName(path string) string {
// Type, p.f1.f2 -> Type.f1.f2
return strings.Join(append([]string{c.typeName}, strings.Split(path, ".")[1:]...), ".")
}
// symbolic size // symbolic size
// consists of numeric & symbolic expression parts // consists of numeric & symbolic expression parts
// size is num + expr1 + expr2 + ... // size is num + expr1 + expr2 + ...
...@@ -631,6 +670,7 @@ type sizerCommon struct { ...@@ -631,6 +670,7 @@ type sizerCommon struct {
size SymSize // currently accumulated size size SymSize // currently accumulated size
} }
type sizerN struct { sizerCommon; N } type sizerN struct { sizerCommon; N }
type sizerM struct { sizerCommon; M }
// encoderX generates code to X-encode a message. // encoderX generates code to X-encode a message.
// //
...@@ -652,6 +692,7 @@ type encoderCommon struct { ...@@ -652,6 +692,7 @@ type encoderCommon struct {
n int // current write position in data n int // current write position in data
} }
type encoderN struct { encoderCommon; N } type encoderN struct { encoderCommon; N }
type encoderM struct { encoderCommon; M }
// decoderX generates code to X-decode a message. // decoderX generates code to X-decode a message.
// //
...@@ -686,12 +727,17 @@ type decoderCommon struct { ...@@ -686,12 +727,17 @@ type decoderCommon struct {
overflow OverflowCheck overflow OverflowCheck
} }
type decoderN struct { decoderCommon; N } type decoderN struct { decoderCommon; N }
type decoderM struct { decoderCommon; M }
var _ CodeGenerator = (*sizerN)(nil) var _ CodeGenerator = (*sizerN)(nil)
var _ CodeGenerator = (*encoderN)(nil) var _ CodeGenerator = (*encoderN)(nil)
var _ CodeGenerator = (*decoderN)(nil) var _ CodeGenerator = (*decoderN)(nil)
var _ CodeGenerator = (*sizerM)(nil)
var _ CodeGenerator = (*encoderM)(nil)
var _ CodeGenerator = (*decoderM)(nil)
func (s *sizerCommon) generatedCode() string { func (s *sizerCommon) generatedCode() string {
code := Buffer{} code := Buffer{}
...@@ -849,7 +895,7 @@ func (d *decoderCommon) generatedCode() string { ...@@ -849,7 +895,7 @@ func (d *decoderCommon) generatedCode() string {
// `goto overflow` is not used only for empty structs // `goto overflow` is not used only for empty structs
// NOTE for >0 check actual X in StdSizes{X} does not particularly matter // NOTE for >0 check actual X in StdSizes{X} does not particularly matter
if (&types.StdSizes{8, 8}).Sizeof(d.typ) > 0 { if (&types.StdSizes{8, 8}).Sizeof(d.typ) > 0 || d.enc != 'N' {
code.emit("\noverflow:") code.emit("\noverflow:")
code.emit("return 0, ErrDecodeOverflow") code.emit("return 0, ErrDecodeOverflow")
} }
...@@ -897,6 +943,228 @@ func (d *decoderN) genBasic(assignto string, typ *types.Basic, userType types.Ty ...@@ -897,6 +943,228 @@ func (d *decoderN) genBasic(assignto string, typ *types.Basic, userType types.Ty
d.overflow.Add(basic.wireSize) d.overflow.Add(basic.wireSize)
} }
// M: emit code to size/encode/decode basic fixed type
func (s *sizerM) genBasic(path string, typ *types.Basic, userType types.Type) {
// upath casts path into basic type if needed
// e.g. p.x -> int32(p.x) if p.x is custom type with underlying int32
upath := path
if userType.Underlying() != userType {
upath = fmt.Sprintf("%s(%s)", typ.Name(), upath)
}
// zodb.Tid and zodb.Oid are encoded as [8]bin
if userType == zodbTid || userType == zodbOid {
s.size.Add(1+1+8) // mbin8 + 8 + [8]data
return
}
// enums are encoded as extensions
if _, isEnum := enumRegistry[userType]; isEnum {
s.size.Add(1+1+1) // fixext1 enumType value
return
}
switch typ.Kind() {
case types.Bool: s.size.Add(1) // mfalse|mtrue
case types.Int8: s.size.AddExpr("msgpack.Int8Size(%s)", upath)
case types.Int16: s.size.AddExpr("msgpack.Int16Size(%s)", upath)
case types.Int32: s.size.AddExpr("msgpack.Int32Size(%s)", upath)
case types.Int64: s.size.AddExpr("msgpack.Int64Size(%s)", upath)
case types.Uint8: s.size.AddExpr("msgpack.Uint8Size(%s)", upath)
case types.Uint16: s.size.AddExpr("msgpack.Uint16Size(%s)", upath)
case types.Uint32: s.size.AddExpr("msgpack.Uint32Size(%s)", upath)
case types.Uint64: s.size.AddExpr("msgpack.Uint64Size(%s)", upath)
case types.Float64: s.size.Add(1+8) // mfloat64 + <value64>
}
}
func (e *encoderM) genBasic(path string, typ *types.Basic, userType types.Type) {
// upath casts path into basic type if needed
// e.g. p.x -> int32(p.x) if p.x is custom type with underlying int32
upath := path
if userType.Underlying() != userType {
upath = fmt.Sprintf("%s(%s)", typ.Name(), upath)
}
// zodb.Tid and zodb.Oid are encoded as [8]bin
if userType == zodbTid || userType == zodbOid {
e.emit("data[%v] = byte(msgpack.Bin8)", e.n); e.n++
e.emit("data[%v] = 8", e.n); e.n++
e.emit("binary.BigEndian.PutUint64(data[%v:], uint64(%s))", e.n, path)
e.n += 8
return
}
// enums are encoded as `fixext1 enumType fixint<value>`
if enum, ok := enumRegistry[userType]; ok {
e.emit("data[%v] = byte(msgpack.FixExt1)", e.n); e.n++
e.emit("data[%v] = %d", e.n, enum); e.n++
e.emit("if !(0 <= %s && %s <= 0x7f) {", path, path) // mposfixint
e.emit(` panic("%s: invalid %s enum value)")`, path, typeName(userType))
e.emit("}")
e.emit("data[%v] = byte(%s)", e.n, path); e.n++
return
}
// mputint emits mput<kind>int<size>(path)
mputint := func(kind string, size int) {
KI := "I" // I or <Kind>i
if kind != "" {
KI = strings.ToUpper(kind) + "i"
}
e.emit("{")
e.emit("n := msgpack.Put%snt%d(data[%v:], %s)", KI, size, e.n, upath)
e.emit("data = data[%v+n:]", e.n)
e.emit("}")
e.n = 0
}
switch typ.Kind() {
case types.Bool:
e.emit("data[%v] = byte(msgpack.Bool(%s))", e.n, path)
e.n += 1
case types.Int8: mputint("", 8)
case types.Int16: mputint("", 16)
case types.Int32: mputint("", 32)
case types.Int64: mputint("", 64)
case types.Uint8: mputint("u", 8)
case types.Uint16: mputint("u", 16)
case types.Uint32: mputint("u", 32)
case types.Uint64: mputint("u", 64)
case types.Float64:
// mfloat64 f64
e.emit("data[%v] = byte(msgpack.Float64)", e.n); e.n++
e.emit("float64_neoEncode(data[%v:], %s)", e.n, upath); e.n += 8
}
}
// decoder expects <op>
func (d *decoderM) expectOp(assignto string, op string) {
d.emit("if op := msgpack.Op(data[%v]); op != %s {", d.n, op); d.n++
d.emit(" return 0, mdecodeOpErr(%q, op, %s)", d.pathName(assignto), op)
d.emit("}")
d.overflow.Add(1)
}
// decoder expects mbin8 l
func (d *decoderM) expectBin8Fix(assignto string, l int) {
d.expectOp(assignto, "msgpack.Bin8")
d.emit("if l := data[%v]; l != %d {", d.n, l); d.n++
d.emit(" return 0, mdecodeLen8Err(%q, l, %d)", d.pathName(assignto), l)
d.emit("}")
d.overflow.Add(1)
}
// decoder expects mfixext1 <enumType>
func (d *decoderM) expectEnum(assignto string, enumType int) {
d.expectOp(assignto, "msgpack.FixExt1")
d.emit("if enumType := data[%v]; enumType != %d {", d.n, enumType); d.n++
d.emit(" return 0, mdecodeEnumTypeErr(%q, enumType, %d)", d.pathName(assignto), enumType)
d.emit("}")
d.overflow.Add(1)
}
func (d *decoderM) genBasic(assignto string, typ *types.Basic, userType types.Type) {
// zodb.Tid and zodb.Oid are encoded as [8]bin
if userType == zodbTid || userType == zodbOid {
d.expectBin8Fix(assignto, 8)
d.emit("%s= %s(binary.BigEndian.Uint64(data[%v:]))", assignto, typeName(userType), d.n)
d.n += 8
d.overflow.Add(8)
return
}
// enums are encoded as `fixext1 enumType fixint<value>`
if enum, ok := enumRegistry[userType]; ok {
d.expectEnum(assignto, enum)
d.emit("{")
d.emit("v := data[%v]", d.n); d.n++
d.emit("if !(0 <= v && v <= 0x7f) {") // mposfixint
d.emit(" return 0, mdecodeEnumValueErr(%q, v)", d.pathName(assignto))
d.emit("}")
d.emit("%s= %s(v)", assignto, typeName(userType))
d.emit("}")
d.overflow.Add(1)
return
}
// v represents basic decoded value casted to user type if needed
v := "v"
if userType.Underlying() != userType {
v = fmt.Sprintf("%s(v)", typeName(userType))
}
// mgetint emits assignto = mget<kind>int<size>()
mgetint := func(kind string, size int) {
// we are going to go into msgp - flush previously queued
// overflow checks; put place for next overflow check after
// msgp is done.
d.overflowCheck()
d.resetPos()
defer d.overflowCheck()
KI := "I" // I or <Kind>i
if kind != "" {
KI = strings.ToUpper(kind) + "i"
}
d.emit("{")
d.emit("v, tail, err := msgp.Read%snt%dBytes(data)", KI, size)
d.emit("if err != nil {")
d.emit(" return 0, mdecodeErr(%q, err)", d.pathName(assignto))
d.emit("}")
d.emit("%s= %s", assignto, v)
d.emit("%v += uint64(len(data) - len(tail))", d.var_("nread"))
d.emit("data = tail")
d.emit("}")
}
// mgetfloat emits mgetfloat<size>
mgetfloat := func(size int) {
// delving into msgp - flush/prepare next site for overflow check
d.overflowCheck()
d.resetPos()
defer d.overflowCheck()
d.emit("{")
d.emit("v, tail, err := msgp.ReadFloat%dBytes(data)", size)
d.emit("if err != nil {")
d.emit(" return 0, mdecodeErr(%q, err)", d.pathName(assignto))
d.emit("}")
d.emit("%s= %s", assignto, v)
d.emit("%v += uint64(len(data) - len(tail))", d.var_("nread"))
d.emit("data = tail")
d.emit("}")
}
switch typ.Kind() {
case types.Bool:
d.emit("switch op := msgpack.Op(data[%v]); op {", d.n)
d.emit("default: return 0, mdecodeOpErr(%q, op, msgpack.True, msgpack.False)", d.pathName(assignto))
d.emit("case msgpack.True: %s = true", assignto)
d.emit("case msgpack.False: %s = false", assignto)
d.emit("}")
d.n++
d.overflow.Add(1)
case types.Int8: mgetint("", 8)
case types.Int16: mgetint("", 16)
case types.Int32: mgetint("", 32)
case types.Int64: mgetint("", 64)
case types.Uint8: mgetint("u", 8)
case types.Uint16: mgetint("u", 16)
case types.Uint32: mgetint("u", 32)
case types.Uint64: mgetint("u", 64)
case types.Float64: mgetfloat(64)
}
}
// emit code to size/encode/decode array with sizeof(elem)==1 // emit code to size/encode/decode array with sizeof(elem)==1
// [len(A)]byte // [len(A)]byte
func (s *sizerN) genArray1(path string, typ *types.Array) { func (s *sizerN) genArray1(path string, typ *types.Array) {
...@@ -915,6 +1183,39 @@ func (d *decoderN) genArray1(assignto string, typ *types.Array) { ...@@ -915,6 +1183,39 @@ func (d *decoderN) genArray1(assignto string, typ *types.Array) {
d.overflow.Add(typLen) d.overflow.Add(typLen)
} }
// binX+lenX
// [len(A)]byte
func (s *sizerM) genArray1(path string, typ *types.Array) {
l := int(typ.Len())
s.size.Add(msgpack.BinHeadSize(l))
s.size.Add(l)
}
func (e *encoderM) genArray1(path string, typ *types.Array) {
l := int(typ.Len())
if l > 0xff {
panic("TODO: array1 with > 255 elements")
}
e.emit("data[%v] = byte(msgpack.Bin8)", e.n); e.n++
e.emit("data[%v] = %d", e.n, l); e.n++
e.emit("copy(data[%v:], %v[:])", e.n, path)
e.n += l
}
func (d *decoderM) genArray1(assignto string, typ *types.Array) {
l := int(typ.Len())
if l > 0xff {
panic("TODO: array1 with > 255 elements")
}
d.expectBin8Fix(assignto, l)
d.emit("copy(%v[:], data[%v:%v])", assignto, d.n, d.n+l)
d.n += l
d.overflow.Add(l)
}
// emit code to size/encode/decode string or []byte // emit code to size/encode/decode string or []byte
// len u32 // len u32
// [len]byte // [len]byte
...@@ -963,15 +1264,71 @@ func (d *decoderN) genSlice1(assignto string, typ types.Type) { ...@@ -963,15 +1264,71 @@ func (d *decoderN) genSlice1(assignto string, typ types.Type) {
d.emit("}") d.emit("}")
} }
// bin8+len8|bin16+len16|bin32+len32
// [len]byte
func (s *sizerM) genSlice1(path string, typ types.Type) {
s.size.AddExpr("msgpack.BinHeadSize(len(%s))", path)
s.size.AddExpr("len(%s)", path)
}
func (e *encoderM) genSlice1(path string, typ types.Type) {
e.emit("{")
e.emit("l := len(%s)", path)
e.emit("n := msgpack.PutBinHead(data[%v:], l)", e.n)
e.emit("data = data[%v+n:]", e.n)
e.emit("copy(data, %v)", path)
e.emit("data = data[l:]")
e.emit("}")
e.n = 0
}
func (d *decoderM) genSlice1(assignto string, typ types.Type) {
// -> msgp: flush queued overflow checks; put place for next overflow
// checks after msgp is done.
d.overflowCheck()
d.resetPos()
defer d.overflowCheck()
d.emit("{")
d.emit("b, tail, err := msgp.ReadBytesZC(data)")
d.emit("if err != nil {")
d.emit(" return 0, mdecodeErr(%q, err)", d.pathName(assignto))
d.emit("}")
switch t := typ.(type) {
case *types.Basic:
if t.Kind() != types.String {
log.Panicf("bad basic type in slice1: %v", t)
}
d.emit("%v= string(b)", assignto)
case *types.Slice:
// TODO eventually do not copy, but reference data from original
d.emit("%v= make(%v, len(b))", assignto, typeName(typ))
d.emit("copy(%v, b)", assignto)
default:
log.Panicf("bad type in slice1: %v", typ)
}
d.emit("%v += uint64(len(data) - len(tail))", d.var_("nread"))
d.emit("data = tail")
d.emit("}")
}
// emit code to size/encode/decode mem.Buf // emit code to size/encode/decode mem.Buf
// same as slice1 but buffer is allocated via mem.BufAlloc // same as slice1 but buffer is allocated via mem.BufAlloc
func (s *sizerN) genBuf(path string) { func (s *sizerN) genBuf(path string) {
s.genSlice1(path+".XData()", nil /* typ unused */) s.genSlice1(path+".XData()", nil /* typ unused */)
} }
func (s *sizerM) genBuf(path string) {
s.genSlice1(path+".XData()", nil /* typ unused */)
}
func (e *encoderN) genBuf(path string) { func (e *encoderN) genBuf(path string) {
e.genSlice1(path+".XData()", nil /* typ unused */) e.genSlice1(path+".XData()", nil /* typ unused */)
} }
func (e *encoderM) genBuf(path string) {
e.genSlice1(path+".XData()", nil /* typ unused */)
}
func (d *decoderN) genBuf(assignto string) { func (d *decoderN) genBuf(assignto string) {
d.emit("{") d.emit("{")
...@@ -990,6 +1347,29 @@ func (d *decoderN) genBuf(assignto string) { ...@@ -990,6 +1347,29 @@ func (d *decoderN) genBuf(assignto string) {
d.emit("}") d.emit("}")
} }
func (d *decoderM) genBuf(assignto string) {
// -> msgp: flush queued overflow checks; put place for next overflow
// checks after msgp is done.
d.overflowCheck()
d.resetPos()
defer d.overflowCheck()
d.emit("{")
d.emit("b, tail, err := msgp.ReadBytesZC(data)")
d.emit("if err != nil {")
d.emit(" return 0, mdecodeErr(%q, err)", d.pathName(assignto))
d.emit("}")
// TODO eventually do not copy but reference original
d.emit("%v= mem.BufAlloc(len(b))", assignto)
d.emit("copy(%v.Data, b)", assignto)
d.emit("%v += uint64(len(data) - len(tail))", d.var_("nread"))
d.emit("data = tail")
d.emit("}")
}
// emit code to size/encode/decode slice // emit code to size/encode/decode slice
// len u32 // len u32
// [len]item // [len]item
...@@ -1087,6 +1467,44 @@ func (d *decoderCommon) genSliceCommon(xd CodeGenCustomize, assignto string, typ ...@@ -1087,6 +1467,44 @@ func (d *decoderCommon) genSliceCommon(xd CodeGenCustomize, assignto string, typ
d.emit("}") d.emit("}")
} }
// fixarray|array16+YYYY|array32+ZZZZ
// [len]item
func (s *sizerM) genSliceHead(path string, typ *types.Slice, obj types.Object) {
s.size.AddExpr("msgpack.ArrayHeadSize(len(%s))", path)
}
func (s *sizerM) genSlice(path string, typ *types.Slice, obj types.Object) {
s.genSliceCommon(s, path, typ, obj)
}
func (e *encoderM) genSliceHead(path string, typ *types.Slice, obj types.Object) {
e.emit("l := len(%s)", path)
e.emit("n := msgpack.PutArrayHead(data[%v:], l)", e.n)
e.emit("data = data[%v+n:]", e.n)
e.n = 0
}
func (e *encoderM) genSlice(path string, typ *types.Slice, obj types.Object) {
e.genSliceCommon(e, path, typ, obj)
}
func (d *decoderM) genSliceHead(assignto string, typ *types.Slice, obj types.Object) {
// -> msgp: flush queued overflow checks; put place for next overflow
// checks after msgp is done.
d.overflowCheck()
d.resetPos()
defer d.overflowCheck()
d.emit("l, tail, err := msgp.ReadArrayHeaderBytes(data)")
d.emit("if err != nil {")
d.emit(" return 0, mdecodeErr(%q, err)", d.pathName(assignto))
d.emit("}")
d.emit("%v += uint64(len(data) - len(tail))", d.var_("nread"))
d.emit("data = tail")
}
func (d *decoderM) genSlice(assignto string, typ *types.Slice, obj types.Object) {
d.genSliceCommon(d, assignto, typ, obj)
}
// generate code to encode/decode map // generate code to encode/decode map
// len u32 // len u32
// [len](key, value) // [len](key, value)
...@@ -1208,6 +1626,44 @@ func (d *decoderCommon) genMapCommon(xd CodeGenCustomize, assignto string, typ * ...@@ -1208,6 +1626,44 @@ func (d *decoderCommon) genMapCommon(xd CodeGenCustomize, assignto string, typ *
d.emit("}") d.emit("}")
} }
// fixmap|map16+YYYY|map32+ZZZZ
// [len]key/value
func (s *sizerM) genMapHead(path string, typ *types.Map, obj types.Object) {
s.size.AddExpr("msgpack.MapHeadSize(len(%s))", path)
}
func (s *sizerM) genMap(path string, typ *types.Map, obj types.Object) {
s.genMapCommon(s, path, typ, obj)
}
func (e *encoderM) genMapHead(path string, typ *types.Map, obj types.Object) {
e.emit("l := len(%s)", path)
e.emit("n := msgpack.PutMapHead(data[%v:], l)", e.n)
e.emit("data = data[%v+n:]", e.n)
e.n = 0
}
func (e *encoderM) genMap(path string, typ *types.Map, obj types.Object) {
e.genMapCommon(e, path, typ, obj)
}
func (d *decoderM) genMapHead(assignto string, typ *types.Map, obj types.Object) {
// -> msgp: flush queued overflow checks; put place for next overflow
// checks after msgp is done.
d.overflowCheck()
d.resetPos()
defer d.overflowCheck()
d.emit("l, tail, err := msgp.ReadMapHeaderBytes(data)")
d.emit("if err != nil {")
d.emit(" return 0, mdecodeErr(%q, err)", d.pathName(assignto))
d.emit("}")
d.emit("%v += uint64(len(data) - len(tail))", d.var_("nread"))
d.emit("data = tail")
}
func (d *decoderM) genMap(assignto string, typ *types.Map, obj types.Object) {
d.genMapCommon(d, assignto, typ, obj)
}
// emit code to size/encode/decode custom type // emit code to size/encode/decode custom type
func (s *sizerN) genCustomN(path string) { func (s *sizerN) genCustomN(path string) {
s.size.AddExpr("%s.neoEncodedLenN()", path) s.size.AddExpr("%s.neoEncodedLenN()", path)
...@@ -1246,6 +1702,35 @@ func (s *sizerN) genStructHead(path string, typ *types.Struct, userType types. ...@@ -1246,6 +1702,35 @@ func (s *sizerN) genStructHead(path string, typ *types.Struct, userType types.
func (e *encoderN) genStructHead(path string, typ *types.Struct, userType types.Type) {} func (e *encoderN) genStructHead(path string, typ *types.Struct, userType types.Type) {}
func (d *decoderN) genStructHead(path string, typ *types.Struct, userType types.Type) {} func (d *decoderN) genStructHead(path string, typ *types.Struct, userType types.Type) {}
// M: array<nfields>
func (s *sizerM) genStructHead(path string, typ *types.Struct, userType types.Type) {
s.size.Add(1) // mfixarray|marray16|marray32
if typ.NumFields() > 0x0f {
panic("TODO: struct with > 15 elements")
}
}
func (e *encoderM) genStructHead(path string, typ *types.Struct, userType types.Type) {
if typ.NumFields() > 0x0f {
panic("TODO: struct with > 15 elements")
}
e.emit("data[%v] = byte(msgpack.FixArray_4 | %d)", e.n, typ.NumFields())
e.n += 1
}
func (d *decoderM) genStructHead(path string, typ *types.Struct, userType types.Type) {
if typ.NumFields() > 0x0f {
panic("TODO: struct with > 15 elements")
}
d.emit("if op, opOk := msgpack.Op(data[%v]), msgpack.FixArray_4 | %d ; op != opOk {", d.n, typ.NumFields())
d.emit("return 0, &mstructDecodeError{%q, op, opOk}", d.pathName(path))
d.emit("}")
d.n += 1
d.overflow.Add(1)
}
// top-level driver for emitting size/encode/decode code for a type // top-level driver for emitting size/encode/decode code for a type
// //
...@@ -1274,7 +1759,7 @@ func codegenType(path string, typ types.Type, obj types.Object, codegen CodeGene ...@@ -1274,7 +1759,7 @@ func codegenType(path string, typ types.Type, obj types.Object, codegen CodeGene
break break
} }
_, ok := basicTypesN[u.Kind()] // ok to check N to see if supported for any encoding _, ok := basicTypesN[u.Kind()] // ok to check N to see if supported for both N and M
if !ok { if !ok {
log.Fatalf("%v: %v: basic type %v not supported", pos(obj), obj.Name(), u) log.Fatalf("%v: %v: basic type %v not supported", pos(obj), obj.Name(), u)
} }
......
This source diff could not be displayed because it is too large. You can view the blob instead.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment