gob: beginning of support for GobEncoder/GobDecoder interfaces.

This allows a data item that can marshal itself to be transmitted by its
own encoding, enabling some types to be handled that cannot be
normally, plus providing a way to use gobs on data with unexported
fields.

In this CL, the necessary methods are protected by leading _, so only
package gob can use the facilities (in its tests, of course); this
code is not ready for real use yet.  I could be talked into enabling
it for experimentation, though.  The main drawback is that the
methods must be implemented by the actual type passed through,
not by an indirection from it.  For instance, if *T implements
GobEncoder, you must send a *T, not a T.  This will be addressed
in due course.

Also there is improved commentary and a couple of unrelated
minor bug fixes.

R=rsc
CC=golang-dev
https://golang.org/cl/4243056

src/pkg/gob/codec_test.go

@@ -303,7 +303,7 @@ func TestScalarEncInstructions(t *testing.T) {
 	}
 }
 
-func execDec(typ string, instr *decInstr, state *decodeState, t *testing.T, p unsafe.Pointer) {
+func execDec(typ string, instr *decInstr, state *decoderState, t *testing.T, p unsafe.Pointer) {
 	defer testError(t)
 	v := int(state.decodeUint())
 	if v+state.fieldnum != 6 {
@@ -313,7 +313,7 @@ func execDec(typ string, instr *decInstr, state *decodeState, t *testing.T, p un
 	state.fieldnum = 6
 }
 
-func newDecodeStateFromData(data []byte) *decodeState {
+func newDecodeStateFromData(data []byte) *decoderState {
 	b := bytes.NewBuffer(data)
 	state := newDecodeState(nil, b)
 	state.fieldnum = -1

src/pkg/gob/debug.go

@@ -155,6 +155,16 @@ func (deb *debugger) dump(format string, args ...interface{}) {
 
 // Debug prints a human-readable representation of the gob data read from r.
 func Debug(r io.Reader) {
+	err := debug(r)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "gob debug: %s\n", err)
+	}
+}
+
+// debug implements Debug, but catches panics and returns
+// them as errors to be printed by Debug.
+func debug(r io.Reader) (err os.Error) {
+	defer catchError(&err)
 	fmt.Fprintln(os.Stderr, "Start of debugging")
 	deb := &debugger{
 		r:        newPeekReader(r),
@@ -166,6 +176,7 @@ func Debug(r io.Reader) {
 		deb.remainingKnown = true
 	}
 	deb.gobStream()
+	return
 }
 
 // note that we've consumed some bytes
@@ -386,11 +397,15 @@ func (deb *debugger) typeDefinition(indent tab, id typeId) {
 		// Field number 1 is type Id of key
 		deb.delta(1)
 		keyId := deb.typeId()
-		wire.SliceT = &sliceType{com, id}
 		// Field number 2 is type Id of elem
 		deb.delta(1)
 		elemId := deb.typeId()
 		wire.MapT = &mapType{com, keyId, elemId}
+	case 4: // GobEncoder type, one field of {{Common}}
+		// Field number 0 is CommonType
+		deb.delta(1)
+		com := deb.common()
+		wire.GobEncoderT = &gobEncoderType{com}
 	default:
 		errorf("bad field in type %d", fieldNum)
 	}
@@ -507,6 +522,8 @@ func (deb *debugger) printWireType(indent tab, wire *wireType) {
 		for i, field := range wire.StructT.Field {
 			fmt.Fprintf(os.Stderr, "%sfield %d:\t%s\tid=%d\n", indent+1, i, field.Name, field.Id)
 		}
+	case wire.GobEncoderT != nil:
+		deb.printCommonType(indent, "GobEncoder", &wire.GobEncoderT.CommonType)
 	}
 	indent--
 	fmt.Fprintf(os.Stderr, "%s}\n", indent)
@@ -538,6 +555,8 @@ func (deb *debugger) fieldValue(indent tab, id typeId) {
 		deb.sliceValue(indent, wire)
 	case wire.StructT != nil:
 		deb.structValue(indent, id)
+	case wire.GobEncoderT != nil:
+		deb.gobEncoderValue(indent, id)
 	default:
 		panic("bad wire type for field")
 	}
@@ -654,3 +673,17 @@ func (deb *debugger) structValue(indent tab, id typeId) {
 	fmt.Fprintf(os.Stderr, "%s} // end %s struct\n", indent, id.name())
 	deb.dump(">> End of struct value of type %d %q", id, id.name())
 }
+
+// GobEncoderValue:
+//	uint(n) byte*n
+func (deb *debugger) gobEncoderValue(indent tab, id typeId) {
+	len := deb.uint64()
+	deb.dump("GobEncoder value of %q id=%d, length %d\n", id.name(), id, len)
+	fmt.Fprintf(os.Stderr, "%s%s (implements GobEncoder)\n", indent, id.name())
+	data := make([]byte, len)
+	_, err := deb.r.Read(data)
+	if err != nil {
+		errorf("gobEncoder data read: %s", err)
+	}
+	fmt.Fprintf(os.Stderr, "%s[% .2x]\n", indent+1, data)
+}

src/pkg/gob/decoder.go

@@ -21,7 +21,7 @@ type Decoder struct {
 	wireType     map[typeId]*wireType                    // map from remote ID to local description
 	decoderCache map[reflect.Type]map[typeId]**decEngine // cache of compiled engines
 	ignorerCache map[typeId]**decEngine                  // ditto for ignored objects
-	countState   *decodeState                            // reads counts from wire
+	countState   *decoderState                           // reads counts from wire
 	countBuf     []byte                                  // used for decoding integers while parsing messages
 	tmp          []byte                                  // temporary storage for i/o; saves reallocating
 	err          os.Error

src/pkg/gob/encoder.go

@@ -78,12 +78,57 @@ func (enc *Encoder) writeMessage(w io.Writer, b *bytes.Buffer) {
 	}
 }
 
+// sendActualType sends the requested type, without further investigation, unless
+// it's been sent before.
+func (enc *Encoder) sendActualType(w io.Writer, state *encoderState, ut *userTypeInfo, actual reflect.Type) (sent bool) {
+	if _, alreadySent := enc.sent[actual]; alreadySent {
+		return false
+	}
+	typeLock.Lock()
+	info, err := getTypeInfo(ut)
+	typeLock.Unlock()
+	if err != nil {
+		enc.setError(err)
+		return
+	}
+	// Send the pair (-id, type)
+	// Id:
+	state.encodeInt(-int64(info.id))
+	// Type:
+	enc.encode(state.b, reflect.NewValue(info.wire), wireTypeUserInfo)
+	enc.writeMessage(w, state.b)
+	if enc.err != nil {
+		return
+	}
+
+	// Remember we've sent this type, both what the user gave us and the base type.
+	enc.sent[ut.base] = info.id
+	if ut.user != ut.base {
+		enc.sent[ut.user] = info.id
+	}
+	// Now send the inner types
+	switch st := actual.(type) {
+	case *reflect.StructType:
+		for i := 0; i < st.NumField(); i++ {
+			enc.sendType(w, state, st.Field(i).Type)
+		}
+	case reflect.ArrayOrSliceType:
+		enc.sendType(w, state, st.Elem())
+	}
+	return true
+}
+
+// sendType sends the type info to the other side, if necessary. 
 func (enc *Encoder) sendType(w io.Writer, state *encoderState, origt reflect.Type) (sent bool) {
-	// Drill down to the base type.
 	ut := userType(origt)
-	rt := ut.base
+	if ut.isGobEncoder {
+		// The rules are different: regardless of the underlying type's representation,
+		// we need to tell the other side that this exact type is a GobEncoder.
+		return enc.sendActualType(w, state, ut, ut.user)
+	}
 
-	switch rt := rt.(type) {
+	// It's a concrete value, so drill down to the base type.
+	switch rt := ut.base.(type) {
 	default:
 		// Basic types and interfaces do not need to be described.
 		return
@@ -109,43 +154,7 @@ func (enc *Encoder) sendType(w io.Writer, state *encoderState, origt reflect.Typ
 		return
 	}
 
-	// Have we already sent this type?  This time we ask about the base type.
-	if _, alreadySent := enc.sent[rt]; alreadySent {
-		return
-	}
-
-	// Need to send it.
-	typeLock.Lock()
-	info, err := getTypeInfo(rt)
-	typeLock.Unlock()
-	if err != nil {
-		enc.setError(err)
-		return
-	}
-	// Send the pair (-id, type)
-	// Id:
-	state.encodeInt(-int64(info.id))
-	// Type:
-	enc.encode(state.b, reflect.NewValue(info.wire), wireTypeUserInfo)
-	enc.writeMessage(w, state.b)
-	if enc.err != nil {
-		return
-	}
-
-	// Remember we've sent this type.
-	enc.sent[rt] = info.id
-	// Remember we've sent the top-level, possibly indirect type too.
-	enc.sent[origt] = info.id
-	// Now send the inner types
-	switch st := rt.(type) {
-	case *reflect.StructType:
-		for i := 0; i < st.NumField(); i++ {
-			enc.sendType(w, state, st.Field(i).Type)
-		}
-	case reflect.ArrayOrSliceType:
-		enc.sendType(w, state, st.Elem())
-	}
-	return true
+	return enc.sendActualType(w, state, ut, ut.base)
 }
 
 // Encode transmits the data item represented by the empty interface value,
@@ -159,11 +168,17 @@ func (enc *Encoder) Encode(e interface{}) os.Error {
 // sent.
 func (enc *Encoder) sendTypeDescriptor(w io.Writer, state *encoderState, ut *userTypeInfo) {
 	// Make sure the type is known to the other side.
-	// First, have we already sent this (base) type?
-	base := ut.base
-	if _, alreadySent := enc.sent[base]; !alreadySent {
+	// First, have we already sent this type?
+	rt := ut.base
+	if ut.isGobEncoder {
+		rt = ut.user
+		if ut.encIndir != 0 {
+			panic("TODO: can't handle non-zero encIndir")
+		}
+	}
+	if _, alreadySent := enc.sent[rt]; !alreadySent {
 		// No, so send it.
-		sent := enc.sendType(w, state, base)
+		sent := enc.sendType(w, state, rt)
 		if enc.err != nil {
 			return
 		}
@@ -172,13 +187,13 @@ func (enc *Encoder) sendTypeDescriptor(w io.Writer, state *encoderState, ut *use
 		// need to send the type info but we do need to update enc.sent.
 		if !sent {
 			typeLock.Lock()
-			info, err := getTypeInfo(base)
+			info, err := getTypeInfo(ut)
 			typeLock.Unlock()
 			if err != nil {
 				enc.setError(err)
 				return
 			}
-			enc.sent[base] = info.id
+			enc.sent[rt] = info.id
 		}
 	}
 }

src/pkg/gob/gobencdec_test.go

+// Copyright 20011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file contains tests of the GobEncoder/GobDecoder support.
+
+package gob
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+)
+
+// Types that implement the GobEncoder/Decoder interfaces.
+
+type ByteStruct struct {
+	a byte // not an exported field
+}
+
+type StringStruct struct {
+	s string // not an exported field
+}
+
+type Gobber int
+
+type ValueGobber string // encodes with a value, decodes with a pointer.
+
+// The relevant methods
+
+func (g *ByteStruct) _GobEncode() ([]byte, os.Error) {
+	b := make([]byte, 3)
+	b[0] = g.a
+	b[1] = g.a + 1
+	b[2] = g.a + 2
+	return b, nil
+}
+
+func (g *ByteStruct) _GobDecode(data []byte) os.Error {
+	if g == nil {
+		return os.ErrorString("NIL RECEIVER")
+	}
+	// Expect N sequential-valued bytes.
+	if len(data) == 0 {
+		return os.EOF
+	}
+	g.a = data[0]
+	for i, c := range data {
+		if c != g.a+byte(i) {
+			return os.ErrorString("invalid data sequence")
+		}
+	}
+	return nil
+}
+
+func (g *StringStruct) _GobEncode() ([]byte, os.Error) {
+	return []byte(g.s), nil
+}
+
+func (g *StringStruct) _GobDecode(data []byte) os.Error {
+	// Expect N sequential-valued bytes.
+	if len(data) == 0 {
+		return os.EOF
+	}
+	a := data[0]
+	for i, c := range data {
+		if c != a+byte(i) {
+			return os.ErrorString("invalid data sequence")
+		}
+	}
+	g.s = string(data)
+	return nil
+}
+
+func (g *Gobber) _GobEncode() ([]byte, os.Error) {
+	return []byte(fmt.Sprintf("VALUE=%d", *g)), nil
+}
+
+func (g *Gobber) _GobDecode(data []byte) os.Error {
+	_, err := fmt.Sscanf(string(data), "VALUE=%d", (*int)(g))
+	return err
+}
+
+func (v ValueGobber) _GobEncode() ([]byte, os.Error) {
+	return []byte(fmt.Sprintf("VALUE=%s", v)), nil
+}
+
+func (v *ValueGobber) _GobDecode(data []byte) os.Error {
+	_, err := fmt.Sscanf(string(data), "VALUE=%s", (*string)(v))
+	return err
+}
+
+// Structs that include GobEncodable fields.
+
+type GobTest0 struct {
+	X int // guarantee we have  something in common with GobTest*
+	G *ByteStruct
+}
+
+type GobTest1 struct {
+	X int // guarantee we have  something in common with GobTest*
+	G *StringStruct
+}
+
+type GobTest2 struct {
+	X int    // guarantee we have  something in common with GobTest*
+	G string // not a GobEncoder - should give us errors
+}
+
+type GobTest3 struct {
+	X int     // guarantee we have  something in common with GobTest*
+	G *Gobber // TODO: should be able to satisfy interface without a pointer
+}
+
+type GobTest4 struct {
+	X int // guarantee we have  something in common with GobTest*
+	V ValueGobber
+}
+
+type GobTest5 struct {
+	X int // guarantee we have  something in common with GobTest*
+	V *ValueGobber
+}
+
+type GobTestIgnoreEncoder struct {
+	X int // guarantee we have  something in common with GobTest*
+}
+
+func TestGobEncoderField(t *testing.T) {
+	b := new(bytes.Buffer)
+	// First a field that's a structure.
+	enc := NewEncoder(b)
+	err := enc.Encode(GobTest0{17, &ByteStruct{'A'}})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTest0)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if x.G.a != 'A' {
+		t.Errorf("expected 'A' got %c", x.G.a)
+	}
+	// Now a field that's not a structure.
+	b.Reset()
+	gobber := Gobber(23)
+	err = enc.Encode(GobTest3{17, &gobber})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	y := new(GobTest3)
+	err = dec.Decode(y)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if *y.G != 23 {
+		t.Errorf("expected '23 got %d", *y.G)
+	}
+}
+
+// As long as the fields have the same name and implement the
+// interface, we can cross-connect them.  Not sure it's useful
+// and may even be bad but it works and it's hard to prevent
+// without exposing the contents of the object, which would
+// defeat the purpose.
+func TestGobEncoderFieldsOfDifferentType(t *testing.T) {
+	// first, string in field to byte in field
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	err := enc.Encode(GobTest1{17, &StringStruct{"ABC"}})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTest0)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if x.G.a != 'A' {
+		t.Errorf("expected 'A' got %c", x.G.a)
+	}
+	// now the other direction, byte in field to string in field
+	b.Reset()
+	err = enc.Encode(GobTest0{17, &ByteStruct{'X'}})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	y := new(GobTest1)
+	err = dec.Decode(y)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if y.G.s != "XYZ" {
+		t.Fatalf("expected `XYZ` got %c", y.G.s)
+	}
+}
+
+// Test that we can encode a value and decode into a pointer.
+func TestGobEncoderValueEncoder(t *testing.T) {
+	// first, string in field to byte in field
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	err := enc.Encode(GobTest4{17, ValueGobber("hello")})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTest5)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if *x.V != "hello" {
+		t.Errorf("expected `hello` got %s", x.V)
+	}
+}
+
+func TestGobEncoderFieldTypeError(t *testing.T) {
+	// GobEncoder to non-decoder: error
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	err := enc.Encode(GobTest1{17, &StringStruct{"ABC"}})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := &GobTest2{}
+	err = dec.Decode(x)
+	if err == nil {
+		t.Fatal("expected decode error for mistmatched fields (encoder to non-decoder)")
+	}
+	if strings.Index(err.String(), "type") < 0 {
+		t.Fatal("expected type error; got", err)
+	}
+	// Non-encoder to GobDecoder: error
+	b.Reset()
+	err = enc.Encode(GobTest2{17, "ABC"})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	y := &GobTest1{}
+	err = dec.Decode(y)
+	if err == nil {
+		t.Fatal("expected decode error for mistmatched fields (non-encoder to decoder)")
+	}
+	if strings.Index(err.String(), "type") < 0 {
+		t.Fatal("expected type error; got", err)
+	}
+}
+
+// Even though ByteStruct is a struct, it's treated as a singleton at the top level.
+func TestGobEncoderStructSingleton(t *testing.T) {
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	err := enc.Encode(&ByteStruct{'A'})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(ByteStruct)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if x.a != 'A' {
+		t.Errorf("expected 'A' got %c", x.a)
+	}
+}
+
+func TestGobEncoderNonStructSingleton(t *testing.T) {
+	b := new(bytes.Buffer)
+	enc := NewEncoder(b)
+	g := Gobber(1234) // TODO: shouldn't need to take the address here.
+	err := enc.Encode(&g)
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	var x Gobber
+	err = dec.Decode(&x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if x != 1234 {
+		t.Errorf("expected 1234 got %c", x)
+	}
+}
+
+func TestGobEncoderIgnoreStructField(t *testing.T) {
+	b := new(bytes.Buffer)
+	// First a field that's a structure.
+	enc := NewEncoder(b)
+	err := enc.Encode(GobTest0{17, &ByteStruct{'A'}})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTestIgnoreEncoder)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if x.X != 17 {
+		t.Errorf("expected 17 got %c", x.X)
+	}
+}
+
+func TestGobEncoderIgnoreNonStructField(t *testing.T) {
+	b := new(bytes.Buffer)
+	// First a field that's a structure.
+	enc := NewEncoder(b)
+	gobber := Gobber(23)
+	err := enc.Encode(GobTest3{17, &gobber})
+	if err != nil {
+		t.Fatal("encode error:", err)
+	}
+	dec := NewDecoder(b)
+	x := new(GobTestIgnoreEncoder)
+	err = dec.Decode(x)
+	if err != nil {
+		t.Fatal("decode error:", err)
+	}
+	if x.X != 17 {
+		t.Errorf("expected 17 got %c", x.X)
+	}
+}

src/pkg/gob/type_test.go

@@ -26,7 +26,7 @@ var basicTypes = []typeT{
 func getTypeUnlocked(name string, rt reflect.Type) gobType {
 	typeLock.Lock()
 	defer typeLock.Unlock()
-	t, err := getType(name, rt)
+	t, err := getBaseType(name, rt)
 	if err != nil {
 		panic("getTypeUnlocked: " + err.String())
 	}