make the low-level encoder and decoder private and have them access byte.Buffers rather

than io.Readers and io.Writers.
change the Encoder/Decoder protocol so that each message is preceded by its length in bytes.

R=rsc
DELTA=468  (119 added, 23 deleted, 326 changed)
OCL=31700
CL=31702

src/pkg/Make.deps

@@ -24,7 +24,7 @@ go/parser.install: bytes.install container/vector.install fmt.install go/ast.ins
 go/printer.install: fmt.install go/ast.install go/token.install io.install os.install reflect.install strings.install
 go/scanner.install: bytes.install container/vector.install fmt.install go/token.install io.install os.install sort.install strconv.install unicode.install utf8.install
 go/token.install: strconv.install
-gob.install: fmt.install io.install math.install os.install reflect.install strings.install sync.install unicode.install
+gob.install: bytes.install fmt.install io.install math.install os.install reflect.install strings.install sync.install unicode.install
 hash.install: io.install
 hash/adler32.install: hash.install os.install
 hash/crc32.install: hash.install os.install

src/pkg/gob/decoder.go

@@ -5,6 +5,7 @@
 package gob
 
 import (
+	"bytes";
 	"gob";
 	"io";
 	"os";
@@ -14,15 +15,19 @@ import (
 
 type Decoder struct {
 	sync.Mutex;	// each item must be received atomically
+	r	io.Reader;	// source of the data
 	seen	map[TypeId] *wireType;	// which types we've already seen described
-	state	*DecState;	// so we can encode integers, strings directly
+	state	*decodeState;	// reads data from in-memory buffer
+	countState	*decodeState;	// reads counts from wire
+	oneByte	[]byte;
 }
 
 func NewDecoder(r io.Reader) *Decoder {
 	dec := new(Decoder);
+	dec.r = r;
 	dec.seen = make(map[TypeId] *wireType);
-	dec.state = new(DecState);
-	dec.state.r = r;	// the rest isn't important; all we need is buffer and reader
+	dec.state = new(decodeState);	// buffer set in Decode(); rest is unimportant
+	dec.oneByte = make([]byte, 1);
 
 	return dec;
 }
@@ -36,7 +41,7 @@ func (dec *Decoder) recvType(id TypeId) {
 
 	// Type:
 	wire := new(wireType);
-	Decode(dec.state.r, wire);
+	decode(dec.state.b, wire);
 	// Remember we've seen this type.
 	dec.seen[id] = wire;
 }
@@ -50,36 +55,56 @@ func (dec *Decoder) Decode(e interface{}) os.Error {
 	dec.Lock();
 	defer dec.Unlock();
 
-	var id TypeId;
-	for dec.state.err == nil {
+	dec.state.err = nil;
+	for {
+		// Read a count.
+		nbytes, err := decodeUintReader(dec.r, dec.oneByte);
+		if err != nil {
+			return err;
+		}
+
+		// Read the data
+		buf := make([]byte, nbytes);	// TODO(r): avoid repeated allocation
+		var n int;
+		n, err = dec.r.Read(buf);
+		if err != nil {
+			return err;
+		}
+		if n < int(nbytes) {
+			return os.ErrorString("gob decode: short read");
+		}
+
+		dec.state.b = bytes.NewBuffer(buf);	// TODO(r): avoid repeated allocation
 		// Receive a type id.
-		id = TypeId(DecodeInt(dec.state));
+		id := TypeId(decodeInt(dec.state));
+		if dec.state.err != nil {
+			return dec.state.err
+		}
 
-		// If the id is positive, we have a value.  0 is the error state
-		if id >= 0 {
-			break;
+		if id < 0 {	// 0 is the error state, handled above
+			// If the id is negative, we have a type.
+			dec.recvType(-id);
+			if dec.state.err != nil {
+				return dec.state.err
+			}
+			continue;
 		}
 
-		// The id is negative; a type descriptor follows.
-		dec.recvType(-id);
-	}
-	if dec.state.err != nil {
-		return dec.state.err
-	}
+		// we have a value
+		info := getTypeInfo(rt);
+
+		// Check type compatibility.
+		// TODO(r): need to make the decoder work correctly if the wire type is compatible
+		// but not equal to the local type (e.g, extra fields).
+		if info.wire.name() != dec.seen[id].name() {
+			dec.state.err = os.ErrorString("gob decode: incorrect type for wire value: want " + info.wire.name() + "; received " + dec.seen[id].name());
+			return dec.state.err
+		}
 
-	info := getTypeInfo(rt);
+		// Receive a value.
+		decode(dec.state.b, e);
 
-	// Check type compatibility.
-	// TODO(r): need to make the decoder work correctly if the wire type is compatible
-	// but not equal to the local type (e.g, extra fields).
-	if info.wire.name() != dec.seen[id].name() {
-		dec.state.err = os.ErrorString("gob decode: incorrect type for wire value: want " + info.wire.name() + "; received " + dec.seen[id].name());
 		return dec.state.err
 	}
-
-	// Receive a value.
-	Decode(dec.state.r, e);
-
-	// Release and return.
-	return dec.state.err
+	return nil	// silence compiler
 }

src/pkg/gob/encoder.go

@@ -5,6 +5,7 @@
 package gob
 
 import (
+	"bytes";
 	"gob";
 	"io";
 	"os";
@@ -14,20 +15,45 @@ import (
 
 type Encoder struct {
 	sync.Mutex;	// each item must be sent atomically
+	w	io.Writer;	// where to send the data
 	sent	map[reflect.Type] TypeId;	// which types we've already sent
-	state	*EncState;	// so we can encode integers, strings directly
+	state	*encoderState;	// so we can encode integers, strings directly
+	countState	*encoderState;	// stage for writing counts
+	buf	[]byte;	// for collecting the output.
 }
 
 func NewEncoder(w io.Writer) *Encoder {
 	enc := new(Encoder);
+	enc.w = w;
 	enc.sent = make(map[reflect.Type] TypeId);
-	enc.state = new(EncState);
-	enc.state.w = w;	// the rest isn't important; all we need is buffer and writer
+	enc.state = new(encoderState);
+	enc.state.b = new(bytes.Buffer);	// the rest isn't important; all we need is buffer and writer
+	enc.countState = new(encoderState);
+	enc.countState.b = new(bytes.Buffer);	// the rest isn't important; all we need is buffer and writer
 	return enc;
 }
 
 func (enc *Encoder) badType(rt reflect.Type) {
-	enc.state.err = os.ErrorString("can't encode type " + rt.String());
+	enc.state.err = os.ErrorString("gob: can't encode type " + rt.String());
+}
+
+// Send the data item preceded by a unsigned count of its length.
+func (enc *Encoder) send() {
+	// Encode the length.
+	encodeUint(enc.countState, uint64(enc.state.b.Len()));
+	// Build the buffer.
+	countLen := enc.countState.b.Len();
+	total := countLen + enc.state.b.Len();
+	if total > len(enc.buf) {
+		enc.buf = make([]byte, total+1000);	// extra for growth
+	}
+	// Place the length before the data.
+	// TODO(r): avoid the extra copy here.
+	enc.countState.b.Read(enc.buf[0:countLen]);
+	// Now the data.
+	enc.state.b.Read(enc.buf[countLen:total]);
+	// Write the data.
+	enc.w.Write(enc.buf[0:total]);
 }
 
 func (enc *Encoder) sendType(origt reflect.Type) {
@@ -63,9 +89,11 @@ func (enc *Encoder) sendType(origt reflect.Type) {
 	info := getTypeInfo(rt);
 	// Send the pair (-id, type)
 	// Id:
-	EncodeInt(enc.state, -int64(info.typeId));
+	encodeInt(enc.state, -int64(info.typeId));
 	// Type:
-	Encode(enc.state.w, info.wire);
+	encode(enc.state.b, info.wire);
+	enc.send();
+
 	// Remember we've sent this type.
 	enc.sent[rt] = info.typeId;
 	// Remember we've sent the top-level, possibly indirect type too.
@@ -78,6 +106,9 @@ func (enc *Encoder) sendType(origt reflect.Type) {
 }
 
 func (enc *Encoder) Encode(e interface{}) os.Error {
+	if enc.state.b.Len() > 0 || enc.countState.b.Len() > 0 {
+		panicln("Encoder: buffer not empty")
+	}
 	rt, indir := indirect(reflect.Typeof(e));
 
 	// Make sure we're single-threaded through here.
@@ -90,16 +121,18 @@ func (enc *Encoder) Encode(e interface{}) os.Error {
 		// No, so send it.
 		enc.sendType(rt);
 		if enc.state.err != nil {
+			enc.state.b.Reset();
+			enc.countState.b.Reset();
 			return enc.state.err
 		}
 	}
 
 	// Identify the type of this top-level value.
-	EncodeInt(enc.state, int64(enc.sent[rt]));
+	encodeInt(enc.state, int64(enc.sent[rt]));
 
-	// Finally, send the data
-	Encode(enc.state.w, e);
+	// Encode the object.
+	encode(enc.state.b, e);
+	enc.send();
 
-	// Release and return.
 	return enc.state.err
 }

src/pkg/gob/encoder_test.go

@@ -57,17 +57,22 @@ func TestBasicEncoder(t *testing.T) {
 	}
 
 	// Decode the result by hand to verify;
-	state := new(DecState);
-	state.r = b;
+	state := new(decodeState);
+	state.b = b;
 	// The output should be:
+	// 0) The length, 38.
+	length := decodeUint(state);
+	if length != 38 {
+		t.Fatal("0. expected length 38; got", length);
+	}
 	// 1) -7: the type id of ET1
-	id1 := DecodeInt(state);
+	id1 := decodeInt(state);
 	if id1 >= 0 {
 		t.Fatal("expected ET1 negative id; got", id1);
 	}
 	// 2) The wireType for ET1
 	wire1 := new(wireType);
-	err := Decode(b, wire1);
+	err := decode(b, wire1);
 	if err != nil {
 		t.Fatal("error decoding ET1 type:", err);
 	}
@@ -76,14 +81,19 @@ func TestBasicEncoder(t *testing.T) {
 	if !reflect.DeepEqual(wire1, trueWire1) {
 		t.Fatalf("invalid wireType for ET1: expected %+v; got %+v\n", *trueWire1, *wire1);
 	}
-	// 3) -8: the type id of ET2
-	id2 := DecodeInt(state);
+	// 3) The length, 21.
+	length = decodeUint(state);
+	if length != 21 {
+		t.Fatal("3. expected length 21; got", length);
+	}
+	// 4) -8: the type id of ET2
+	id2 := decodeInt(state);
 	if id2 >= 0 {
 		t.Fatal("expected ET2 negative id; got", id2);
 	}
-	// 4) The wireType for ET2
+	// 5) The wireType for ET2
 	wire2 := new(wireType);
-	err = Decode(b, wire2);
+	err = decode(b, wire2);
 	if err != nil {
 		t.Fatal("error decoding ET2 type:", err);
 	}
@@ -92,21 +102,26 @@ func TestBasicEncoder(t *testing.T) {
 	if !reflect.DeepEqual(wire2, trueWire2) {
 		t.Fatalf("invalid wireType for ET2: expected %+v; got %+v\n", *trueWire2, *wire2);
 	}
-	// 5) The type id for the et1 value
-	newId1 := DecodeInt(state);
+	// 6) The length, 6.
+	length = decodeUint(state);
+	if length != 6 {
+		t.Fatal("6. expected length 6; got", length);
+	}
+	// 7) The type id for the et1 value
+	newId1 := decodeInt(state);
 	if newId1 != -id1 {
 		t.Fatal("expected Et1 id", -id1, "got", newId1);
 	}
-	// 6) The value of et1
+	// 8) The value of et1
 	newEt1 := new(ET1);
-	err = Decode(b, newEt1);
+	err = decode(b, newEt1);
 	if err != nil {
 		t.Fatal("error decoding ET1 value:", err);
 	}
 	if !reflect.DeepEqual(et1, newEt1) {
 		t.Fatalf("invalid data for et1: expected %+v; got %+v\n", *et1, *newEt1);
 	}
-	// 7) EOF
+	// 9) EOF
 	if b.Len() != 0 {
 		t.Error("not at eof;", b.Len(), "bytes left")
 	}
@@ -117,14 +132,19 @@ func TestBasicEncoder(t *testing.T) {
 	if enc.state.err != nil {
 		t.Error("2nd round: encoder fail:", enc.state.err)
 	}
+	// The length.
+	length = decodeUint(state);
+	if length != 6 {
+		t.Fatal("6. expected length 6; got", length);
+	}
 	// 5a) The type id for the et1 value
-	newId1 = DecodeInt(state);
+	newId1 = decodeInt(state);
 	if newId1 != -id1 {
 		t.Fatal("2nd round: expected Et1 id", -id1, "got", newId1);
 	}
 	// 6a) The value of et1
 	newEt1 = new(ET1);
-	err = Decode(b, newEt1);
+	err = decode(b, newEt1);
 	if err != nil {
 		t.Fatal("2nd round: error decoding ET1 value:", err);
 	}

src/pkg/gob/type.go

@@ -17,7 +17,7 @@ import (
 // Internally, they are used as keys to a map to recover the underlying type info.
 type TypeId int32
 
-var id	TypeId	// incremented for each new type we build
+var nextId	TypeId	// incremented for each new type we build
 var typeLock	sync.Mutex	// set while building a type
 
 type gobType interface {
@@ -31,9 +31,9 @@ var types = make(map[reflect.Type] gobType)
 var idToType = make(map[TypeId] gobType)
 
 func setTypeId(typ gobType) {
-	id++;
-	typ.setId(id);
-	idToType[id] = typ;
+	nextId++;
+	typ.setId(nextId);
+	idToType[nextId] = typ;
 }
 
 func (t TypeId) gobType() gobType {
@@ -296,7 +296,7 @@ func bootstrapType(name string, e interface{}) TypeId {
 	typ := &commonType{ name: name };
 	types[rt] = typ;
 	setTypeId(typ);
-	return id
+	return nextId
 }
 
 // Representation of the information we send and receive about this type.