net/textproto: new package, with example net/dict

Generic text-based network protcol library for SMTP-like protocols.
HTTP and NNTP should be changed to use this package,
and I expect that SMTP and POP3 will be able to use it too.

R=cemeyer, nigeltao_golang, r
CC=golang-dev, petar-m
https://golang.org/cl/889041

src/pkg/Makefile

@@ -95,6 +95,8 @@ DIRS=\
 	mime\
 	mime/multipart\
 	net\
+	net/dict\
+	net/textproto\
 	netchan\
 	nntp\
 	once\
@@ -139,6 +141,7 @@ NOTEST=\
 	http/pprof\
 	image\
 	image/jpeg\
+	net/dict\
 	rand\
 	runtime\
 	runtime/pprof\

src/pkg/net/dict/Makefile

+include ../../../Make.$(GOARCH)
+
+TARG=net/dict
+GOFILES=\
+	dict.go\
+
+include ../../../Make.pkg

src/pkg/net/dict/dict.go

+// Copyright 2010 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package dict implements the Dictionary Server Protocol
+// as defined in RFC 2229.
+package dict
+
+import (
+	"container/vector"
+	"net/textproto"
+	"os"
+	"strconv"
+	"strings"
+)
+
+// A Client represents a client connection to a dictionary server.
+type Client struct {
+	text *textproto.Conn
+}
+
+// Dial returns a new client connected to a dictionary server at
+// addr on the given network.
+func Dial(network, addr string) (*Client, os.Error) {
+	text, err := textproto.Dial(network, addr)
+	if err != nil {
+		return nil, err
+	}
+	_, _, err = text.ReadCodeLine(220)
+	if err != nil {
+		text.Close()
+		return nil, err
+	}
+	return &Client{text: text}, nil
+}
+
+// Close closes the connection to the dictionary server.
+func (c *Client) Close() os.Error {
+	return c.text.Close()
+}
+
+// A Dict represents a dictionary available on the server.
+type Dict struct {
+	Name string // short name of dictionary
+	Desc string // long description
+}
+
+// Dicts returns a list of the dictionaries available on the server.
+func (c *Client) Dicts() ([]Dict, os.Error) {
+	id, err := c.text.Cmd("SHOW DB")
+	if err != nil {
+		return nil, err
+	}
+
+	c.text.StartResponse(id)
+	defer c.text.EndResponse(id)
+
+	lines, err := c.text.ReadDotLines()
+	if err != nil {
+		return nil, err
+	}
+	_, _, err = c.text.ReadCodeLine(250)
+
+	dicts := make([]Dict, len(lines))
+	for i := range dicts {
+		d := &dicts[i]
+		a, _ := fields(lines[i])
+		if len(a) < 2 {
+			return nil, textproto.ProtocolError("invalid dictionary: " + lines[i])
+		}
+		d.Name = a[0]
+		d.Desc = a[1]
+	}
+	return dicts, err
+}
+
+// A Defn represents a definition.
+type Defn struct {
+	Dict Dict   // Dict where definition was found
+	Word string // Word being defined
+	Text []byte // Definition text, typically multiple lines
+}
+
+// Define requests the definition of the given word.
+// The argument dict names the dictionary to use,
+// the Name field of a Dict returned by Dicts.
+//
+// The special dictionary name "!" means to look in all the
+// server's dictionaries.
+// The special dictionary name "*" means to look in all the
+// server's dictionaries in turn, stopping after finding the word
+// in one of them.
+func (c *Client) Define(dict, word string) ([]*Defn, os.Error) {
+	id, err := c.text.Cmd("DEFINE %s %q", dict, word)
+	if err != nil {
+		return nil, err
+	}
+
+	c.text.StartResponse(id)
+	defer c.text.EndResponse(id)
+
+	_, line, err := c.text.ReadCodeLine(150)
+	a, _ := fields(line)
+	if len(a) < 1 {
+		return nil, textproto.ProtocolError("malformed response: " + line)
+	}
+	n, err := strconv.Atoi(a[0])
+	if err != nil {
+		return nil, textproto.ProtocolError("invalid definition count: " + a[0])
+	}
+	def := make([]*Defn, n)
+	for i := 0; i < n; i++ {
+		_, line, err = c.text.ReadCodeLine(151)
+		if err != nil {
+			return nil, err
+		}
+		a, _ := fields(line)
+		if len(a) < 3 {
+			// skip it, to keep protocol in sync
+			i--
+			n--
+			def = def[0:n]
+			continue
+		}
+		d := &Defn{Word: a[0], Dict: Dict{a[1], a[2]}}
+		d.Text, err = c.text.ReadDotBytes()
+		if err != nil {
+			return nil, err
+		}
+		def[i] = d
+	}
+	_, _, err = c.text.ReadCodeLine(250)
+	return def, err
+}
+
+// Fields returns the fields in s.
+// Fields are space separated unquoted words
+// or quoted with single or double quote.
+func fields(s string) ([]string, os.Error) {
+	var v vector.StringVector
+	i := 0
+	for {
+		for i < len(s) && (s[i] == ' ' || s[i] == '\t') {
+			i++
+		}
+		if i >= len(s) {
+			break
+		}
+		if s[i] == '"' || s[i] == '\'' {
+			q := s[i]
+			// quoted string
+			var j int
+			for j = i + 1; ; j++ {
+				if j >= len(s) {
+					return nil, textproto.ProtocolError("malformed quoted string")
+				}
+				if s[j] == '\\' {
+					j++
+					continue
+				}
+				if s[j] == q {
+					j++
+					break
+				}
+			}
+			v.Push(unquote(s[i+1 : j-1]))
+			i = j
+		} else {
+			// atom
+			var j int
+			for j = i; j < len(s); j++ {
+				if s[j] == ' ' || s[j] == '\t' || s[j] == '\\' || s[j] == '"' || s[j] == '\'' {
+					break
+				}
+			}
+			v.Push(s[i:j])
+			i = j
+		}
+		if i < len(s) {
+			c := s[i]
+			if c != ' ' && c != '\t' {
+				return nil, textproto.ProtocolError("quotes not on word boundaries")
+			}
+		}
+	}
+	return v, nil
+}
+
+func unquote(s string) string {
+	if strings.Index(s, "\\") < 0 {
+		return s
+	}
+	b := []byte(s)
+	w := 0
+	for r := 0; r < len(b); r++ {
+		c := b[r]
+		if c == '\\' {
+			r++
+			c = b[r]
+		}
+		b[w] = c
+		w++
+	}
+	return string(b[0:w])
+}

src/pkg/net/textproto/Makefile

+# Copyright 2010 The Go Authors.  All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.$(GOARCH)
+
+TARG=net/textproto
+GOFILES=\
+	pipeline.go\
+	reader.go\
+	textproto.go\
+	writer.go\
+
+include ../../../Make.pkg

src/pkg/net/textproto/pipeline.go

+// Copyright 2010 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package textproto
+
+import (
+	"sync"
+)
+
+// A Pipeline manages a pipelined in-order request/response sequence.
+//
+// To use a Pipeline p to manage multiple clients on a connection,
+// each client should run:
+//
+//	id := p.Next()	// take a number
+//
+//	p.StartRequest(id)	// wait for turn to send request
+//	«send request»
+//	p.EndRequest(id)	// notify Pipeline that request is sent
+//
+//	p.StartResponse(id)	// wait for turn to read response
+//	«read response»
+//	p.EndResponse(id)	// notify Pipeline that response is read
+//
+// A pipelined server can use the same calls to ensure that
+// responses computed in parallel are written in the correct order.
+type Pipeline struct {
+	mu       sync.Mutex
+	id       uint
+	request  sequencer
+	response sequencer
+}
+
+// Next returns the next id for a request/response pair.
+func (p *Pipeline) Next() uint {
+	p.mu.Lock()
+	id := p.id
+	p.id++
+	p.mu.Unlock()
+	return id
+}
+
+// StartRequest blocks until it is time to send (or, if this is a server, receive)
+// the request with the given id.
+func (p *Pipeline) StartRequest(id uint) {
+	p.request.Start(id)
+}
+
+// EndRequest notifies p that the request with the given id has been sent
+// (or, if this is a server, received).
+func (p *Pipeline) EndRequest(id uint) {
+	p.request.End(id)
+}
+
+// StartResponse blocks until it is time to receive (or, if this is a server, send)
+// the request with the given id.
+func (p *Pipeline) StartResponse(id uint) {
+	p.response.Start(id)
+}
+
+// EndResponse notifies p that the response with the given id has been received
+// (or, if this is a server, sent).
+func (p *Pipeline) EndResponse(id uint) {
+	p.response.End(id)
+}
+
+// A sequencer schedules a sequence of numbered events that must
+// happen in order, one after the other.  The event numbering must start
+// at 0 and increment without skipping.  The event number wraps around
+// safely as long as there are not 2^32 simultaneous events pending.
+type sequencer struct {
+	mu   sync.Mutex
+	id   uint
+	wait map[uint]chan uint
+}
+
+// Start waits until it is time for the event numbered id to begin.
+// That is, except for the first event, it waits until End(id-1) has
+// been called.
+func (s *sequencer) Start(id uint) {
+	s.mu.Lock()
+	if s.id == id {
+		s.mu.Unlock()
+		return
+	}
+	c := make(chan uint)
+	if s.wait == nil {
+		s.wait = make(map[uint]chan uint)
+	}
+	s.wait[id] = c
+	s.mu.Unlock()
+	<-c
+}
+
+// End notifies the sequencer that the event numbered id has completed,
+// allowing it to schedule the event numbered id+1.  It is a run-time error
+// to call End with an id that is not the number of the active event.
+func (s *sequencer) End(id uint) {
+	s.mu.Lock()
+	if s.id != id {
+		panic("out of sync")
+	}
+	id++
+	s.id = id
+	if s.wait == nil {
+		s.wait = make(map[uint]chan uint)
+	}
+	c, ok := s.wait[id]
+	if ok {
+		s.wait[id] = nil, false
+	}
+	s.mu.Unlock()
+	if ok {
+		c <- 1
+	}
+}

src/pkg/net/textproto/reader_test.go

+// Copyright 2010 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package textproto
+
+import (
+	"bufio"
+	"io"
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+type canonicalHeaderKeyTest struct {
+	in, out string
+}
+
+var canonicalHeaderKeyTests = []canonicalHeaderKeyTest{
+	canonicalHeaderKeyTest{"a-b-c", "A-B-C"},
+	canonicalHeaderKeyTest{"a-1-c", "A-1-C"},
+	canonicalHeaderKeyTest{"User-Agent", "User-Agent"},
+	canonicalHeaderKeyTest{"uSER-aGENT", "User-Agent"},
+	canonicalHeaderKeyTest{"user-agent", "User-Agent"},
+	canonicalHeaderKeyTest{"USER-AGENT", "User-Agent"},
+}
+
+func TestCanonicalHeaderKey(t *testing.T) {
+	for _, tt := range canonicalHeaderKeyTests {
+		if s := CanonicalHeaderKey(tt.in); s != tt.out {
+			t.Errorf("CanonicalHeaderKey(%q) = %q, want %q", tt.in, s, tt.out)
+		}
+	}
+}
+
+func reader(s string) *Reader {
+	return NewReader(bufio.NewReader(strings.NewReader(s)))
+}
+
+func TestReadLine(t *testing.T) {
+	r := reader("line1\nline2\n")
+	s, err := r.ReadLine()
+	if s != "line1" || err != nil {
+		t.Fatalf("Line 1: %s, %v", s, err)
+	}
+	s, err = r.ReadLine()
+	if s != "line2" || err != nil {
+		t.Fatalf("Line 2: %s, %v", s, err)
+	}
+	s, err = r.ReadLine()
+	if s != "" || err != os.EOF {
+		t.Fatalf("EOF: %s, %v", s, err)
+	}
+}
+
+func TestReadContinuedLine(t *testing.T) {
+	r := reader("line1\nline\n 2\nline3\n")
+	s, err := r.ReadContinuedLine()
+	if s != "line1" || err != nil {
+		t.Fatalf("Line 1: %s, %v", s, err)
+	}
+	s, err = r.ReadContinuedLine()
+	if s != "line 2" || err != nil {
+		t.Fatalf("Line 2: %s, %v", s, err)
+	}
+	s, err = r.ReadContinuedLine()
+	if s != "line3" || err != nil {
+		t.Fatalf("Line 3: %s, %v", s, err)
+	}
+	s, err = r.ReadContinuedLine()
+	if s != "" || err != os.EOF {
+		t.Fatalf("EOF: %s, %v", s, err)
+	}
+}
+
+func TestReadCodeLine(t *testing.T) {
+	r := reader("123 hi\n234 bye\n345 no way\n")
+	code, msg, err := r.ReadCodeLine(0)
+	if code != 123 || msg != "hi" || err != nil {
+		t.Fatalf("Line 1: %d, %s, %v", code, msg, err)
+	}
+	code, msg, err = r.ReadCodeLine(23)
+	if code != 234 || msg != "bye" || err != nil {
+		t.Fatalf("Line 2: %d, %s, %v", code, msg, err)
+	}
+	code, msg, err = r.ReadCodeLine(346)
+	if code != 345 || msg != "no way" || err == nil {
+		t.Fatalf("Line 3: %d, %s, %v", code, msg, err)
+	}
+	if e, ok := err.(*Error); !ok || e.Code != code || e.Msg != msg {
+		t.Fatalf("Line 3: wrong error %v\n", err)
+	}
+	code, msg, err = r.ReadCodeLine(1)
+	if code != 0 || msg != "" || err != os.EOF {
+		t.Fatalf("EOF: %d, %s, %v", code, msg, err)
+	}
+}
+
+func TestReadDotLines(t *testing.T) {
+	r := reader("dotlines\r\n.foo\r\n..bar\n...baz\nquux\r\n\r\n.\r\nanother\n")
+	s, err := r.ReadDotLines()
+	want := []string{"dotlines", "foo", ".bar", "..baz", "quux", ""}
+	if !reflect.DeepEqual(s, want) || err != nil {
+		t.Fatalf("ReadDotLines: %v, %v", s, err)
+	}
+
+	s, err = r.ReadDotLines()
+	want = []string{"another"}
+	if !reflect.DeepEqual(s, want) || err != io.ErrUnexpectedEOF {
+		t.Fatalf("ReadDotLines2: %v, %v", s, err)
+	}
+}
+
+func TestReadDotBytes(t *testing.T) {
+	r := reader("dotlines\r\n.foo\r\n..bar\n...baz\nquux\r\n\r\n.\r\nanot.her\r\n")
+	b, err := r.ReadDotBytes()
+	want := []byte("dotlines\nfoo\n.bar\n..baz\nquux\n\n")
+	if !reflect.DeepEqual(b, want) || err != nil {
+		t.Fatalf("ReadDotBytes: %q, %v", b, err)
+	}
+
+	b, err = r.ReadDotBytes()
+	want = []byte("anot.her\n")
+	if !reflect.DeepEqual(b, want) || err != io.ErrUnexpectedEOF {
+		t.Fatalf("ReadDotBytes2: %q, %v", b, err)
+	}
+}
+
+func TestReadMIMEHeader(t *testing.T) {
+	r := reader("my-key: Value 1  \r\nLong-key: Even \n Longer Value\r\nmy-Key: Value 2\r\n\n")
+	m, err := r.ReadMIMEHeader()
+	want := map[string][]string{
+		"My-Key":   []string{"Value 1", "Value 2"},
+		"Long-Key": []string{"Even Longer Value"},
+	}
+	if !reflect.DeepEqual(m, want) || err != nil {
+		t.Fatalf("ReadMIMEHeader: %v, %v; want %v", m, err, want)
+	}
+}

src/pkg/net/textproto/textproto.go

+// Copyright 2010 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The textproto package implements generic support for
+// text-based request/response protocols in the style of
+// HTTP, NNTP, and SMTP.
+//
+// The package provides:
+//
+// Error, which represents a numeric error response from
+// a server.
+//
+// Pipeline, to manage pipelined requests and responses
+// in a client.
+//
+// Reader, to read numeric response code lines,
+// key: value headers, lines wrapped with leading spaces
+// on continuation lines, and whole text blocks ending
+// with a dot on a line by itself.
+//
+// Writer, to write dot-encoded text blocks.
+//
+package textproto
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"net"
+	"os"
+)
+
+// An Error represents a numeric error response from a server.
+type Error struct {
+	Code int
+	Msg  string
+}
+
+func (e *Error) String() string {
+	return fmt.Sprintf("%03d %s", e.Code, e.Msg)
+}
+
+// A ProtocolError describes a protocol violation such
+// as an invalid response or a hung-up connection.
+type ProtocolError string
+
+func (p ProtocolError) String() string {
+	return string(p)
+}
+
+// A Conn represents a textual network protocol connection.
+// It consists of a Reader and Writer to manage I/O
+// and a Pipeline to sequence concurrent requests on the connection.
+// These embedded types carry methods with them;
+// see the documentation of those types for details.
+type Conn struct {
+	Reader
+	Writer
+	Pipeline
+	conn io.ReadWriteCloser
+}
+
+// NewConn returns a new Conn using conn for I/O.
+func NewConn(conn io.ReadWriteCloser) *Conn {
+	return &Conn{
+		Reader: Reader{R: bufio.NewReader(conn)},
+		Writer: Writer{W: bufio.NewWriter(conn)},
+		conn:   conn,
+	}
+}
+
+// Close closes the connection.
+func (c *Conn) Close() os.Error {
+	return c.conn.Close()
+}
+
+// Dial connects to the given address on the given network using net.Dial
+// and then returns a new Conn for the connection.
+func Dial(network, addr string) (*Conn, os.Error) {
+	c, err := net.Dial(network, "", addr)
+	if err != nil {
+		return nil, err
+	}
+	return NewConn(c), nil
+}
+
+// Cmd is a convenience method that sends a command after
+// waiting its turn in the pipeline.  The command text is the
+// result of formatting format with args and appending \r\n.
+// Cmd returns the id of the command, for use with StartResponse and EndResponse.
+//
+// For example, a client might run a HELP command that returns a dot-body
+// by using:
+//
+//	id, err := c.Cmd("HELP")
+//	if err != nil {
+//		return nil, err
+//	}
+//
+//	c.StartResponse(id)
+//	defer c.EndResponse(id)
+//
+//	if _, _, err = c.ReadCodeLine(110); err != nil {
+//		return nil, err
+//	}
+//	text, err := c.ReadDotAll()
+//	if err != nil {
+//		return nil, err
+//	}
+//	return c.ReadCodeLine(250)
+//
+func (c *Conn) Cmd(format string, args ...interface{}) (id uint, err os.Error) {
+	id = c.Next()
+	c.StartRequest(id)
+	err = c.PrintfLine(format, args)
+	c.EndRequest(id)
+	if err != nil {
+		return 0, err
+	}
+	return id, nil
+}

src/pkg/net/textproto/writer.go

+// Copyright 2010 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package textproto
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+)
+
+// A Writer implements convenience methods for writing
+// requests or responses to a text protocol network connection.
+type Writer struct {
+	W   *bufio.Writer
+	dot *dotWriter
+}
+
+// NewWriter returns a new Writer writing to w.
+func NewWriter(w *bufio.Writer) *Writer {
+	return &Writer{W: w}
+}
+
+var crnl = []byte{'\r', '\n'}
+var dotcrnl = []byte{'.', '\r', '\n'}
+
+// PrintfLine writes the formatted output followed by \r\n.
+func (w *Writer) PrintfLine(format string, args ...interface{}) os.Error {
+	w.closeDot()
+	fmt.Fprintf(w.W, format, args)
+	w.W.Write(crnl)
+	return w.W.Flush()
+}
+
+// DotWriter returns a writer that can be used to write a dot-encoding to w.
+// It takes care of inserting leading dots when necessary,
+// translating line-ending \n into \r\n, and adding the final .\r\n line
+// when the DotWriter is closed.  The caller should close the
+// DotWriter before the next call to a method on w.
+//
+// See the documentation for Reader's DotReader method for details about dot-encoding.
+func (w *Writer) DotWriter() io.WriteCloser {
+	w.closeDot()
+	w.dot = &dotWriter{w: w}
+	return w.dot
+}
+
+func (w *Writer) closeDot() {
+	if w.dot != nil {
+		w.dot.Close() // sets w.dot = nil
+	}
+}
+
+type dotWriter struct {
+	w     *Writer
+	state int
+}
+
+const (
+	wstateBeginLine = iota // beginning of line; initial state; must be zero
+	wstateCR               // wrote \r (possibly at end of line)
+	wstateData             // writing data in middle of line
+)
+
+func (d *dotWriter) Write(b []byte) (n int, err os.Error) {
+	bw := d.w.W
+	for n < len(b) {
+		c := b[n]
+		switch d.state {
+		case wstateBeginLine:
+			d.state = wstateData
+			if c == '.' {
+				// escape leading dot
+				bw.WriteByte('.')
+			}
+			fallthrough
+
+		case wstateData:
+			if c == '\r' {
+				d.state = wstateCR
+			}
+			if c == '\n' {
+				bw.WriteByte('\r')
+				d.state = wstateBeginLine
+			}
+
+		case wstateCR:
+			d.state = wstateData
+			if c == '\n' {
+				d.state = wstateBeginLine
+			}
+		}
+		if err = bw.WriteByte(c); err != nil {
+			break
+		}
+		n++
+	}
+	return
+}
+
+func (d *dotWriter) Close() os.Error {
+	if d.w.dot == d {
+		d.w.dot = nil
+	}
+	bw := d.w.W
+	switch d.state {
+	default:
+		bw.WriteByte('\r')
+		fallthrough
+	case wstateCR:
+		bw.WriteByte('\n')
+		fallthrough
+	case wstateBeginLine:
+		bw.Write(dotcrnl)
+	}
+	return bw.Flush()
+}

src/pkg/net/textproto/writer_test.go

+// Copyright 2010 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package textproto
+
+import (
+	"bufio"
+	"bytes"
+	"testing"
+)
+
+func TestPrintfLine(t *testing.T) {
+	var buf bytes.Buffer
+	w := NewWriter(bufio.NewWriter(&buf))
+	err := w.PrintfLine("foo %d", 123)
+	if s := buf.String(); s != "foo 123\r\n" || err != nil {
+		t.Fatalf("s=%q; err=%s", s, err)
+	}
+}
+
+func TestDotWriter(t *testing.T) {
+	var buf bytes.Buffer
+	w := NewWriter(bufio.NewWriter(&buf))
+	d := w.DotWriter()
+	n, err := d.Write([]byte("abc\n.def\n..ghi\n.jkl\n."))
+	if n != 21 || err != nil {
+		t.Fatalf("Write: %d, %s", n, err)
+	}
+	d.Close()
+	want := "abc\r\n..def\r\n...ghi\r\n..jkl\r\n..\r\n.\r\n"
+	if s := buf.String(); s != want {
+		t.Fatalf("wrote %q", s)
+	}
+}