Commit 92210eef authored by Brad Fitzpatrick's avatar Brad Fitzpatrick

http: client gzip support

R=adg, rsc, bradfitzwork
CC=golang-dev
https://golang.org/cl/4389048
parent c34aadf0
......@@ -6,6 +6,7 @@ package http
import (
"bufio"
"compress/gzip"
"crypto/tls"
"encoding/base64"
"fmt"
......@@ -39,8 +40,9 @@ type Transport struct {
// TODO: tunable on timeout on cached connections
// TODO: optional pipelining
IgnoreEnvironment bool // don't look at environment variables for proxy configuration
DisableKeepAlives bool
IgnoreEnvironment bool // don't look at environment variables for proxy configuration
DisableKeepAlives bool
DisableCompression bool
// MaxIdleConnsPerHost, if non-zero, controls the maximum idle
// (keep-alive) to keep to keep per-host. If zero,
......@@ -474,6 +476,19 @@ func (pc *persistConn) roundTrip(req *Request) (resp *Response, err os.Error) {
pc.mutateRequestFunc(req)
}
// Ask for a compressed version if the caller didn't set their
// own value for Accept-Encoding. We only attempted to
// uncompress the gzip stream if we were the layer that
// requested it.
requestedGzip := false
if !pc.t.DisableCompression && req.Header.Get("Accept-Encoding") == "" {
// Request gzip only, not deflate. Deflate is ambiguous and
// as universally supported anyway.
// See: http://www.gzip.org/zlib/zlib_faq.html#faq38
requestedGzip = true
req.Header.Set("Accept-Encoding", "gzip")
}
pc.lk.Lock()
pc.numExpectedResponses++
pc.lk.Unlock()
......@@ -490,6 +505,19 @@ func (pc *persistConn) roundTrip(req *Request) (resp *Response, err os.Error) {
pc.lk.Lock()
pc.numExpectedResponses--
pc.lk.Unlock()
if re.err == nil && requestedGzip && re.res.Header.Get("Content-Encoding") == "gzip" {
re.res.Header.Del("Content-Encoding")
re.res.Header.Del("Content-Length")
re.res.ContentLength = -1
var err os.Error
re.res.Body, err = gzip.NewReader(re.res.Body)
if err != nil {
pc.close()
return nil, err
}
}
return re.res, re.err
}
......
......@@ -7,6 +7,8 @@
package http_test
import (
"bytes"
"compress/gzip"
"fmt"
. "http"
"http/httptest"
......@@ -24,7 +26,7 @@ var hostPortHandler = HandlerFunc(func(w ResponseWriter, r *Request) {
if r.FormValue("close") == "true" {
w.Header().Set("Connection", "close")
}
fmt.Fprintf(w, "%s", r.RemoteAddr)
w.Write([]byte(r.RemoteAddr))
})
// Two subsequent requests and verify their response is the same.
......@@ -179,7 +181,7 @@ func TestTransportIdleCacheKeys(t *testing.T) {
func TestTransportMaxPerHostIdleConns(t *testing.T) {
ch := make(chan string)
ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) {
fmt.Fprintf(w, "%s", <-ch)
w.Write([]byte(<-ch))
}))
defer ts.Close()
maxIdleConns := 2
......@@ -338,6 +340,7 @@ func TestTransportNilURL(t *testing.T) {
req.Proto = "HTTP/1.1"
req.ProtoMajor = 1
req.ProtoMinor = 1
req.Header = make(Header)
tr := &Transport{}
res, err := tr.RoundTrip(req)
......@@ -349,3 +352,99 @@ func TestTransportNilURL(t *testing.T) {
t.Fatalf("Expected response body of %q; got %q", e, g)
}
}
func TestTransportGzip(t *testing.T) {
const testString = "The test string aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) {
if g, e := r.Header.Get("Accept-Encoding"), "gzip"; g != e {
t.Errorf("Accept-Encoding = %q, want %q", g, e)
}
w.Header().Set("Content-Encoding", "gzip")
gz, _ := gzip.NewWriter(w)
defer gz.Close()
gz.Write([]byte(testString))
}))
defer ts.Close()
c := &Client{Transport: &Transport{}}
res, _, err := c.Get(ts.URL)
if err != nil {
t.Fatal(err)
}
body, err := ioutil.ReadAll(res.Body)
if err != nil {
t.Fatal(err)
}
if g, e := string(body), testString; g != e {
t.Fatalf("body = %q; want %q", g, e)
}
if g, e := res.Header.Get("Content-Encoding"), ""; g != e {
t.Fatalf("Content-Encoding = %q; want %q", g, e)
}
}
// TestTransportGzipRecursive sends a gzip quine and checks that the
// client gets the same value back. This is more cute than anything,
// but checks that we don't recurse forever, and checks that
// Content-Encoding is removed.
func TestTransportGzipRecursive(t *testing.T) {
ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) {
w.Header().Set("Content-Encoding", "gzip")
w.Write(rgz)
}))
defer ts.Close()
c := &Client{Transport: &Transport{}}
res, _, err := c.Get(ts.URL)
if err != nil {
t.Fatal(err)
}
body, err := ioutil.ReadAll(res.Body)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(body, rgz) {
t.Fatalf("Incorrect result from recursive gz:\nhave=%x\nwant=%x",
body, rgz)
}
if g, e := res.Header.Get("Content-Encoding"), ""; g != e {
t.Fatalf("Content-Encoding = %q; want %q", g, e)
}
}
// rgz is a gzip quine that uncompresses to itself.
var rgz = []byte{
0x1f, 0x8b, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73,
0x69, 0x76, 0x65, 0x00, 0x92, 0xef, 0xe6, 0xe0,
0x60, 0x00, 0x83, 0xa2, 0xd4, 0xe4, 0xd2, 0xa2,
0xe2, 0xcc, 0xb2, 0x54, 0x06, 0x00, 0x00, 0x17,
0x00, 0xe8, 0xff, 0x92, 0xef, 0xe6, 0xe0, 0x60,
0x00, 0x83, 0xa2, 0xd4, 0xe4, 0xd2, 0xa2, 0xe2,
0xcc, 0xb2, 0x54, 0x06, 0x00, 0x00, 0x17, 0x00,
0xe8, 0xff, 0x42, 0x12, 0x46, 0x16, 0x06, 0x00,
0x05, 0x00, 0xfa, 0xff, 0x42, 0x12, 0x46, 0x16,
0x06, 0x00, 0x05, 0x00, 0xfa, 0xff, 0x00, 0x05,
0x00, 0xfa, 0xff, 0x00, 0x14, 0x00, 0xeb, 0xff,
0x42, 0x12, 0x46, 0x16, 0x06, 0x00, 0x05, 0x00,
0xfa, 0xff, 0x00, 0x05, 0x00, 0xfa, 0xff, 0x00,
0x14, 0x00, 0xeb, 0xff, 0x42, 0x88, 0x21, 0xc4,
0x00, 0x00, 0x14, 0x00, 0xeb, 0xff, 0x42, 0x88,
0x21, 0xc4, 0x00, 0x00, 0x14, 0x00, 0xeb, 0xff,
0x42, 0x88, 0x21, 0xc4, 0x00, 0x00, 0x14, 0x00,
0xeb, 0xff, 0x42, 0x88, 0x21, 0xc4, 0x00, 0x00,
0x14, 0x00, 0xeb, 0xff, 0x42, 0x88, 0x21, 0xc4,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00,
0x00, 0xff, 0xff, 0x00, 0x17, 0x00, 0xe8, 0xff,
0x42, 0x88, 0x21, 0xc4, 0x00, 0x00, 0x00, 0x00,
0xff, 0xff, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00,
0x17, 0x00, 0xe8, 0xff, 0x42, 0x12, 0x46, 0x16,
0x06, 0x00, 0x00, 0x00, 0xff, 0xff, 0x01, 0x08,
0x00, 0xf7, 0xff, 0x3d, 0xb1, 0x20, 0x85, 0xfa,
0x00, 0x00, 0x00, 0x42, 0x12, 0x46, 0x16, 0x06,
0x00, 0x00, 0x00, 0xff, 0xff, 0x01, 0x08, 0x00,
0xf7, 0xff, 0x3d, 0xb1, 0x20, 0x85, 0xfa, 0x00,
0x00, 0x00, 0x3d, 0xb1, 0x20, 0x85, 0xfa, 0x00,
0x00, 0x00,
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment