http: sniffing algorithm.

This follows draft-ietf-websec-mime-sniff-03 in its intent, though not its algorithmic specification. R=rsc CC=golang-dev https://golang.org/cl/4746042

http: sniffing algorithm.
This follows draft-ietf-websec-mime-sniff-03 in its intent, though not its algorithmic specification. R=rsc CC=golang-dev https://golang.org/cl/4746042
78c89d21 · David Symonds · 7911965f · 78c89d21 · 78c89d21 · 78c89d21
Commit 78c89d21 authored Jul 21, 2011 by David Symonds
Hide whitespace changes
Inline Side-by-side

Showing with 205 additions and 10 deletions

src/pkg/http/server.go src/pkg/http/server.go +1 -4

src/pkg/http/sniff.go src/pkg/http/sniff.go +164 -6

src/pkg/http/sniff_test.go src/pkg/http/sniff_test.go +40 -0

No files found.
--- a/src/pkg/http/server.go
+++ b/src/pkg/http/server.go
@@ -359,10 +359,7 @@ func (w *response) sniff() {
 	w.needSniff = false

 	data := w.conn.body
-	ctype := detectContentType(data)
-	if ctype != "" {
-		fmt.Fprintf(w.conn.buf, "Content-Type: %s\r\n", ctype)
-	}
+	fmt.Fprintf(w.conn.buf, "Content-Type: %s\r\n", DetectContentType(data))
 	io.WriteString(w.conn.buf, "\r\n")

 	if w.chunking && len(data) > 0 {

--- a/src/pkg/http/sniff.go
+++ b/src/pkg/http/sniff.go
@@ -4,15 +4,173 @@

 package http

+import (
+	"bytes"
+)
+
 // Content-type sniffing algorithm.
-// http://tools.ietf.org/html/draft-ietf-websec-mime-sniff-03
+// References in this file refer to this draft specification:
+//   http://tools.ietf.org/html/draft-ietf-websec-mime-sniff-03

 // The algorithm prefers to use sniffLen bytes to make its decision.
-const sniffLen = 1024
+const sniffLen = 512
+
+// DetectContentType returns the sniffed Content-Type string
+// for the given data. This function always returns a valid MIME type.
+func DetectContentType(data []byte) string {
+	if len(data) > sniffLen {
+		data = data[:sniffLen]
+	}
+
+	// Index of the first non-whitespace byte in data.
+	firstNonWS := 0
+	for ; firstNonWS < len(data) && isWS(data[firstNonWS]); firstNonWS++ {
+	}
+
+	for _, sig := range sniffSignatures {
+		if ct := sig.match(data, firstNonWS); ct != "" {
+			return ct
+		}
+	}
+
+	return "application/octet-stream" // fallback
+}
+
+func isWS(b byte) bool {
+	return bytes.IndexByte([]byte("\t\n\x0C\n "), b) != -1
+}
+
+type sniffSig interface {
+	// match returns the MIME type of the data, or "" if unknown.
+	match(data []byte, firstNonWS int) string
+}
+
+// Data matching the table in section 6.
+var sniffSignatures = []sniffSig{
+	htmlSig([]byte("<!DOCTYPE HTML")),
+	htmlSig([]byte("<HTML")),
+	htmlSig([]byte("<HEAD")),
+	htmlSig([]byte("<SCRIPT")),
+	htmlSig([]byte("<IFRAME")),
+	htmlSig([]byte("<H1")),
+	htmlSig([]byte("<DIV")),
+	htmlSig([]byte("<FONT")),
+	htmlSig([]byte("<TABLE")),
+	htmlSig([]byte("<A")),
+	htmlSig([]byte("<STYLE")),
+	htmlSig([]byte("<TITLE")),
+	htmlSig([]byte("<B")),
+	htmlSig([]byte("<BODY")),
+	htmlSig([]byte("<BR")),
+	htmlSig([]byte("<P")),
+	htmlSig([]byte("<!--")),
+
+	&maskedSig{mask: []byte("\xFF\xFF\xFF\xFF\xFF"), pat: []byte("<?xml"), skipWS: true, ct: "text/xml; charset=utf-8"},
+
+	&exactSig{[]byte("%PDF-"), "application/pdf"},
+	&exactSig{[]byte("%!PS-Adobe-"), "application/postscript"},
+
+	// UTF BOMs.
+	&maskedSig{mask: []byte("\xFF\xFF\x00\x00"), pat: []byte("\xFE\xFF\x00\x00"), ct: "text/plain; charset=utf-16be"},
+	&maskedSig{mask: []byte("\xFF\xFF\x00\x00"), pat: []byte("\xFF\xFE\x00\x00"), ct: "text/plain; charset=utf-16le"},
+	&maskedSig{mask: []byte("\xFF\xFF\xFF\x00"), pat: []byte("\xEF\xBB\xBF\x00"), ct: "text/plain; charset=utf-8"},
+
+	&exactSig{[]byte("GIF87a"), "image/gif"},
+	&exactSig{[]byte("GIF89a"), "image/gif"},
+	&exactSig{[]byte("\x89\x50\x4E\x47\x0D\x0A\x1A\x0A"), "image/png"},
+	&exactSig{[]byte("\xFF\xD8\xFF"), "image/jpeg"},
+	&exactSig{[]byte("BM"), "image/bmp"},
+	&maskedSig{
+		mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF"),
+		pat:  []byte("RIFF\x00\x00\x00\x00WEBPVP"),
+		ct:   "image/webp",
+	},
+	&exactSig{[]byte("\x00\x00\x01\x00"), "image/vnd.microsoft.icon"},
+	&exactSig{[]byte("\x4F\x67\x67\x53\x00"), "application/ogg"},
+	&maskedSig{
+		mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"),
+		pat:  []byte("RIFF\x00\x00\x00\x00WAVE"),
+		ct:   "audio/wave",
+	},
+	&exactSig{[]byte("\x1A\x45\xDF\xA3"), "video/webm"},
+	&exactSig{[]byte("\x52\x61\x72\x20\x1A\x07\x00"), "application/x-rar-compressed"},
+	&exactSig{[]byte("\x50\x4B\x03\x04"), "application/zip"},
+	&exactSig{[]byte("\x1F\x8B\x08"), "application/x-gzip"},

-// detectContentType returns the sniffed Content-Type string
-// for the given data.
-func detectContentType(data []byte) string {
-	// TODO(dsymonds,rsc): Implement algorithm from draft.
+	// TODO(dsymonds): MP4.
+
+	textSig(0), // should be last
+}
+
+type exactSig struct {
+	sig []byte
+	ct  string
+}
+
+func (e *exactSig) match(data []byte, firstNonWS int) string {
+	if bytes.HasPrefix(data, e.sig) {
+		return e.ct
+	}
+	return ""
+}
+
+type maskedSig struct {
+	mask, pat []byte
+	skipWS    bool
+	ct        string
+}
+
+func (m *maskedSig) match(data []byte, firstNonWS int) string {
+	if m.skipWS {
+		data = data[firstNonWS:]
+	}
+	if len(data) < len(m.mask) {
+		return ""
+	}
+	for i, mask := range m.mask {
+		db := data[i] & mask
+		if db != m.pat[i] {
+			return ""
+		}
+	}
+	return m.ct
+}
+
+type htmlSig []byte
+
+func (h htmlSig) match(data []byte, firstNonWS int) string {
+	data = data[firstNonWS:]
+	if len(data) < len(h)+1 {
+		return ""
+	}
+	for i, b := range h {
+		db := data[i]
+		if 'A' <= b && b <= 'Z' {
+			db &= 0xDF
+		}
+		if b != db {
+			return ""
+		}
+	}
+	// Next byte must be space or right angle bracket.
+	if db := data[len(h)]; db != ' ' && db != '>' {
+		return ""
+	}
 	return "text/html; charset=utf-8"
 }
+
+type textSig int
+
+func (textSig) match(data []byte, firstNonWS int) string {
+	// c.f. section 5, step 4.
+	for _, b := range data[firstNonWS:] {
+		switch {
+		case 0x00 <= b && b <= 0x08,
+			b == 0x0B,
+			0x0E <= b && b <= 0x1A,
+			0x1C <= b && b <= 0x1F:
+			return ""
+		}
+	}
+	return "text/plain; charset=utf-8"
+}
--- a/src/pkg/http/sniff_test.go
+++ b/src/pkg/http/sniff_test.go
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package http
+
+import (
+	"testing"
+)
+
+var sniffTests = []struct {
+	desc string
+	data []byte
+	exp  string
+}{
+	// Some nonsense.
+	{"Empty", []byte{}, "text/plain; charset=utf-8"},
+	{"Binary", []byte{1, 2, 3}, "application/octet-stream"},
+
+	{"HTML document #1", []byte(`<HtMl><bOdY>blah blah blah</body></html>`), "text/html; charset=utf-8"},
+	{"HTML document #2", []byte(`<HTML></HTML>`), "text/html; charset=utf-8"},
+	{"HTML document #3 (leading whitespace)", []byte(`   <!DOCTYPE HTML>...`), "text/html; charset=utf-8"},
+
+	{"Plain text", []byte(`This is not HTML. It has ☃ though.`), "text/plain; charset=utf-8"},
+
+	{"XML", []byte("\n<?xml!"), "text/xml; charset=utf-8"},
+
+	// Image types.
+	{"GIF 87a", []byte(`GIF87a`), "image/gif"},
+	{"GIF 89a", []byte(`GIF89a...`), "image/gif"},
+}
+
+func TestSniffing(t *testing.T) {
+	for _, st := range sniffTests {
+		got := DetectContentType(st.data)
+		if got != st.exp {
+			t.Errorf("%v: sniffed as %v, want %v", st.desc, got, st.exp)
+		}
+	}
+}